diff --git a/src/codegen/buffering_consumer.cpp b/src/codegen/buffering_consumer.cpp index 55300ee6a4f..bfd7dd1d5c6 100644 --- a/src/codegen/buffering_consumer.cpp +++ b/src/codegen/buffering_consumer.cpp @@ -78,6 +78,46 @@ void BufferingConsumer::Prepare(CompilationContext &ctx) { runtime_state.RegisterState("consumerState", codegen.CharPtrType()); } +void BufferingConsumer::AddToTupleBuffer(Value &val, CodeGen &codegen, llvm::Value *tuple_buffer, size_t &i) { + const auto &sql_type = val.GetType().GetSqlType(); + + // Check if it's NULL + Value null_val; + lang::If val_is_null{codegen, val.IsNull(codegen)}; + { + // If the value is NULL (i.e., has the NULL bit set), produce the NULL + // value for the given type. + null_val = sql_type.GetNullValue(codegen); + } + val_is_null.EndIf(); + val = val_is_null.BuildPHI(null_val, val); + + // Output the value using the type's output function + auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); + + // Setup the function arguments + std::vector args = {tuple_buffer, codegen.Const32(i), + val.GetValue()}; + // If the value is a string, push back the length + if (val.GetLength() != nullptr) { + args.push_back(val.GetLength()); + } + + // If the value is a boolean, push back the NULL bit. We don't do that for + // the other data types because we have special values for NULL. Booleans + // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the + // system. Since, we cannot have a special value for NULL in a 1-bit boolean + // system, we pass along the NULL bit during output. + if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { + args.push_back(val.IsNull(codegen)); + } + + // Call the function + codegen.CallFunc(output_func, args); +} + + + // For each output attribute, we write out the attribute's value into the // currently active output tuple. When all attributes have been written, we // call BufferTuple(...) to append the currently active tuple into the output. @@ -95,41 +135,7 @@ void BufferingConsumer::ConsumeResult(ConsumerContext &ctx, Value val = row.DeriveValue(codegen, output_ais_[i]); PELOTON_ASSERT(output_ais_[i]->type == val.GetType()); - const auto &sql_type = val.GetType().GetSqlType(); - - // Check if it's NULL - Value null_val; - lang::If val_is_null{codegen, val.IsNull(codegen)}; - { - // If the value is NULL (i.e., has the NULL bit set), produce the NULL - // value for the given type. - null_val = sql_type.GetNullValue(codegen); - } - val_is_null.EndIf(); - val = val_is_null.BuildPHI(null_val, val); - - // Output the value using the type's output function - auto *output_func = sql_type.GetOutputFunction(codegen, val.GetType()); - - // Setup the function arguments - std::vector args = {tuple_buffer_, codegen.Const32(i), - val.GetValue()}; - // If the value is a string, push back the length - if (val.GetLength() != nullptr) { - args.push_back(val.GetLength()); - } - - // If the value is a boolean, push back the NULL bit. We don't do that for - // the other data types because we have special values for NULL. Booleans - // in codegen are 1-bit types, as opposed to 1-byte types in the rest of the - // system. Since, we cannot have a special value for NULL in a 1-bit boolean - // system, we pass along the NULL bit during output. - if (sql_type.TypeId() == peloton::type::TypeId::BOOLEAN) { - args.push_back(val.IsNull(codegen)); - } - - // Call the function - codegen.CallFunc(output_func, args); + AddToTupleBuffer(val, codegen, tuple_buffer_, i); } // Append the tuple to the output buffer (by calling BufferTuple(...)) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index ce652cf746e..e068edce22d 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -14,12 +14,10 @@ #include "codegen/transaction_runtime.h" #include "common/container_tuple.h" #include "concurrency/transaction_manager_factory.h" -#include "executor/executor_context.h" #include "executor/logical_tile.h" #include "executor/logical_tile_factory.h" -#include "storage/data_table.h" -#include "storage/tile_group.h" #include "storage/tile.h" +#include "threadpool/logger_queue_pool.h" namespace peloton { namespace codegen { @@ -48,7 +46,7 @@ peloton::type::AbstractPool *Inserter::GetPool() { return tile_->GetPool(); } -void Inserter::Insert() { +void Inserter::Insert(char *values_buf, uint32_t values_size) { PELOTON_ASSERT(table_ && executor_context_ && tile_); auto *txn = executor_context_->GetTransaction(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); @@ -61,7 +59,7 @@ void Inserter::Insert() { txn_manager.SetTransactionResult(txn, ResultType::FAILURE); return; } - txn_manager.PerformInsert(txn, location_, index_entry_ptr); + txn_manager.PerformInsert(txn, location_, index_entry_ptr, values_buf, values_size); executor_context_->num_processed++; } diff --git a/src/codegen/operator/insert_translator.cpp b/src/codegen/operator/insert_translator.cpp index 3e36d8c8e68..756edbe2c23 100644 --- a/src/codegen/operator/insert_translator.cpp +++ b/src/codegen/operator/insert_translator.cpp @@ -10,11 +10,14 @@ // //===----------------------------------------------------------------------===// +#include "codegen/buffering_consumer.h" #include "codegen/proxy/inserter_proxy.h" #include "codegen/proxy/query_parameters_proxy.h" #include "codegen/proxy/storage_manager_proxy.h" #include "codegen/proxy/transaction_runtime_proxy.h" #include "codegen/proxy/tuple_proxy.h" +#include "codegen/proxy/value_proxy.h" +#include "codegen/proxy/values_runtime_proxy.h" #include "codegen/operator/insert_translator.h" #include "planner/insert_plan.h" #include "storage/data_table.h" @@ -79,15 +82,23 @@ void InsertTranslator::Produce() const { // Transform into the codegen values and store values in the tuple storage std::vector values; - for (uint32_t column_id = 0; column_id < num_columns; column_id++) { + auto *values_buf = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), num_columns, + "values"); + values_buf = + codegen->CreatePointerCast(values_buf, codegen.CharPtrType()); + llvm::Value *values_size = codegen.Const32((int32_t)num_columns); + for (size_t column_id = 0; column_id < num_columns; column_id++) { auto value = parameter_cache.GetValue(column_id + tuple_idx * num_columns); values.push_back(value); + peloton::codegen::BufferingConsumer::AddToTupleBuffer(value, codegen, values_buf, column_id); } table_storage_.StoreValues(codegen, tuple_ptr, values, pool); + std::vector insert_args = {inserter, values_buf, values_size}; // Complete the insertion - codegen.Call(InserterProxy::Insert, {inserter}); + codegen.Call(InserterProxy::Insert, insert_args); } } } @@ -103,14 +114,25 @@ void InsertTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { // Generate/Materialize tuple data from row and attribute information std::vector values; auto &ais = insert_plan_.GetAttributeInfos(); + auto *values_buf = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), static_cast(ais.size()), + "values"); + values_buf = + codegen->CreatePointerCast(values_buf, codegen.CharPtrType()); + llvm::Value *values_size = codegen.Const32((int32_t)ais.size()); + size_t i = 0; for (const auto *ai : ais) { codegen::Value v = row.DeriveValue(codegen, ai); values.push_back(v); + peloton::codegen::BufferingConsumer::AddToTupleBuffer(v, codegen, values_buf, i); + i++; } table_storage_.StoreValues(codegen, tuple_ptr, values, pool); + std::vector insert_args = {inserter, values_buf, values_size}; + // Call Inserter to insert the reserved tuple storage area - codegen.Call(InserterProxy::Insert, {inserter}); + codegen.Call(InserterProxy::Insert, insert_args); } void InsertTranslator::TearDownState() { diff --git a/src/codegen/operator/update_translator.cpp b/src/codegen/operator/update_translator.cpp index e8dd905d569..54c8a38b8ba 100644 --- a/src/codegen/operator/update_translator.cpp +++ b/src/codegen/operator/update_translator.cpp @@ -10,12 +10,16 @@ // //===----------------------------------------------------------------------===// +#include "codegen/buffering_consumer.h" #include "codegen/lang/if.h" #include "codegen/proxy/storage_manager_proxy.h" #include "codegen/proxy/target_proxy.h" #include "codegen/proxy/updater_proxy.h" +#include "codegen/proxy/value_proxy.h" +#include "codegen/proxy/values_runtime_proxy.h" #include "codegen/operator/update_translator.h" #include "codegen/table_storage.h" +#include "codegen/type/sql_type.h" #include "planner/update_plan.h" #include "storage/data_table.h" @@ -92,17 +96,45 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { static_cast(target_list.size() + direct_map_list.size()); auto &ais = update_plan_.GetAttributeInfos(); + auto is_primary_key = update_plan_.GetUpdatePrimaryKey(); + llvm::Value *diff = nullptr, *diff_size = nullptr; + if (is_primary_key) { + diff_size = codegen.Const32((int32_t)column_num); + diff = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), column_num, + "diff"); + } else { + diff_size = codegen.Const32((int32_t)target_list.size()); + diff = codegen.AllocateBuffer( + ValueProxy::GetType(codegen), static_cast(target_list.size()), + "diff"); + } + + + diff = + codegen->CreatePointerCast(diff, codegen.CharPtrType()); // Collect all the column values std::vector values; - for (uint32_t i = 0; i < column_num; i++) { + for (size_t i = 0, target_id = 0; i < column_num; i++) { codegen::Value val; uint32_t target_index = GetTargetIndex(target_list, i); if (target_index != INVALID_OID) { // Set the value for the update const auto &derived_attribute = target_list[target_index].second; val = row.DeriveValue(codegen, *derived_attribute.expr); + size_t offset = 0; + if (is_primary_key) { + offset = i; + } else { + offset = target_id; + } + peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, offset); + target_id++; } else { val = row.DeriveValue(codegen, ais[i]); + if (is_primary_key) { + peloton::codegen::BufferingConsumer::AddToTupleBuffer(val, codegen, diff, i); + } } values.push_back(val); } @@ -112,7 +144,7 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { llvm::Value *tuple_ptr; std::vector prep_args = {updater, row.GetTileGroupID(), row.GetTID(codegen)}; - if (update_plan_.GetUpdatePrimaryKey() == false) { + if (is_primary_key == false) { tuple_ptr = codegen.Call(UpdaterProxy::Prepare, prep_args); } else { tuple_ptr = codegen.Call(UpdaterProxy::PreparePK, prep_args); @@ -130,8 +162,8 @@ void UpdateTranslator::Consume(ConsumerContext &, RowBatch::Row &row) const { table_storage_.StoreValues(codegen, tuple_ptr, values, pool_ptr); // Finally, update with help from the Updater - std::vector update_args = {updater}; - if (update_plan_.GetUpdatePrimaryKey() == false) { + std::vector update_args = {updater, diff, diff_size}; + if (is_primary_key == false) { codegen.Call(UpdaterProxy::Update, update_args); } else { codegen.Call(UpdaterProxy::UpdatePK, update_args); diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 88ef489aed1..0a25502fbdc 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -9,7 +9,7 @@ // Copyright (c) 2015-17, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - +#include "codegen/buffering_consumer.h" #include "codegen/updater.h" #include "codegen/transaction_runtime.h" #include "common/container_tuple.h" @@ -23,6 +23,13 @@ #include "type/abstract_pool.h" #include "common/internal_types.h" #include "type/value.h" +#include "threadpool/mono_queue_pool.h" +#include "logging/log_record.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" +#include "threadpool/logger_queue_pool.h" +#include "../include/type/value.h" +#include "../include/codegen/value.h" namespace peloton { namespace codegen { @@ -126,7 +133,7 @@ peloton::type::AbstractPool *Updater::GetPool() { return tile_->GetPool(); } -void Updater::Update() { +void Updater::Update(char *diff_array, uint32_t diff_size) { PELOTON_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, @@ -138,7 +145,7 @@ void Updater::Update() { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); // Either update in-place if (is_owner_ == true) { - txn_manager.PerformUpdate(txn, old_location_); + txn_manager.PerformUpdate(txn, old_location_, diff_array, diff_size, target_list_); // we do not need to add any item pointer to statement-level write set // here, because we do not generate any new version executor_context_->num_processed++; @@ -157,12 +164,13 @@ void Updater::Update() { old_location_.offset); return; } - txn_manager.PerformUpdate(txn, old_location_, new_location_); - AddToStatementWriteSet(new_location_); + txn_manager.PerformUpdate(txn, old_location_, new_location_, diff_array, diff_size, target_list_); + executor_context_->num_processed++; + AddToStatementWriteSet(new_location_); } -void Updater::UpdatePK() { +void Updater::UpdatePK(char *diff_array, uint32_t diff_size) { PELOTON_ASSERT(table_ != nullptr && executor_context_ != nullptr); LOG_TRACE("Updating tuple <%u, %u> from table '%s' (db ID: %u, table ID: %u)", old_location_.block, old_location_.offset, @@ -181,7 +189,7 @@ void Updater::UpdatePK() { txn_manager.SetTransactionResult(txn, ResultType::FAILURE); return; } - txn_manager.PerformInsert(txn, new_location_, index_entry_ptr); + txn_manager.PerformInsert(txn, new_location_, index_entry_ptr, diff_array, diff_size); AddToStatementWriteSet(new_location_); executor_context_->num_processed++; } diff --git a/src/common/init.cpp b/src/common/init.cpp index c4ab64adc51..318f1aa5464 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -26,6 +26,7 @@ #include "tuning/index_tuner.h" #include "tuning/layout_tuner.h" + namespace peloton { ThreadPool thread_pool; @@ -90,6 +91,21 @@ void PelotonInit::Initialize() { // Initialize the Statement Cache Manager StatementCacheManager::Init(); + + bool enable_logging = settings::SettingsManager::GetBool(settings::SettingId::enable_logging); + if(enable_logging){ + if(!logging::LogManager::GetInstance().init()){ + LOG_ERROR("LogManager Initialization failed"); + } + } + + bool enable_recovery = settings::SettingsManager::GetBool(settings::SettingId::enable_recovery); + if(enable_recovery){ + logging::LogManager::GetInstance().DoRecovery(); + } + + threadpool::LoggerQueuePool::GetInstance().Startup(); + } void PelotonInit::Shutdown() { diff --git a/src/concurrency/timestamp_ordering_transaction_manager.cpp b/src/concurrency/timestamp_ordering_transaction_manager.cpp index c86c53292e0..fcba33f4cf0 100644 --- a/src/concurrency/timestamp_ordering_transaction_manager.cpp +++ b/src/concurrency/timestamp_ordering_transaction_manager.cpp @@ -12,7 +12,10 @@ #include "concurrency/timestamp_ordering_transaction_manager.h" #include - +#include "catalog/table_catalog.h" +#include "catalog/schema_catalog.h" +#include "catalog/system_catalogs.h" +#include "catalog/catalog.h" #include "catalog/catalog_defaults.h" #include "catalog/manager.h" #include "common/exception.h" @@ -20,7 +23,10 @@ #include "common/platform.h" #include "concurrency/transaction_context.h" #include "gc/gc_manager_factory.h" -#include "logging/log_manager_factory.h" +#include "logging/log_record.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" +#include "threadpool/logger_queue_pool.h" #include "settings/settings_manager.h" namespace peloton { @@ -436,9 +442,8 @@ bool TimestampOrderingTransactionManager::PerformRead( void TimestampOrderingTransactionManager::PerformInsert( TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr) { - PELOTON_ASSERT(current_txn->GetIsolationLevel() != - IsolationLevelType::READ_ONLY); + ItemPointer *index_entry_ptr, char *values_buf, uint32_t values_size) { + PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); oid_t tile_group_id = location.block; oid_t tuple_id = location.offset; @@ -466,6 +471,33 @@ void TimestampOrderingTransactionManager::PerformInsert( // Write down the head pointer's address in tile group header tile_group_header->SetIndirection(tuple_id, index_entry_ptr); + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); + record.SetValuesArray(values_buf, values_size); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_TRACE("Submitting log buffer %p", current_txn->GetLogBuffer()); + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } + } + + // Increment table insert op stats if (static_cast(settings::SettingsManager::GetInt( settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -476,9 +508,9 @@ void TimestampOrderingTransactionManager::PerformInsert( void TimestampOrderingTransactionManager::PerformUpdate( TransactionContext *const current_txn, const ItemPointer &location, - const ItemPointer &new_location) { - PELOTON_ASSERT(current_txn->GetIsolationLevel() != - IsolationLevelType::READ_ONLY); + const ItemPointer &new_location, char *values_buf, + uint32_t values_size, TargetList *offsets) { + PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); ItemPointer old_location = location; @@ -551,6 +583,36 @@ void TimestampOrderingTransactionManager::PerformUpdate( // Add the old tuple into the update set current_txn->RecordUpdate(old_location); + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, new_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); + + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } + } + + // Increment table update op stats if (static_cast(settings::SettingsManager::GetInt( settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -560,11 +622,12 @@ void TimestampOrderingTransactionManager::PerformUpdate( } void TimestampOrderingTransactionManager::PerformUpdate( - TransactionContext *const current_txn UNUSED_ATTRIBUTE, - const ItemPointer &location) { - PELOTON_ASSERT(current_txn->GetIsolationLevel() != - IsolationLevelType::READ_ONLY); + TransactionContext *const current_txn, + const ItemPointer &location, char *values_buf, + uint32_t values_size, TargetList *offsets) { + PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); + oid_t tile_group_id = location.block; UNUSED_ATTRIBUTE oid_t tuple_id = location.offset; @@ -577,6 +640,35 @@ void TimestampOrderingTransactionManager::PerformUpdate( PELOTON_ASSERT(tile_group_header->GetBeginCommitId(tuple_id) == MAX_CID); PELOTON_ASSERT(tile_group_header->GetEndCommitId(tuple_id) == MAX_CID); + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + if (values_buf != nullptr) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, location, + current_txn->GetEpochId(), current_txn->GetTransactionId(), + current_txn->GetCommitId(), schema_oid); + + record.SetOldItemPointer(location); + record.SetValuesArray(values_buf, values_size); + record.SetOffsetsArray(offsets); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } + } + // no need to add the older version into the update set. // if there exists older version, then the older version must already // been added to the update set. @@ -674,6 +766,32 @@ void TimestampOrderingTransactionManager::PerformDelete( current_txn->RecordDelete(old_location); + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, old_location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + + } + // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt( settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -682,6 +800,7 @@ void TimestampOrderingTransactionManager::PerformDelete( } } +// called when the current transaction creates a new version and then deletes that version. void TimestampOrderingTransactionManager::PerformDelete( TransactionContext *const current_txn, const ItemPointer &location) { PELOTON_ASSERT(current_txn->GetIsolationLevel() != @@ -701,14 +820,39 @@ void TimestampOrderingTransactionManager::PerformDelete( // Add the old tuple into the delete set auto old_location = tile_group_header->GetNextItemPointer(tuple_id); - if (old_location.IsNull() == false) { + + if (old_location.IsNull() == false) { // update and delete by the current transaction // if this version is not newly inserted. current_txn->RecordDelete(old_location); - } else { + } else { // insert and delete by the current transaction // if this version is newly inserted. current_txn->RecordDelete(location); } + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + auto tile_group = tile_group_header->GetTileGroup(); + auto table_object = catalog::Catalog::GetInstance()->GetTableObject(tile_group->GetDatabaseId(), tile_group->GetTableId(), current_txn); + auto schema_oid = (catalog::Catalog::GetInstance()->GetSystemCatalogs(tile_group->GetDatabaseId())->GetSchemaCatalog()->GetSchemaObject(table_object->GetSchemaName(), current_txn))->GetSchemaOid(); + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, location, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId(), schema_oid); + + current_txn->GetLogBuffer()->WriteRecord(record); + + if (current_txn->GetLogBuffer()->HasThresholdExceeded()) { + + LOG_DEBUG("Submitting log buffer %p", current_txn->GetLogBuffer()); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + /* allocate a new buffer for the current transaction */ + current_txn->ResetLogBuffer(); + } + } + // Increment table delete op stats if (static_cast(settings::SettingsManager::GetInt( settings::SettingId::stats_mode)) != StatsType::INVALID) { @@ -718,7 +862,7 @@ void TimestampOrderingTransactionManager::PerformDelete( } ResultType TimestampOrderingTransactionManager::CommitTransaction( - TransactionContext *const current_txn) { + TransactionContext *const current_txn, std::function task_callback) { LOG_TRACE("Committing peloton txn : %" PRId64, current_txn->GetTransactionId()); @@ -735,9 +879,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( ////////////////////////////////////////////////////////// auto &manager = catalog::Manager::GetInstance(); - auto &log_manager = logging::LogManager::GetInstance(); - - log_manager.StartLogging(); // generate transaction id. cid_t end_commit_id = current_txn->GetCommitId(); @@ -823,8 +964,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_UPDATE; - log_manager.LogUpdate(new_version); - } else if (tuple_entry.second == RWType::DELETE) { ItemPointer new_version = tile_group_header->GetPrevItemPointer(tuple_slot); @@ -856,8 +995,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_DELETE; - log_manager.LogDelete(ItemPointer(tile_group_id, tuple_slot)); - } else if (tuple_entry.second == RWType::INSERT) { PELOTON_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); @@ -872,8 +1009,6 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( // nothing to be added to gc set. - log_manager.LogInsert(ItemPointer(tile_group_id, tuple_slot)); - } else if (tuple_entry.second == RWType::INS_DEL) { PELOTON_ASSERT(tile_group_header->GetTransactionId(tuple_slot) == current_txn->GetTransactionId()); @@ -890,29 +1025,55 @@ ResultType TimestampOrderingTransactionManager::CommitTransaction( // add to gc set. gc_set->operator[](tile_group_id)[tuple_slot] = GCVersionType::COMMIT_INS_DEL; - - // no log is needed for this case } } ResultType result = current_txn->GetResult(); - log_manager.LogEnd(); - EndTransaction(current_txn); + + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + // no need to log read-only transactions + if (!current_txn->IsReadOnly() && task_callback != nullptr) { + + auto on_flush = [this, result, task_callback]() { + task_callback(result); + }; + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_COMMIT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + result = ResultType::QUEUING; + + } + } + + this->EndTransaction(current_txn); // Increment # txns committed metric if (static_cast(settings::SettingsManager::GetInt( settings::SettingId::stats_mode)) != StatsType::INVALID) { stats::BackendStatsContext::GetInstance()->IncrementTxnCommitted( - database_id); + database_id); } + return result; } ResultType TimestampOrderingTransactionManager::AbortTransaction( - TransactionContext *const current_txn) { + TransactionContext *const current_txn, std::function task_callback) { + // a pre-declared read-only transaction will never abort. PELOTON_ASSERT(current_txn->GetIsolationLevel() != IsolationLevelType::READ_ONLY); @@ -1089,7 +1250,35 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( } current_txn->SetResult(ResultType::ABORTED); - EndTransaction(current_txn); + + ResultType result = current_txn->GetResult(); + + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + // no need to log read-only transactions + if (!current_txn->IsReadOnly() && task_callback != nullptr) { + auto on_flush = [this, result, task_callback]() { + task_callback(result); + }; + + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_ABORT, current_txn->GetEpochId(), + current_txn->GetTransactionId(), current_txn->GetCommitId()); + + current_txn->GetLogBuffer()->WriteRecord(record); + + current_txn->GetLogBuffer()->SetLoggerCallback(on_flush); + + /* insert to the queue */ + threadpool::LoggerQueuePool::GetInstance().SubmitLogBuffer( + current_txn->GetLogToken(), current_txn->GetLogBuffer()); + + + result = ResultType::QUEUING; + } + } + + this->EndTransaction(current_txn); // Increment # txns aborted metric if (static_cast(settings::SettingsManager::GetInt( @@ -1097,7 +1286,7 @@ ResultType TimestampOrderingTransactionManager::AbortTransaction( stats::BackendStatsContext::GetInstance()->IncrementTxnAborted(database_id); } - return ResultType::ABORTED; + return result; } } // namespace concurrency diff --git a/src/concurrency/transaction_context.cpp b/src/concurrency/transaction_context.cpp index 87b8fa235d9..e05e649f091 100644 --- a/src/concurrency/transaction_context.cpp +++ b/src/concurrency/transaction_context.cpp @@ -18,6 +18,7 @@ #include "common/macros.h" #include "common/platform.h" #include "trigger/trigger.h" +#include "logging/log_buffer.h" #include #include @@ -95,11 +96,16 @@ void TransactionContext::Init(const size_t thread_id, insert_count_ = 0; + single_statement_txn_ = true; + rw_set_.Clear(); gc_set_.reset(new GCSet()); gc_object_set_.reset(new GCObjectSet()); on_commit_triggers_.reset(); + + log_token_ = threadpool::LoggerQueuePool::GetInstance().GetLogToken(); + ResetLogBuffer(); } RWType TransactionContext::GetRWType(const ItemPointer &location) { diff --git a/src/concurrency/transaction_manager.cpp b/src/concurrency/transaction_manager.cpp index 93c2c6e8f1f..d51b83d57c5 100644 --- a/src/concurrency/transaction_manager.cpp +++ b/src/concurrency/transaction_manager.cpp @@ -16,7 +16,6 @@ #include "concurrency/transaction_context.h" #include "function/date_functions.h" #include "gc/gc_manager_factory.h" -#include "logging/log_manager.h" #include "settings/settings_manager.h" #include "statistics/stats_aggregator.h" #include "storage/tile_group.h" @@ -73,6 +72,15 @@ TransactionContext *TransactionManager::BeginTransaction( .StartTimer(); } + if(logging::LogManager::GetInstance().IsLoggingEnabled()) { + logging::LogRecord record = + logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TRANSACTION_BEGIN, txn->GetEpochId(), + txn->GetTransactionId(), txn->GetCommitId()); + + txn->GetLogBuffer()->WriteRecord(record); + } + txn->SetTimestamp(function::DateFunctions::Now()); return txn; @@ -97,7 +105,6 @@ void TransactionManager::EndTransaction(TransactionContext *current_txn) { stats::BackendStatsContext::GetInstance() ->GetTxnLatencyMetric() .RecordLatency(); - } } diff --git a/src/executor/insert_executor.cpp b/src/executor/insert_executor.cpp index 93fa76b220e..d895de38e34 100644 --- a/src/executor/insert_executor.cpp +++ b/src/executor/insert_executor.cpp @@ -110,10 +110,13 @@ bool InsertExecutor::DExecute() { for (oid_t tuple_id : *logical_tile) { ContainerTuple cur_tuple(logical_tile.get(), tuple_id); + std::vector values; + // Materialize the logical tile tuple for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { type::Value val = (cur_tuple.GetValue(column_itr)); tuple->SetValue(column_itr, val, executor_pool); + values.push_back(val); } // insert tuple into the table. @@ -130,7 +133,7 @@ bool InsertExecutor::DExecute() { return false; } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); executor_context_->num_processed += 1; // insert one } @@ -245,7 +248,13 @@ bool InsertExecutor::DExecute() { return false; } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + std::vector values; + uint32_t num_columns = schema->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(new_tuple->GetValue(col_id)); + } + + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); LOG_TRACE("Number of tuples in table after insert: %lu", target_table->GetTupleCount()); diff --git a/src/executor/update_executor.cpp b/src/executor/update_executor.cpp index a67a53d4719..dcb4c875a62 100644 --- a/src/executor/update_executor.cpp +++ b/src/executor/update_executor.cpp @@ -110,6 +110,12 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( peloton::ItemPointer location = target_table_->InsertTuple(&new_tuple, current_txn, &index_entry_ptr); + std::vector values; + uint32_t num_columns = target_table_schema->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(new_tuple.GetValue(col_id)); + } + // it is possible that some concurrent transactions have inserted the // same tuple. In this case, abort the transaction. if (location.block == INVALID_OID) { @@ -139,7 +145,7 @@ bool UpdateExecutor::PerformUpdatePrimaryKey( } } - transaction_manager.PerformInsert(current_txn, location, index_entry_ptr); + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, reinterpret_cast(values.data()), values.size()); statement_write_set_.insert(location); return true; } @@ -270,6 +276,7 @@ bool UpdateExecutor::DExecute() { project_info_->Evaluate(&old_tuple, &old_tuple, nullptr, executor_context_); + //TODO(Anirudh): Do we need the values buf and offset buf transaction_manager.PerformUpdate(current_txn, old_location); // we do not need to add any item pointer to statement-level write set // here, because we do not generate any new version @@ -368,6 +375,7 @@ bool UpdateExecutor::DExecute() { old_location.offset); LOG_TRACE("perform update new location: %u, %u", new_location.block, new_location.offset); + //TODO(Anirudh): Do we need to create the values and offset buffer transaction_manager.PerformUpdate(current_txn, old_location, new_location); statement_write_set_.insert(new_location); diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index 0dfc3f51fa9..64bc2328514 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -144,9 +144,12 @@ class TableCatalog : public AbstractCatalog { //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// - private: + + // used by logger during recovery std::shared_ptr GetTableObject( - oid_t table_oid, concurrency::TransactionContext *txn); + oid_t table_oid, concurrency::TransactionContext *txn); + private: + std::shared_ptr GetTableObject( const std::string &table_name, const std::string &schema_name, concurrency::TransactionContext *txn); diff --git a/src/include/codegen/buffering_consumer.h b/src/include/codegen/buffering_consumer.h index 257a9a1dd09..b559e348276 100644 --- a/src/include/codegen/buffering_consumer.h +++ b/src/include/codegen/buffering_consumer.h @@ -14,6 +14,7 @@ #include +#include "codegen/codegen.h" #include "codegen/compilation_context.h" #include "codegen/query_result_consumer.h" #include "codegen/value.h" @@ -72,6 +73,8 @@ class BufferingConsumer : public QueryResultConsumer { // Called from compiled query code to buffer the tuple static void BufferTuple(char *state, char *tuple, uint32_t num_cols); + static void AddToTupleBuffer(Value &val, CodeGen &codegen, llvm::Value *tuple_buffer, size_t &i); + //===--------------------------------------------------------------------===// // ACCESSORS //===--------------------------------------------------------------------===// diff --git a/src/include/codegen/inserter.h b/src/include/codegen/inserter.h index 94167730001..d386e71995c 100644 --- a/src/include/codegen/inserter.h +++ b/src/include/codegen/inserter.h @@ -54,7 +54,7 @@ class Inserter { peloton::type::AbstractPool *GetPool(); // Insert a tuple - void Insert(); + void Insert(char *values_buf, uint32_t values_size); // Finalize the instance void TearDown(); diff --git a/src/include/codegen/proxy/target_proxy.h b/src/include/codegen/proxy/target_proxy.h index 2b0e8924d7e..dee8c4d3e72 100644 --- a/src/include/codegen/proxy/target_proxy.h +++ b/src/include/codegen/proxy/target_proxy.h @@ -16,6 +16,7 @@ #include "codegen/proxy/type_builder.h" #include "common/internal_types.h" #include "planner/project_info.h" +#include "codegen/value.h" namespace peloton { namespace codegen { @@ -26,6 +27,5 @@ PROXY(Target) { }; TYPE_BUILDER(Target, peloton::Target); - } // namespace codegen } // namespace peloton diff --git a/src/include/codegen/updater.h b/src/include/codegen/updater.h index 474bfd2e457..b262ae1cda0 100644 --- a/src/include/codegen/updater.h +++ b/src/include/codegen/updater.h @@ -52,10 +52,10 @@ class Updater { peloton::type::AbstractPool *GetPool(); // Update a tuple - void Update(); + void Update(char *diff_array, uint32_t diff_size); // Update a tuple with primary key - void UpdatePK(); + void UpdatePK(char *diff_array, uint32_t diff_size); // Finalize the instance void TearDown(); diff --git a/src/include/common/container/lock_free_queue.h b/src/include/common/container/lock_free_queue.h index fb721cabfb9..6986fb7a367 100644 --- a/src/include/common/container/lock_free_queue.h +++ b/src/include/common/container/lock_free_queue.h @@ -12,6 +12,7 @@ #pragma once +#include #include "common/macros.h" #include "concurrentqueue/concurrentqueue.h" @@ -21,6 +22,8 @@ namespace peloton { // Lock-free Queue -- Supports multiple consumers and multiple producers. //===--------------------------------------------------------------------===// +using ProducerToken = moodycamel::ProducerToken; + template class LockFreeQueue { public: @@ -32,12 +35,27 @@ class LockFreeQueue { void Enqueue(const T &item) { queue_.enqueue(item); } + void Enqueue(const ProducerToken &token, T &&item) { + queue_.enqueue(token, std::move(item)); + } + + void Enqueue(const ProducerToken &token, const T &item) { + queue_.enqueue(token, item); + } + // Dequeues one item, returning true if an item was found // or false if the queue appeared empty bool Dequeue(T &item) { return queue_.try_dequeue(item); } bool IsEmpty() { return queue_.size_approx() == 0; } + void GenerateTokens(std::vector &tokens, int num_tokens){ + for(int i=0; i queue_; diff --git a/src/include/common/init.h b/src/include/common/init.h index 8e386d46a2e..5aede8c715a 100644 --- a/src/include/common/init.h +++ b/src/include/common/init.h @@ -10,6 +10,8 @@ // //===----------------------------------------------------------------------===// +#include +#include #pragma once namespace peloton { diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 17020512944..9c9abceb85d 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -36,6 +36,7 @@ #define EIGEN_DEFAULT_TO_ROW_MAJOR #include "eigen3/Eigen/Dense" + namespace peloton { class ItemPointer; @@ -941,6 +942,7 @@ enum class LogRecordType { // TransactionContext-related records TRANSACTION_BEGIN = 1, TRANSACTION_COMMIT = 2, + TRANSACTION_ABORT = 3, // Generic dml records TUPLE_INSERT = 11, @@ -1296,7 +1298,6 @@ struct DerivedAttribute; } typedef std::pair Target; - typedef std::vector TargetList; /** diff --git a/src/include/common/notifiable_task.h b/src/include/common/notifiable_task.h index 8ea65efb26b..c774597d083 100644 --- a/src/include/common/notifiable_task.h +++ b/src/include/common/notifiable_task.h @@ -15,6 +15,7 @@ #include #include #include "common/event_util.h" +#include "common/logger.h" namespace peloton { diff --git a/src/include/concurrency/timestamp_ordering_transaction_manager.h b/src/include/concurrency/timestamp_ordering_transaction_manager.h index 6e24b10b4c8..a48ee076489 100644 --- a/src/include/concurrency/timestamp_ordering_transaction_manager.h +++ b/src/include/concurrency/timestamp_ordering_transaction_manager.h @@ -19,6 +19,9 @@ #include "common/synchronization/spin_latch.h" namespace peloton { + +namespace logging {} + namespace concurrency { //===--------------------------------------------------------------------===// @@ -163,7 +166,9 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ virtual void PerformInsert(TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr = nullptr); + ItemPointer *index_entry_ptr = nullptr, + char *values_buf = nullptr, + uint32_t values_size = 0); /** * @brief Perform a read operation @@ -185,7 +190,10 @@ class TimestampOrderingTransactionManager : public TransactionManager { */ virtual void PerformUpdate(TransactionContext *const current_txn, const ItemPointer &old_location, - const ItemPointer &new_location); + const ItemPointer &new_location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr); /** * @brief Perform a delete operation. Used when the transaction is the @@ -206,7 +214,10 @@ class TimestampOrderingTransactionManager : public TransactionManager { * @param[in] location The location */ virtual void PerformUpdate(TransactionContext *const current_txn, - const ItemPointer &location); + const ItemPointer &location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr); /** * @brief Perform a delete operation. Used when the transaction is not @@ -218,23 +229,11 @@ class TimestampOrderingTransactionManager : public TransactionManager { virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location); - /** - * @brief Commits a transaction. - * - * @param current_txn The current transaction - * - * @return The result type - */ - virtual ResultType CommitTransaction(TransactionContext *const current_txn); + virtual ResultType CommitTransaction(TransactionContext *const current_txn, + std::function task_callback = nullptr); - /** - * @brief Abort a transaction - * - * @param current_txn The current transaction - * - * @return The result type - */ - virtual ResultType AbortTransaction(TransactionContext *const current_txn); + virtual ResultType AbortTransaction(TransactionContext *const current_txn, + std::function task_callback = nullptr); private: diff --git a/src/include/concurrency/transaction_context.h b/src/include/concurrency/transaction_context.h index 892149b510e..1796d2de22d 100644 --- a/src/include/concurrency/transaction_context.h +++ b/src/include/concurrency/transaction_context.h @@ -23,6 +23,8 @@ #include "common/item_pointer.h" #include "common/printable.h" #include "common/internal_types.h" +#include "logging/log_buffer.h" +#include "threadpool/logger_queue_pool.h" #define INTITIAL_RW_SET_SIZE 64 @@ -117,6 +119,14 @@ class TransactionContext : public Printable { */ inline uint64_t GetTimestamp() const { return timestamp_; } + inline int GetLogToken() const { return log_token_; } + + inline logging::LogBuffer* GetLogBuffer() const { return log_buffer_; } + + inline void ResetLogBuffer() { + log_buffer_ = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + } + /** * @brief Gets the query strings. * @@ -281,6 +291,14 @@ class TransactionContext : public Printable { return isolation_level_; } + inline bool IsSingleStatementTxn(){ + return single_statement_txn_; + } + + inline void SetSingleStatementTxn(bool single_statement_txn){ + single_statement_txn_ = single_statement_txn; + } + /** cache for table catalog objects */ catalog::CatalogCache catalog_cache; @@ -289,6 +307,9 @@ class TransactionContext : public Printable { // Data members //===--------------------------------------------------------------------===// + //single statement txn + bool single_statement_txn_; + /** transaction id */ txn_id_t txn_id_; @@ -341,6 +362,9 @@ class TransactionContext : public Printable { IsolationLevelType isolation_level_; std::unique_ptr on_commit_triggers_; + + int log_token_; + logging::LogBuffer *log_buffer_; }; } // namespace concurrency diff --git a/src/include/concurrency/transaction_manager.h b/src/include/concurrency/transaction_manager.h index 0a4cd593b44..bb24f924818 100644 --- a/src/include/concurrency/transaction_manager.h +++ b/src/include/concurrency/transaction_manager.h @@ -187,7 +187,9 @@ class TransactionManager { */ virtual void PerformInsert(TransactionContext *const current_txn, const ItemPointer &location, - ItemPointer *index_entry_ptr = nullptr) = 0; + ItemPointer *index_entry_ptr = nullptr, + char *values_buf = nullptr, + uint32_t values_size = 0) = 0; virtual bool PerformRead(TransactionContext *const current_txn, const ItemPointer &location, @@ -196,14 +198,20 @@ class TransactionManager { virtual void PerformUpdate(TransactionContext *const current_txn, const ItemPointer &old_location, - const ItemPointer &new_location) = 0; + const ItemPointer &new_location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr) = 0; virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &old_location, const ItemPointer &new_location) = 0; virtual void PerformUpdate(TransactionContext *const current_txn, - const ItemPointer &location) = 0; + const ItemPointer &location, + char *values_buf = nullptr, + uint32_t values_size = 0, + TargetList *offsets = nullptr) = 0; virtual void PerformDelete(TransactionContext *const current_txn, const ItemPointer &location) = 0; @@ -232,9 +240,16 @@ class TransactionManager { */ void EndTransaction(TransactionContext *current_txn); - virtual ResultType CommitTransaction(TransactionContext *const current_txn) = 0; - virtual ResultType AbortTransaction(TransactionContext *const current_txn) = 0; + virtual ResultType CommitTransaction( + TransactionContext *const current_txn, + std::function task_callback = nullptr) = 0; + + + virtual ResultType AbortTransaction( + TransactionContext *const current_txn, + std::function task_callback = nullptr) = 0; + /** * This function generates the maximum commit id of committed transactions. diff --git a/src/include/index/bwtree.h b/src/include/index/bwtree.h index 4849682ab3c..f9352aad09a 100755 --- a/src/include/index/bwtree.h +++ b/src/include/index/bwtree.h @@ -7585,7 +7585,7 @@ class BwTree : public BwTreeBase { // would always fail, until we have cleaned all epoch nodes current_epoch_p = nullptr; - LOG_DEBUG("Clearing the epoch in ~EpochManager()..."); + LOG_TRACE("Clearing the epoch in ~EpochManager()..."); // If all threads has exited then all thread counts are // 0, and therefore this should proceed way to the end diff --git a/src/include/logging/checkpoint_manager.h b/src/include/logging/checkpoint_manager.h deleted file mode 100644 index 78d8bc3c120..00000000000 --- a/src/include/logging/checkpoint_manager.h +++ /dev/null @@ -1,69 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager.h -// -// Identification: src/include/logging/checkpoint_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/item_pointer.h" -#include "common/logger.h" -#include "common/macros.h" -#include "common/internal_types.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// checkpoint Manager -//===--------------------------------------------------------------------===// - -class CheckpointManager { - public: - CheckpointManager(const CheckpointManager &) = delete; - CheckpointManager &operator=(const CheckpointManager &) = delete; - CheckpointManager(CheckpointManager &&) = delete; - CheckpointManager &operator=(CheckpointManager &&) = delete; - - CheckpointManager() : is_running_(false) {} - - virtual ~CheckpointManager() {} - - static CheckpointManager &GetInstance() { - static CheckpointManager checkpoint_manager; - return checkpoint_manager; - } - - virtual void Reset() { is_running_ = false; } - - // Get status of whether logging threads are running or not - bool GetStatus() { return this->is_running_; } - - virtual void StartCheckpointing(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartCheckpointing() {} - - virtual void StopCheckpointing() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - protected: - volatile bool is_running_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/checkpoint_manager_factory.h b/src/include/logging/checkpoint_manager_factory.h deleted file mode 100644 index 44492147ba3..00000000000 --- a/src/include/logging/checkpoint_manager_factory.h +++ /dev/null @@ -1,54 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager_factory.h -// -// Identification: src/include/logging/checkpoint_manager_factory.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/checkpoint_manager.h" -#include "logging/logical_checkpoint_manager.h" - -namespace peloton { -namespace logging { - -class CheckpointManagerFactory { - public: - - static CheckpointManager& GetInstance() { - switch (checkpointing_type_) { - - case CheckpointingType::ON: - return LogicalCheckpointManager::GetInstance(checkpointing_thread_count_); - - default: - return CheckpointManager::GetInstance(); - } - } - - static void Configure(const int thread_count = 1) { - if (thread_count == 0) { - checkpointing_type_ = CheckpointingType::OFF; - } else { - checkpointing_type_ = CheckpointingType::ON; - checkpointing_thread_count_ = thread_count; - } - } - - inline static CheckpointingType GetCheckpointingType() { return checkpointing_type_; } - -private: - // checkpointing type - static CheckpointingType checkpointing_type_; - - static int checkpointing_thread_count_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_buffer.h b/src/include/logging/log_buffer.h index 4c05744b2b7..140e472359c 100644 --- a/src/include/logging/log_buffer.h +++ b/src/include/logging/log_buffer.h @@ -4,62 +4,57 @@ // // log_buffer.h // -// Identification: src/backend/logging/log_buffer.h +// Identification: src/include/logging/log_buffer.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// #pragma once +#include "type/serializeio.h" +#include "logging/log_record.h" -#include "common/macros.h" -#include "common/internal_types.h" +namespace peloton{ +namespace logging{ -namespace peloton { -namespace logging { +using LoggerCallback = std::function; -class LogBuffer { - LogBuffer(const LogBuffer &) = delete; - LogBuffer &operator=(const LogBuffer &) = delete; - LogBuffer(LogBuffer &&) = delete; - LogBuffer &operator=(LogBuffer &&) = delete; - - friend class LogBufferPool; -private: - // constant - const static size_t log_buffer_capacity_ = 1024 * 1024 * 32; // 32 MB +class LogBuffer{ public: - LogBuffer(const size_t thread_id, const size_t eid) : - thread_id_(thread_id), eid_(eid), size_(0){ - data_ = new char[log_buffer_capacity_]; - PELOTON_MEMSET(data_, 0, log_buffer_capacity_); - } - ~LogBuffer() { - delete[] data_; - data_ = nullptr; - } - inline void Reset() { size_ = 0; eid_ = INVALID_EID; } + LogBuffer(size_t threshold) + : log_buffer_threshold_(threshold), + on_flush_(nullptr) {} + + ~LogBuffer() {} - inline char *GetData() { return data_; } + void WriteRecord(LogRecord &record); - inline size_t GetSize() { return size_; } + inline const char *GetData() { return log_buffer_.Data(); } + inline size_t GetSize() { return log_buffer_.Size(); } - inline size_t GetEpochId() { return eid_; } - inline size_t GetThreadId() { return thread_id_; } + inline CopySerializeOutput &GetCopySerializedOutput() { return log_buffer_; } - inline bool Empty() { return size_ == 0; } + inline bool HasThresholdExceeded() { return log_buffer_.Size() >= log_buffer_threshold_; } + inline size_t GetThreshold() { return log_buffer_threshold_; } - bool WriteData(const char *data, size_t len); + + inline void SetLoggerCallback(const LoggerCallback &on_flush) { + on_flush_ = std::move(on_flush); + } + + inline LoggerCallback &GetLoggerCallback() { + return on_flush_; + } private: - size_t thread_id_; - size_t eid_; - size_t size_; - char* data_; + CopySerializeOutput log_buffer_; + size_t log_buffer_threshold_; + // The current callback to be invoked after logging completes. + LoggerCallback on_flush_; }; } -} +} \ No newline at end of file diff --git a/src/include/logging/log_buffer_pool.h b/src/include/logging/log_buffer_pool.h deleted file mode 100644 index e40925a29c5..00000000000 --- a/src/include/logging/log_buffer_pool.h +++ /dev/null @@ -1,63 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool.h -// -// Identification: src/logging/log_buffer_pool.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include - -#include "common/internal_types.h" -#include "common/macros.h" -#include "common/platform.h" - -#include "logging/log_buffer.h" - -namespace peloton { -namespace logging { - class LogBufferPool { - LogBufferPool(const LogBufferPool &) = delete; - LogBufferPool &operator=(const LogBufferPool &) = delete; - LogBufferPool(const LogBufferPool &&) = delete; - LogBufferPool &operator=(const LogBufferPool &&) = delete; - - public: - LogBufferPool(const size_t thread_id) : - head_(0), - tail_(buffer_queue_size_), - thread_id_(thread_id), - local_buffer_queue_(buffer_queue_size_) {} - - std::unique_ptr GetBuffer(const size_t current_eid); - - void PutBuffer(std::unique_ptr buf); - - inline size_t GetThreadId() const { return thread_id_; } - - inline size_t GetEmptySlotCount() const { - return tail_.load() - head_.load(); - } - - inline size_t GetMaxSlotCount() const { - return buffer_queue_size_; - } - - private: - static const size_t buffer_queue_size_ = 16; - std::atomic head_; - std::atomic tail_; - - size_t thread_id_; - - std::vector> local_buffer_queue_; -}; - -} -} diff --git a/src/include/logging/log_manager.h b/src/include/logging/log_manager.h deleted file mode 100644 index 123fce8f56a..00000000000 --- a/src/include/logging/log_manager.h +++ /dev/null @@ -1,79 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager.h -// -// Identification: src/include/logging/log_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include - -#include "common/item_pointer.h" -#include "common/logger.h" -#include "common/macros.h" -#include "common/internal_types.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// log Manager -//===--------------------------------------------------------------------===// - -class LogManager { - public: - LogManager(const LogManager &) = delete; - LogManager &operator=(const LogManager &) = delete; - LogManager(LogManager &&) = delete; - LogManager &operator=(LogManager &&) = delete; - - LogManager() : is_running_(false) {} - - virtual ~LogManager() {} - - static LogManager &GetInstance() { - static LogManager log_manager; - return log_manager; - } - - void Reset() { is_running_ = false; } - - // Get status of whether logging threads are running or not - bool GetStatus() { return this->is_running_; } - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartLogging() {} - - virtual void StopLogging() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - virtual void LogBegin() {} - - virtual void LogEnd() {} - - virtual void LogInsert(const ItemPointer & UNUSED_ATTRIBUTE) {} - - virtual void LogUpdate(const ItemPointer & UNUSED_ATTRIBUTE) {} - - virtual void LogDelete(const ItemPointer & UNUSED_ATTRIBUTE) {} - - protected: - volatile bool is_running_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_manager_factory.h b/src/include/logging/log_manager_factory.h deleted file mode 100644 index 01b9dc3d521..00000000000 --- a/src/include/logging/log_manager_factory.h +++ /dev/null @@ -1,54 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager_factory.h -// -// Identification: src/include/logging/log_manager_factory.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/log_manager.h" -#include "logging/logical_log_manager.h" - -namespace peloton { -namespace logging { - -class LogManagerFactory { - public: - - static LogManager& GetInstance() { - switch (logging_type_) { - - case LoggingType::ON: - return LogicalLogManager::GetInstance(logging_thread_count_); - - default: - return LogManager::GetInstance(); - } - } - - static void Configure(const int thread_count = 1) { - if (thread_count == 0) { - logging_type_ = LoggingType::OFF; - } else { - logging_type_ = LoggingType::ON; - logging_thread_count_ = thread_count; - } - } - - inline static LoggingType GetLoggingType() { return logging_type_; } - -private: - // checkpointing type - static LoggingType logging_type_; - - static int logging_thread_count_; -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/log_record.h b/src/include/logging/log_record.h index 6a5089c1a7b..9a4b181b66e 100644 --- a/src/include/logging/log_record.h +++ b/src/include/logging/log_record.h @@ -6,15 +6,15 @@ // // Identification: src/logging/log_record.h // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// - #pragma once - #include "common/internal_types.h" #include "common/item_pointer.h" #include "common/macros.h" +#include "type/value.h" +#include "../common/internal_types.h" namespace peloton { namespace logging { @@ -26,12 +26,15 @@ namespace logging { class LogRecord { friend class LogRecordFactory; private: - LogRecord(LogRecordType log_type, const ItemPointer &pos, - const eid_t epoch_id, const cid_t commit_id) - : log_record_type_(log_type), + LogRecord(LogRecordType log_type, const ItemPointer &old_pos, const ItemPointer &pos, + const eid_t epoch_id, const txn_id_t txn_id, const cid_t commit_id, oid_t schema_id) + : log_record_type_(log_type), + old_tuple_pos_(old_pos), tuple_pos_(pos), - eid_(epoch_id), - cid_(commit_id) {} + eid_(epoch_id), + txn_id_(txn_id), + cid_(commit_id), + schema_id_(schema_id) {} public: virtual ~LogRecord() {} @@ -40,47 +43,100 @@ class LogRecord { inline void SetItemPointer(const ItemPointer &pos) { tuple_pos_ = pos; } + inline void SetOldItemPointer(const ItemPointer &pos) { old_tuple_pos_ = pos; } + inline void SetEpochId(const eid_t epoch_id) { eid_ = epoch_id; } inline void SetCommitId(const cid_t commit_id) { cid_ = commit_id; } + inline void SetTransactionId(const txn_id_t txn_id) { txn_id_ = txn_id; } + + inline void SetValuesArray(char *diff_array, uint32_t num_values) { + values_ = diff_array; + num_values_ = num_values; + } + + inline void SetOffsetsArray(TargetList *arr) { + offsets_ = arr; + } + + inline void SetSchemaId(oid_t schema_id) { + schema_id_ = schema_id; + } + inline const ItemPointer &GetItemPointer() { return tuple_pos_; } + inline const ItemPointer &GetOldItemPointer() { return old_tuple_pos_; } + inline eid_t GetEpochId() { return eid_; } inline cid_t GetCommitId() { return cid_; } + inline char *GetValuesArray() { return values_; } + + inline uint32_t GetNumValues() { return num_values_; } + + inline TargetList *GetOffsets() { return offsets_; } + + inline txn_id_t GetTransactionId() { return txn_id_; } + + inline oid_t GetSchemaId() { return schema_id_; } + private: LogRecordType log_record_type_; + ItemPointer old_tuple_pos_; + ItemPointer tuple_pos_; eid_t eid_; + txn_id_t txn_id_; + cid_t cid_; + + char *values_; + + TargetList *offsets_; + + uint32_t num_values_; + + oid_t schema_id_; }; class LogRecordFactory { public: - static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &pos) { - PELOTON_ASSERT(log_type == LogRecordType::TUPLE_INSERT || - log_type == LogRecordType::TUPLE_DELETE || - log_type == LogRecordType::TUPLE_UPDATE); - return LogRecord(log_type, pos, INVALID_EID, INVALID_CID); + + static LogRecord CreateTupleRecord(const LogRecordType log_type, + const ItemPointer &pos, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid, oid_t schema_oid) { + PELOTON_ASSERT(log_type == LogRecordType::TUPLE_INSERT || + log_type == LogRecordType::TUPLE_DELETE); + + return LogRecord(log_type, INVALID_ITEMPOINTER, pos, current_eid, txn_id, current_cid, schema_oid); } - static LogRecord CreateTxnRecord(const LogRecordType log_type, const cid_t commit_id) { - PELOTON_ASSERT(log_type == LogRecordType::TRANSACTION_BEGIN || - log_type == LogRecordType::TRANSACTION_COMMIT); - return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_EID, commit_id); + static LogRecord CreateTupleRecord(const LogRecordType log_type, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid) { + + PELOTON_ASSERT(log_type == LogRecordType::TRANSACTION_COMMIT || + log_type == LogRecordType::TRANSACTION_ABORT || + log_type == LogRecordType::TRANSACTION_BEGIN); + + return LogRecord(log_type, INVALID_ITEMPOINTER, INVALID_ITEMPOINTER, + current_eid, txn_id, current_cid, INVALID_OID); } - static LogRecord CreateEpochRecord(const LogRecordType log_type, const eid_t epoch_id) { - PELOTON_ASSERT(log_type == LogRecordType::EPOCH_BEGIN || - log_type == LogRecordType::EPOCH_END); - return LogRecord(log_type, INVALID_ITEMPOINTER, epoch_id, INVALID_CID); + static LogRecord CreateTupleRecord(const LogRecordType log_type, const ItemPointer &old_pos, + const ItemPointer &pos, eid_t current_eid, + txn_id_t txn_id, cid_t current_cid, oid_t schema_oid) { + + PELOTON_ASSERT(log_type == LogRecordType::TUPLE_UPDATE); + + return LogRecord(log_type, old_pos, pos, current_eid, txn_id, current_cid, schema_oid); } + }; } // namespace logging diff --git a/src/include/logging/log_util.h b/src/include/logging/log_util.h new file mode 100644 index 00000000000..7f34a10078a --- /dev/null +++ b/src/include/logging/log_util.h @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * logger.h + * file description + * + * Copyright(c) 2015, CMU + * + * /src/backend/logging/logging_util.h + * + *------------------------------------------------------------------------- + */ + +#pragma once + +#include "common/internal_types.h" +#include "common/logger.h" +#include "storage/data_table.h" +#include "type/serializer.h" + +namespace peloton { +namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +class LoggingUtil { +public: + // FILE SYSTEM RELATED OPERATIONS + + static bool CheckDirectoryExistence(const char *dir_name); + + static bool CreateDirectory(const char *dir_name, int mode); + + static bool RemoveDirectory(const char *dir_name, bool only_remove_file); + + + static bool OpenFile(const char *name, std::ios_base::openmode mode, + std::fstream &fs); + + static void CloseFile(std::fstream &fs); + + static int32_t ReadNBytesFromFile(std::fstream &fs, char *bytes_read, + size_t n); +}; + +} // namespace logging +} // namespace peloton diff --git a/src/include/logging/logical_checkpoint_manager.h b/src/include/logging/logical_checkpoint_manager.h deleted file mode 100644 index b93a89e680f..00000000000 --- a/src/include/logging/logical_checkpoint_manager.h +++ /dev/null @@ -1,60 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_checkpoint_manager.h -// -// Identification: src/include/logging/logical_checkpoint_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/checkpoint_manager.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// logical checkpoint Manager -//===--------------------------------------------------------------------===// - -class LogicalCheckpointManager : public CheckpointManager { - public: - LogicalCheckpointManager(const LogicalCheckpointManager &) = delete; - LogicalCheckpointManager &operator=(const LogicalCheckpointManager &) = delete; - LogicalCheckpointManager(LogicalCheckpointManager &&) = delete; - LogicalCheckpointManager &operator=(LogicalCheckpointManager &&) = delete; - - LogicalCheckpointManager(const int thread_count) : checkpointer_thread_count_(thread_count) {} - - virtual ~LogicalCheckpointManager() {} - - static LogicalCheckpointManager &GetInstance(const int thread_count = 1) { - static LogicalCheckpointManager checkpoint_manager(thread_count); - return checkpoint_manager; - } - - virtual void Reset() { is_running_ = false; } - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) {} - - virtual void StartLogging() {} - - virtual void StopLogging() {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) {} - - virtual size_t GetTableCount() { return 0; } - - private: - int checkpointer_thread_count_; - -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/logical_log_manager.h b/src/include/logging/logical_log_manager.h deleted file mode 100644 index a8c46ed5e16..00000000000 --- a/src/include/logging/logical_log_manager.h +++ /dev/null @@ -1,129 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_log_manager.h -// -// Identification: src/include/logging/logical_log_manager.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include "logging/log_manager.h" - -namespace peloton { -namespace logging { - -//===--------------------------------------------------------------------===// -// logical log Manager -//===--------------------------------------------------------------------===// - - -/** - * logging file name layout : - * - * dir_name + "/" + prefix + "_" + epoch_id - * - * - * logging file layout : - * - * ----------------------------------------------------------------------------- - * | txn_id | database_id | table_id | operation_type | data | ... | txn_end_flag - * ----------------------------------------------------------------------------- - * - * NOTE: this layout is designed for logical logging. - * - * NOTE: tuple length can be obtained from the table schema. - * - */ - -class LogicalLogManager : public LogManager { - public: - LogicalLogManager(const LogicalLogManager &) = delete; - LogicalLogManager &operator=(const LogicalLogManager &) = delete; - LogicalLogManager(LogicalLogManager &&) = delete; - LogicalLogManager &operator=(LogicalLogManager &&) = delete; - - LogicalLogManager(const int thread_count) : logger_thread_count_(thread_count) {} - - virtual ~LogicalLogManager() {} - - static LogicalLogManager &GetInstance(const int thread_count = 1) { - static LogicalLogManager log_manager(thread_count); - return log_manager; - } - - // virtual void SetDirectories(const std::vector &logging_dirs) override { - // logger_dirs_ = logging_dirs; - - // if (logging_dirs.size() > 0) { - // pepoch_dir_ = logging_dirs.at(0); - // } - // // check the existence of logging directories. - // // if not exists, then create the directory. - // for (auto logging_dir : logging_dirs) { - // if (LoggingUtil::CheckDirectoryExistence(logging_dir.c_str()) == false) { - // LOG_INFO("Logging directory %s is not accessible or does not exist", logging_dir.c_str()); - // bool res = LoggingUtil::CreateDirectory(logging_dir.c_str(), 0700); - // if (res == false) { - // LOG_ERROR("Cannot create directory: %s", logging_dir.c_str()); - // } - // } - // } - - // logger_count_ = logging_dirs.size(); - // for (size_t i = 0; i < logger_count_; ++i) { - // loggers_.emplace_back(new PhyLogLogger(i, logging_dirs.at(i))); - // } - // } - - // virtual const std::vector &GetDirectories() override { - // return logger_dirs_; - // } - - - virtual void StartLogging(std::vector> & UNUSED_ATTRIBUTE) override {} - - virtual void StartLogging() override {} - - virtual void StopLogging() override {} - - virtual void RegisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) override {} - - virtual void DeregisterTable(const oid_t &table_id UNUSED_ATTRIBUTE) override {} - - virtual size_t GetTableCount() override { return 0; } - - virtual void LogBegin() override {} - - virtual void LogEnd() override {} - - virtual void LogInsert(const ItemPointer & UNUSED_ATTRIBUTE) override {} - - virtual void LogUpdate(const ItemPointer & UNUSED_ATTRIBUTE) override {} - - virtual void LogDelete(const ItemPointer & UNUSED_ATTRIBUTE) override { } - - private: - int logger_thread_count_; - - // std::atomic worker_count_; - - // std::vector logger_dirs_; - - // std::vector> loggers_; - - // std::unique_ptr pepoch_thread_; - // volatile bool is_running_; - - // std::string pepoch_dir_; - - // const std::string pepoch_filename_ = "pepoch"; - -}; - -} // namespace logging -} // namespace peloton diff --git a/src/include/logging/logical_logger.h b/src/include/logging/logical_logger.h deleted file mode 100644 index 9bcde68db3c..00000000000 --- a/src/include/logging/logical_logger.h +++ /dev/null @@ -1,148 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// logical_logger.h -// -// Identification: src/logging/logical_logger.h -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#pragma once - -#include -#include -#include -#include -#include - -#include "concurrency/transaction_context.h" -#include "concurrency/epoch_manager.h" -#include "logging/log_buffer.h" -#include "logging/log_record.h" -#include "logging/log_buffer_pool.h" -#include "logging/log_manager.h" -#include "logging/worker_context.h" -#include "type/types.h" -#include "common/serializer.h" -#include "common/lockfree_queue.h" -#include "common/logger.h" -#include "common/pool.h" -#include "common/synchronization/spin_lock.h" - -namespace peloton { - -namespace storage { - class TileGroupHeader; -} - -namespace logging { - - class PhyLogLogger { - - public: - PhyLogLogger(const size_t &logger_id, const std::string &log_dir) : - logger_id_(logger_id), - log_dir_(log_dir), - logger_thread_(nullptr), - is_running_(false), - logger_output_buffer_(), - persist_epoch_id_(INVALID_EID), - worker_map_lock_(), - worker_map_() {} - - ~PhyLogLogger() {} - - void StartRecovery(const size_t checkpoint_eid, const size_t persist_eid, const size_t recovery_thread_count); - void StartIndexRebulding(const size_t logger_count); - - void WaitForRecovery(); - void WaitForIndexRebuilding(); - - void StartLogging() { - is_running_ = true; - logger_thread_.reset(new std::thread(&PhyLogLogger::Run, this)); - } - - void StopLogging() { - is_running_ = false; - logger_thread_->join(); - } - - void RegisterWorker(WorkerContext *phylog_worker_ctx); - void DeregisterWorker(WorkerContext *phylog_worker_ctx); - - size_t GetPersistEpochId() const { - return persist_epoch_id_; - } - - -private: - void Run(); - - void PersistEpochBegin(FileHandle &file_handle, const size_t epoch_id); - void PersistEpochEnd(FileHandle &file_handle, const size_t epoch_id); - void PersistLogBuffer(FileHandle &file_handle, std::unique_ptr log_buffer); - - std::string GetLogFileFullPath(size_t epoch_id) { - return log_dir_ + "/" + logging_filename_prefix_ + "_" + std::to_string(logger_id_) + "_" + std::to_string(epoch_id); - } - - void GetSortedLogFileIdList(const size_t checkpoint_eid, const size_t persist_eid); - - void RunRecoveryThread(const size_t thread_id, const size_t checkpoint_eid, const size_t persist_eid); - - void RunSecIndexRebuildThread(const size_t logger_count); - - void RebuildSecIndexForTable(const size_t logger_count, storage::DataTable *table); - - bool ReplayLogFile(const size_t thread_id, FileHandle &file_handle, size_t checkpoint_eid, size_t pepoch_eid); - bool InstallTupleRecord(LogRecordType type, storage::Tuple *tuple, storage::DataTable *table, cid_t cur_cid); - - // Return value is the swapped txn id, either INVALID_TXNID or INITIAL_TXNID - txn_id_t LockTuple(storage::TileGroupHeader *tg_header, oid_t tuple_offset); - void UnlockTuple(storage::TileGroupHeader *tg_header, oid_t tuple_offset, txn_id_t new_txn_id); - - private: - size_t logger_id_; - std::string log_dir_; - - // recovery threads - std::vector> recovery_threads_; - std::vector file_eids_; - std::atomic max_replay_file_id_; - - /* Recovery */ - // TODO: Check if we can discard the recovery pool after the recovery is done. - // Since every thing is copied to the tile group and tile group related pool - std::vector> recovery_pools_; - - // logger thread - std::unique_ptr logger_thread_; - volatile bool is_running_; - - /* File system related */ - CopySerializeOutput logger_output_buffer_; - - /* Log buffers */ - size_t persist_epoch_id_; - - // The spin lock to protect the worker map. - // We only update this map when creating/terminating a new worker - common::synchronization::SpinLatch worker_map_lock_; - - // map from worker id to the worker's context. - std::unordered_map> worker_map_; - - const std::string logging_filename_prefix_ = "log"; - - const size_t sleep_period_us_ = 40000; - - const int new_file_interval_ = 500; // 500 milliseconds. - }; - - -} -} \ No newline at end of file diff --git a/src/include/logging/wal_log_manager.h b/src/include/logging/wal_log_manager.h new file mode 100644 index 00000000000..61f972d74e3 --- /dev/null +++ b/src/include/logging/wal_log_manager.h @@ -0,0 +1,99 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_log_manager.h +// +// Identification: src/logging/wal_log_manager.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// +#pragma once +#include + +#include "settings/settings_manager.h" +#include "util/file_util.h" +#include "logging/wal_recovery.h" + + +namespace peloton{ +namespace logging{ + +class LogManager{ +public: + + LogManager() : enable_logging_(false) {} + ~LogManager() {} + + inline bool init() { + + log_dir_ = settings::SettingsManager::GetString(settings::SettingId::log_directory_name); + log_file_ = settings::SettingsManager::GetString(settings::SettingId::log_file_name); + + if (!peloton::FileUtil::Exists(log_dir_)) { + boost::filesystem::path dir(log_dir_.c_str()); + try { + boost::filesystem::create_directory(dir); + } + catch (boost::filesystem::filesystem_error &e) { + LOG_INFO("Failed to create log directory %s. Reason %s", log_dir_.c_str(), e.what()); + return false; + } + } + + if(!logger_ofstream_.is_open()){ + logger_ofstream_.open(log_dir_ + "/" + log_file_, std::ofstream::out | std::ofstream::app); + + if(!logger_ofstream_.fail()) { + enable_logging_ = true; + return true; + } + } + + return false; + } + + inline static LogManager &GetInstance(){ + static LogManager log_manager; + return log_manager; + } + + inline static size_t GetTransactionBufferSize() { + return settings::SettingsManager::GetInt(settings::SettingId::transaction_buffer_size); + } + + inline static size_t GetLoggerBufferSize() { + return settings::SettingsManager::GetInt(settings::SettingId::log_buffer_size); + } + + inline bool IsLoggingEnabled() { return enable_logging_; } + + inline std::string GetDirectory() { return log_dir_; } + inline std::string GetLogFilePath() { return log_dir_ + "/" + log_file_; } + + + + inline void DoRecovery(){ + WalRecovery* wr = new WalRecovery(GetLogFilePath()); + wr->StartRecovery(); + delete wr; + } + + + inline std::ofstream *GetFileStream() { return &logger_ofstream_; } + + + +private: + // NOTE: ofstream is not thread safe, might need to change it if more than one logger thread is used + bool enable_logging_; + std::string log_dir_; + std::string log_file_; + std::ofstream logger_ofstream_; + + +}; + +} +} diff --git a/src/include/logging/wal_logger.h b/src/include/logging/wal_logger.h new file mode 100644 index 00000000000..4b44e938a8d --- /dev/null +++ b/src/include/logging/wal_logger.h @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_logger.h +// +// Identification: src/logging/wal_logger.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "common/container/lock_free_queue.h" +#include "logging/log_buffer.h" +#include "logging/log_record.h" +#include "logging/wal_log_manager.h" +#include "type/serializeio.h" + + +namespace peloton { +namespace logging { + +using LogToken = peloton::ProducerToken; + +class WalLogger { +public: + + + WalLogger() { + disk_buffer_ = new LogBuffer(LogManager::GetLoggerBufferSize()); + } + + ~WalLogger() {} + + bool IsFlushNeeded(bool pending_buffers); + void FlushToDisk(); + void PerformCompaction(LogBuffer *buffer); + +private: + LogBuffer *disk_buffer_; + std::deque callbacks_; +}; +} +} diff --git a/src/include/logging/wal_recovery.h b/src/include/logging/wal_recovery.h new file mode 100644 index 00000000000..dde14713564 --- /dev/null +++ b/src/include/logging/wal_recovery.h @@ -0,0 +1,106 @@ +#pragma once + +#include +#include + +#include "type/serializeio.h" +#include "logging/log_record.h" +#include "storage/storage_manager.h" +#include "storage/tuple.h" + +namespace peloton{ +namespace logging { + + + +class WalRecovery{ + +private: + + enum class ReplayStage{ + PASS_1, + PASS_2 + }; + + // (graghura): this is a dirty hack to establish the + // replay order. This is tightly coupled with the way + // transaction id is generated :( + // The transaction with the smaller epoch_id is replayed + // first. If the epoch's are the same, then the txn with the + // lower next_txn_id (decentralized_epoch_manager.cpp) + // is replayed first. + + // TODO(graghura) : change this after the mvcc bugs are fixed. + + struct RecoveryComparator { + bool operator()(txn_id_t txn_id_1, txn_id_t txn_id_2) const { + eid_t epoch_txn_1 = txn_id_1 >> 32; + eid_t epoch_txn_2 = txn_id_2 >> 32; + + int next_id_1 = txn_id_1 & 0xFFFFFFFF; + int next_id_2 = txn_id_2 & 0xFFFFFFFF; + + if(txn_id_1==txn_id_2) + return false; + + if(epoch_txn_1==epoch_txn_2){ + if(next_id_1<=next_id_2) + return true; + else + return false; + } else if(epoch_txn_1, RecoveryComparator>; + + +public: + WalRecovery(std::string logpath) : + log_path_(logpath), + max_epoch_id_(INVALID_EID), + log_buffer_size_(0), + log_buffer_(nullptr) {} + + ~WalRecovery(){} + + + void StartRecovery(); + +private: + + void ReplayLogFile(); + void ParseFromDisk(ReplayStage stage); + void ReplayAllTxns(); + void ReplaySingleTxn(txn_id_t txn_id); + + bool InstallCatalogTuple(LogRecordType type, storage::Tuple *tuple, + storage::DataTable *table, cid_t cur_cid, + ItemPointer location); + + + void CreateTableOnRecovery(std::unique_ptr &tuple, + std::vector &columns); + + + std::string log_path_; + std::fstream fstream_; + + eid_t max_epoch_id_; + + int log_buffer_size_; + char *log_buffer_; + + ReplayTxnMap all_txns_; + std::map> commited_txns_; // {txn_id, {log_buf_start_offset, log_buf_curr_offset}} + + +}; + +} +} \ No newline at end of file diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index e90cb78b7da..6b7f7d36e4b 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -116,9 +116,42 @@ SETTING_int(gc_num_threads, 1, 128, true, true) +SETTING_int(log_buffer_size, + "The default log buffer size (default: 512KB)", + 512 * 1024, + 1024, 1024 * 1024, + true, true) + +SETTING_int(transaction_buffer_size, + "The default log buffer size for each transaction (default: 16KB)", + 16 * 1024, + 512, 256 * 1024, + true, true) + //===----------------------------------------------------------------------===// // WRITE AHEAD LOG //===----------------------------------------------------------------------===// +SETTING_string(log_directory_name, + "The relative path(name) to the directory where the WAL will be stored (default : ./logging)", + "./logging", + true, true) + +SETTING_string(log_file_name, + "The name of the write ahead log file in the directory specified above (default : wal.log)", + "wal.log", + true, true) + +// Enable logging. The default value is true for testing purposes +// TODO(graghura): logging should be turned off by default +SETTING_bool(enable_logging, + "Enable logging for DB persistence (default: true)", + true, + false, false) + +SETTING_bool(enable_recovery, + "Enable recovery for DB persistence (default: false)", + false, + false, false) //===----------------------------------------------------------------------===// // ERROR REPORTING AND LOGGING diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index c8348485115..0b024aad7da 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -76,7 +76,6 @@ class DataTable : public AbstractTable { friend class TileGroup; friend class TileGroupFactory; friend class TableFactory; - friend class logging::LogManager; DataTable() = delete; DataTable(DataTable const &) = delete; @@ -194,7 +193,7 @@ class DataTable : public AbstractTable { // FOREIGN KEYS //===--------------------------------------------------------------------===// - bool CheckForeignKeySrcAndCascade(storage::Tuple *prev_tuple, + bool CheckForeignKeySrcAndCascade(storage::Tuple *prev_tuple, storage::Tuple *new_tuple, concurrency::TransactionContext *transaction, executor::ExecutorContext *context, diff --git a/src/include/threadpool/logger_queue_pool.h b/src/include/threadpool/logger_queue_pool.h new file mode 100644 index 00000000000..14f5e84a6f8 --- /dev/null +++ b/src/include/threadpool/logger_queue_pool.h @@ -0,0 +1,109 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logger_queue_pool.h +// +// Identification: test/threadpool/Logger_queue_pool.h +// +// Copyright (c) 2015-17, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once +#include +#include "worker_pool.h" +#include "logging/log_buffer.h" +#include "logging/wal_logger.h" + + +// TODO: tune these variables +constexpr static size_t kDefaultLoggerQueueSize = 32; +constexpr static size_t kDefaultLoggerPoolSize = 1; +constexpr static size_t kDefaultNumTokens = 8; + + +namespace peloton { +namespace threadpool { + +using LoggerQueue = peloton::LockFreeQueue; + +void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue); + + +/** +* @class LoggerQueuePool +* @brief Wrapper class for single queue and single pool +* One should use this if possible. +*/ +class LoggerQueuePool { + public: + LoggerQueuePool(size_t num_workers) + : logger_queue_(kDefaultLoggerQueueSize), + num_workers_(num_workers), + is_running_(false) { + PELOTON_ASSERT(num_workers == 1); + logger_queue_.GenerateTokens(log_tokens_, kDefaultNumTokens); + } + + ~LoggerQueuePool() { + if(is_running_) + Shutdown(); + } + + inline int GetLogToken(){ + return next_token_++; + } + + + void Startup() { + is_running_ = true; + + for (size_t i = 0; i < num_workers_; i++) { + loggers_.emplace_back(LoggerFunc, &is_running_, &logger_queue_); + } + } + + void Shutdown() { + is_running_ = false; + + for (auto &logger : loggers_) { + logger.join(); + } + loggers_.clear(); + } + + void SubmitLogBuffer(logging::LogBuffer *buffer) { + if (is_running_ == false) + PELOTON_ASSERT(false); + + logger_queue_.Enqueue(std::move(buffer)); + } + + void SubmitLogBuffer(int token, logging::LogBuffer *buffer) { + if (is_running_ == false) + PELOTON_ASSERT(false); + + int idx = (token)%kDefaultNumTokens; + logger_queue_.Enqueue(*(log_tokens_[idx]), std::move(buffer)); + } + + static LoggerQueuePool &GetInstance() { + static LoggerQueuePool logger_queue_pool(kDefaultLoggerPoolSize); + return logger_queue_pool; + } + + private: + LoggerQueue logger_queue_; + size_t num_workers_; + + std::vector loggers_; + std::vector log_tokens_; + std::atomic_bool is_running_; + std::atomic next_token_; + + +}; + +} // namespace threadpool +} // namespace peloton diff --git a/src/include/traffic_cop/traffic_cop.h b/src/include/traffic_cop/traffic_cop.h index e324b87fe82..a5d64c29816 100644 --- a/src/include/traffic_cop/traffic_cop.h +++ b/src/include/traffic_cop/traffic_cop.h @@ -100,7 +100,9 @@ class TrafficCop { ResultType CommitQueryHelper(); - void ExecuteStatementPlanGetResult(); + ResultType AbortQueryHelper(); + + ResultType ExecuteStatementPlanGetResult(); ResultType ExecuteStatementGetResult(); @@ -145,6 +147,14 @@ class TrafficCop { default_database_name_ = std::move(default_database_name); } + inline std::function GetOnCompleteCallback(){ + auto on_complete = [this](ResultType result) { + this->p_status_.m_result = result; + this->task_callback_(this->task_callback_arg_); + }; + return on_complete; + } + // TODO: this member variable should be in statement_ after parser part // finished std::string query_; @@ -189,7 +199,7 @@ class TrafficCop { ResultType BeginQueryHelper(size_t thread_id); - ResultType AbortQueryHelper(); + // Get all data tables from a TableRef. // For multi-way join diff --git a/src/logging/checkpoint_manager_factory.cpp b/src/logging/checkpoint_manager_factory.cpp deleted file mode 100644 index 4f025729a7d..00000000000 --- a/src/logging/checkpoint_manager_factory.cpp +++ /dev/null @@ -1,23 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// checkpoint_manager_factory.cpp -// -// Identification: src/logging/checkpoint_manager_factory.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/checkpoint_manager_factory.h" - -namespace peloton { -namespace logging { - -CheckpointingType CheckpointManagerFactory::checkpointing_type_ = CheckpointingType::ON; -int CheckpointManagerFactory::checkpointing_thread_count_ = 1; - -} // namespace gc -} // namespace peloton diff --git a/src/logging/log_buffer.cpp b/src/logging/log_buffer.cpp index fc37fee21d2..d109f0695f6 100644 --- a/src/logging/log_buffer.cpp +++ b/src/logging/log_buffer.cpp @@ -1,32 +1,115 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer.cpp -// -// Identification: src/logging/log_buffer.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "common/macros.h" -#include "logging/log_buffer.h" - -namespace peloton { -namespace logging { - -bool LogBuffer::WriteData(const char *data, size_t len) { - if (unlikely_branch(size_ + len > log_buffer_capacity_)) { - return false; - } else { - PELOTON_ASSERT(data); - PELOTON_ASSERT(len); - PELOTON_MEMCPY(data_ + size_, data, len); - size_ += len; - return true; + +#include "logging/wal_logger.h" +#include "catalog/manager.h" +#include "common/container_tuple.h" +#include "storage/tile_group.h" +#include "../include/type/value.h" +#include "../include/logging/log_buffer.h" +#include "../include/common/internal_types.h" +#include "type/value_peeker.h" + + +namespace peloton{ +namespace logging{ + + + +void LogBuffer::WriteRecord(LogRecord &record) { + + // Reserve for the frame length + size_t start = log_buffer_.Position(); + log_buffer_.WriteInt(0); + + + LogRecordType type = record.GetType(); + log_buffer_.WriteEnumInSingleByte( + static_cast(type)); + + log_buffer_.WriteLong(record.GetTransactionId()); + log_buffer_.WriteLong(record.GetEpochId()); + log_buffer_.WriteLong(record.GetCommitId()); + + switch (type) { + case LogRecordType::TUPLE_INSERT: { + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + std::vector columns; + + // Write down the database id and the table id + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); + log_buffer_.WriteLong(tg->GetTableId()); + + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); + + peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); + for (uint32_t i = 0; i < record.GetNumValues(); i++) { + values_array[i].SerializeTo(log_buffer_); + } + + break; + } + case LogRecordType::TUPLE_DELETE: { + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + + // Write down the database id and the table id + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); + log_buffer_.WriteLong(tg->GetTableId()); + + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); + + break; + } + case LogRecordType::TUPLE_UPDATE: { + auto &manager = catalog::Manager::GetInstance(); + auto tuple_pos = record.GetItemPointer(); + auto old_tuple_pos = record.GetOldItemPointer(); + auto tg = manager.GetTileGroup(tuple_pos.block).get(); + + // Write down the database id and the table id + log_buffer_.WriteLong(tg->GetDatabaseId()); + log_buffer_.WriteLong(record.GetSchemaId()); + log_buffer_.WriteLong(tg->GetTableId()); + + log_buffer_.WriteLong(old_tuple_pos.block); + log_buffer_.WriteLong(old_tuple_pos.offset); + + log_buffer_.WriteLong(tuple_pos.block); + log_buffer_.WriteLong(tuple_pos.offset); + + log_buffer_.WriteLong(record.GetNumValues()); + + peloton::type::Value *values_array = reinterpret_cast(record.GetValuesArray()); + TargetList *offsets = record.GetOffsets(); + for (uint32_t i = 0; i < record.GetNumValues(); i++) { + log_buffer_.WriteInt(((*offsets)[i]).first); + values_array[i].SerializeTo(log_buffer_); + } + + break; + } + + case LogRecordType::TRANSACTION_BEGIN: + case LogRecordType::TRANSACTION_COMMIT: + case LogRecordType::TRANSACTION_ABORT: { + break; + } + + default: { + PELOTON_ASSERT(false); + } } + + // Add the frame length + // XXX: We rely on the fact that the serializer treat a int32_t as 4 bytes + int32_t length = log_buffer_.Position() - start - sizeof(int32_t); + log_buffer_.WriteIntAt(start, length); } } diff --git a/src/logging/log_buffer_pool.cpp b/src/logging/log_buffer_pool.cpp deleted file mode 100644 index 303f30dd55f..00000000000 --- a/src/logging/log_buffer_pool.cpp +++ /dev/null @@ -1,63 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool.cpp -// -// Identification: src/logging/log_buffer_pool.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/log_buffer_pool.h" -#include "common/logger.h" - -namespace peloton { -namespace logging { - - // Acquire a log buffer from the buffer pool. - // This function will be blocked until there is an available buffer. - // Note that only the corresponding worker thread can call this function. - std::unique_ptr LogBufferPool::GetBuffer(size_t current_eid) { - size_t head_idx = head_ % buffer_queue_size_; - while (true) { - if (head_.load() < tail_.load() - 1) { - if (local_buffer_queue_[head_idx].get() == nullptr) { - // Not any buffer allocated now - local_buffer_queue_[head_idx].reset(new LogBuffer(thread_id_, current_eid)); - } - break; - } - - // sleep a while, and try to get a new buffer - _mm_pause(); - LOG_TRACE("Worker %d uses up its buffer", (int) thread_id_); - } - - head_.fetch_add(1, std::memory_order_relaxed); - local_buffer_queue_[head_idx]->eid_ = current_eid; - return std::move(local_buffer_queue_[head_idx]); - } - - // This function is called only by the corresponding logger. - void LogBufferPool::PutBuffer(std::unique_ptr buf) { - PELOTON_ASSERT(buf.get() != nullptr); - PELOTON_ASSERT(buf->GetThreadId() == thread_id_); - - size_t tail_idx = tail_ % buffer_queue_size_; - - // The buffer pool must not be full - PELOTON_ASSERT(tail_idx != head_ % buffer_queue_size_); - // The tail pos must be null - PELOTON_ASSERT(local_buffer_queue_[tail_idx].get() == nullptr); - // The returned buffer must be empty - PELOTON_ASSERT(buf->Empty() == true); - local_buffer_queue_[tail_idx].reset(buf.release()); - - tail_.fetch_add(1, std::memory_order_relaxed); - } - -} -} diff --git a/src/logging/log_manager_factory.cpp b/src/logging/log_manager_factory.cpp deleted file mode 100644 index e537c5dcbc8..00000000000 --- a/src/logging/log_manager_factory.cpp +++ /dev/null @@ -1,24 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_manager_factory.cpp -// -// Identification: src/logging/log_manager_factory.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - - -#include "logging/log_manager_factory.h" - -namespace peloton { -namespace logging { - -LoggingType LogManagerFactory::logging_type_ = LoggingType::ON; - -int LogManagerFactory::logging_thread_count_ = 1; - -} // namespace gc -} // namespace peloton diff --git a/src/logging/log_util.cpp b/src/logging/log_util.cpp new file mode 100644 index 00000000000..16ffe927d38 --- /dev/null +++ b/src/logging/log_util.cpp @@ -0,0 +1,123 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// log_til.cpp +// +// Identification: src/logging/log_til.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include +#include +#include + +#include "catalog/manager.h" +#include "logging/log_util.h" + +namespace peloton { + namespace logging { + +//===--------------------------------------------------------------------===// +// LoggingUtil +//===--------------------------------------------------------------------===// + +bool LoggingUtil::CreateDirectory(const char *dir_name, int mode) { + int return_val = mkdir(dir_name, mode); + if (return_val == 0) { + LOG_DEBUG("Created directory %s successfully", dir_name); + } else if (errno == EEXIST) { + LOG_DEBUG("Directory %s already exists", dir_name); + } else { + LOG_DEBUG("Creating directory failed: %s", strerror(errno)); + return false; + } + return true; +} + +bool LoggingUtil::CheckDirectoryExistence(const char *dir_name) { + struct stat info; + int return_val = stat(dir_name, &info); + return return_val == 0 && S_ISDIR(info.st_mode); +} + +/** + * @return false if fail to remove directory + */ +bool LoggingUtil::RemoveDirectory(const char *dir_name, bool only_remove_file) { + struct dirent *file; + DIR *dir; + + dir = opendir(dir_name); + if (dir == nullptr) { + return true; + } + + // XXX readdir is not thread safe??? + while ((file = readdir(dir)) != nullptr) { + if (strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0) { + continue; + } + char complete_path[256]; + strcpy(complete_path, dir_name); + strcat(complete_path, "/"); + strcat(complete_path, file->d_name); + auto ret_val = remove(complete_path); + if (ret_val != 0) { + LOG_ERROR("Failed to delete file: %s, error: %s", complete_path, + strerror(errno)); + } + } + closedir(dir); + if (!only_remove_file) { + auto ret_val = remove(dir_name); + if (ret_val != 0) { + LOG_ERROR("Failed to delete dir: %s, error: %s", file->d_name, + strerror(errno)); + } + } + return true; +} + +/** + * @return false if fail to open file + */ +bool LoggingUtil::OpenFile(const char *name, std::ios_base::openmode mode, + std::fstream &fs) { + if(fs.is_open()){ + LOG_ERROR("Failed to open stream: already open"); + return false; + } + + fs.open(name, mode); + + if (fs.fail()){ + LOG_ERROR("Failed to open stream: %s", strerror(errno)); + return false; + } + + return true; +} + + +void LoggingUtil::CloseFile(std::fstream &fs) { + fs.close(); +} + +/** + * @return the number of bytes read from the file + */ +int32_t LoggingUtil::ReadNBytesFromFile(std::fstream &fs, char *bytes_read, + size_t n) { + + PELOTON_ASSERT(!fs.fail()); + fs.read(bytes_read, n); + return (int32_t)fs.gcount(); +} + + +} // namespace logging +} // namespace peloton diff --git a/src/logging/wal_logger.cpp b/src/logging/wal_logger.cpp new file mode 100644 index 00000000000..9a64a9cea0f --- /dev/null +++ b/src/logging/wal_logger.cpp @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_logger.cpp +// +// Identification: src/logging/wal_logger.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include +#include "logging/wal_logger.h" +#include "logging/log_buffer.h" +#include "catalog/manager.h" +#include "common/container_tuple.h" + + +namespace peloton{ +namespace logging{ + +bool WalLogger::IsFlushNeeded(bool pending_buffers){ + + /* if the disk buffer is full then we flush to the disk + * irrespective of whether there are pending callbacks + */ + if(disk_buffer_->HasThresholdExceeded()) + return true; + + /* if the disk buffer is not full and there are pending buffers + * in the queue then we don't flush to the disk now. We try to + * consolidate the pending buffers to achieve group commit. + */ + if(pending_buffers) + return false; + + /* if there are no pending buffers (at the time of check), but + * there are pending callbacks, we go ahead and flush the + * buffers. + */ + if(!callbacks_.empty()) + return true; + + return false; +} + + +void WalLogger::FlushToDisk(){ + + if(disk_buffer_->GetSize()==0){ + return; + } + + std::ofstream *stream = LogManager::GetInstance().GetFileStream(); + stream->write(disk_buffer_->GetData(), disk_buffer_->GetSize()); + + if(stream->fail()){ + PELOTON_ASSERT(false); + } + + stream->flush(); + + if(stream->fail()){ + PELOTON_ASSERT(false); + } + + + disk_buffer_->GetCopySerializedOutput().Reset(); + + + /* send out the callbacks */ + while(!callbacks_.empty()){ + auto callback = callbacks_.front(); + callbacks_.pop_front(); + callback(); + } + +} + + +void WalLogger::PerformCompaction(LogBuffer *log_buffer){ + + if(nullptr==log_buffer) + PELOTON_ASSERT(false); + + disk_buffer_->GetCopySerializedOutput() + .WriteBytes(log_buffer->GetData(), + log_buffer->GetSize()); + + auto callback = log_buffer->GetLoggerCallback(); + + if(nullptr != callback){ + callbacks_.push_back(std::move(callback)); + } + + +} +} +} \ No newline at end of file diff --git a/src/logging/wal_recovery.cpp b/src/logging/wal_recovery.cpp new file mode 100644 index 00000000000..f8fda17ece4 --- /dev/null +++ b/src/logging/wal_recovery.cpp @@ -0,0 +1,590 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// wal_recovery.cpp +// +// Identification: src/logging/wal_recovery.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "index/index_factory.h" + +#include "catalog/catalog.h" +#include "catalog/index_catalog.h" +#include "catalog/column_catalog.h" +#include "catalog/table_catalog.h" +#include "catalog/database_catalog.h" +#include "catalog/system_catalogs.h" +#include "concurrency/transaction_manager_factory.h" +#include "catalog/catalog_defaults.h" +#include "type/ephemeral_pool.h" +#include "catalog/manager.h" +#include "concurrency/epoch_manager_factory.h" +#include "logging/wal_recovery.h" +#include "logging/log_util.h" +#include "common/logger.h" + + +namespace peloton{ +namespace logging{ + +void WalRecovery::StartRecovery(){ + LOG_INFO("log_path_: %s", log_path_.c_str()); + + if(!LoggingUtil::OpenFile(log_path_.c_str(), + std::ifstream::in, fstream_)) { + + LOG_ERROR("Recovery failed"); + return; + } + + ReplayLogFile(); +} + + +void WalRecovery::ParseFromDisk(ReplayStage stage){ + + int buf_size = 4096, buf_rem = 0, buf_curr = 0,total_len=0; + char *buf = new char[buf_size]; + bool replay_complete = false; + + fstream_.clear(); + fstream_.seekg(0, std::ios::beg); + + while(!replay_complete) { + + int record_len, nbytes = buf_size-buf_rem; + int bytes_read = LoggingUtil::ReadNBytesFromFile(fstream_, buf+buf_curr, nbytes); + + LOG_INFO("Read %d bytes from the log", bytes_read); + + buf_rem += bytes_read; + + if(bytes_read != nbytes){ + if(fstream_.eof()){ + replay_complete = true; + + if(buf_rem==0) + continue; + } + } + + while(buf_rem >= (int)sizeof(record_len)){ + CopySerializeInput length_decode((const void *)(buf+buf_curr), 4); + record_len = length_decode.ReadInt(); + + if(buf_rem >= (int)(record_len + sizeof(record_len))){ + + total_len += record_len; + + int record_offset = buf_curr + sizeof(record_len); + + /***********/ + /* PASS 1 */ + /***********/ + + if(stage==ReplayStage::PASS_1){ + CopySerializeInput record_decode((const void *)(buf+record_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + + int req_mem_size = record_len + sizeof(record_len); + + switch(record_type){ + case LogRecordType::TRANSACTION_BEGIN:{ + if(all_txns_.find(txn_id) != all_txns_.end()){ + LOG_ERROR("Duplicate transaction"); + PELOTON_ASSERT(false); + } + + auto value = std::make_pair(record_type, req_mem_size); + all_txns_.insert(std::make_pair(txn_id, value)); + + break; + } + + case LogRecordType::TRANSACTION_COMMIT:{ + // keeps track of the memory that needs to be allocated. + log_buffer_size_ += (req_mem_size+all_txns_[txn_id].second); + } + + // intentional fall through + case LogRecordType::TRANSACTION_ABORT: + case LogRecordType::TUPLE_DELETE: + case LogRecordType::TUPLE_UPDATE: + case LogRecordType::TUPLE_INSERT: { + + if(all_txns_.find(txn_id) == all_txns_.end()){ + LOG_ERROR("Transaction not found"); + PELOTON_ASSERT(false); + } + + all_txns_[txn_id].first = record_type; + all_txns_[txn_id].second += req_mem_size; + + break; + } + + default: { + LOG_ERROR("Unknown LogRecordType."); + PELOTON_ASSERT(false); + } + } + } + + else if (stage==ReplayStage::PASS_2) { + + CopySerializeInput record_decode((const void *)(buf+record_offset), record_len); + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + + (void) record_type; + + if(commited_txns_.find(txn_id)!=commited_txns_.end()) { + int copy_len = record_len + sizeof(record_len); + int curr_offset = commited_txns_[txn_id].second; + + LOG_INFO("txn %lu writing from %d to %d", txn_id, curr_offset, curr_offset+copy_len-1); + + PELOTON_MEMCPY(log_buffer_+curr_offset, buf+buf_curr, copy_len); + commited_txns_[txn_id].second += copy_len; + } + } + + buf_curr += (record_len + sizeof(record_len)); + buf_rem -= (record_len + sizeof(record_len)); + } else { + break; + } + } + + PELOTON_MEMCPY(buf, buf+buf_curr, buf_rem); + PELOTON_MEMSET(buf+buf_rem, 0, buf_size - buf_rem); + buf_curr = buf_rem; + } + + delete[] buf; +} + + + +bool WalRecovery::InstallCatalogTuple(LogRecordType record_type, storage::Tuple *tuple, + storage::DataTable *table, cid_t cur_cid, + ItemPointer location){ + + ItemPointer *i = nullptr; + + auto& txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(IsolationLevelType::SERIALIZABLE); + + oid_t tile_group_id = location.block; + auto tile_group = table->GetTileGroupById(location.block); + auto tile_group_header = + catalog::Manager::GetInstance().GetTileGroup(tile_group_id)->GetHeader(); + + if(record_type==LogRecordType::TUPLE_INSERT){ + + PELOTON_ASSERT(tile_group != nullptr); + + auto status = tile_group_header->GetEmptyTupleSlot(location.offset); + + (void) status; + + tile_group->CopyTuple(tuple, location.offset); + bool result = table->InsertTuple(tuple, location, txn, &i); + + (void) result; + + tile_group_header->SetBeginCommitId(location.offset, cur_cid); + tile_group_header->SetEndCommitId(location.offset, MAX_CID); + tile_group_header->SetTransactionId(location.offset, INITIAL_TXN_ID); + tile_group_header->SetNextItemPointer(location.offset, + INVALID_ITEMPOINTER); + tile_group_header->SetIndirection(location.offset,i); + } + + txn_manager.CommitTransaction(txn); + return true; +} + +void WalRecovery::CreateTableOnRecovery(std::unique_ptr &tuple, + std::vector &columns){ + + // NOTE: The third column in pg_table stores the database oid. This value should always be corresponding to the location of the database oid in pg_table + // Getting database + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + tuple->GetValue(3).GetAs()); + + // register table with catalog + database->AddTable(new storage::DataTable( + new catalog::Schema(columns), tuple->GetValue(1).ToString(), + database->GetOid(), tuple->GetValue(0).GetAs(), + DEFAULT_TUPLES_PER_TILEGROUP, true, false, false)); + + columns.clear(); +} + +void WalRecovery::ReplaySingleTxn(txn_id_t txn_id){ + int start_offset = commited_txns_[txn_id].first; + int curr_offset = start_offset; + int total_len = all_txns_[txn_id].second; + std::unique_ptr pool(new type::EphemeralPool()); + + std::vector> indexes; + + + bool pending_table_create = false; + std::unique_ptr tuple_table_create; + std::vector columns; + + while(total_len > 0){ + CopySerializeInput length_decode((const void *)(log_buffer_+curr_offset), sizeof(int)); + int record_len = length_decode.ReadInt(); + + curr_offset += sizeof(int); + + CopySerializeInput record_decode((const void *)(log_buffer_+curr_offset), record_len); + + LogRecordType record_type = (LogRecordType)(record_decode.ReadEnumInSingleByte()); + txn_id_t txn_id = record_decode.ReadLong(); + eid_t epoch_id = record_decode.ReadLong(); + cid_t commit_id = record_decode.ReadLong(); + + (void) txn_id; + + if(max_epoch_id_==INVALID_EID) + max_epoch_id_ = epoch_id; + else if(epoch_id>max_epoch_id_) + max_epoch_id_ = epoch_id; + + // complete table_creation if pending + if(record_type != LogRecordType::TUPLE_INSERT){ + if(pending_table_create) { + CreateTableOnRecovery(tuple_table_create, columns); + pending_table_create = false; + } + } + + if(record_type==LogRecordType::TUPLE_INSERT) { + + oid_t database_id = (oid_t) record_decode.ReadLong(); + oid_t schema_id = (oid_t) record_decode.ReadLong(); + oid_t table_id = (oid_t) record_decode.ReadLong(); + + if(schema_id != CATALOG_SCHEMA_OID || table_id != COLUMN_CATALOG_OID){ + if(pending_table_create){ + CreateTableOnRecovery(tuple_table_create, columns); + pending_table_create = false; + } + } + + oid_t tg_block = (oid_t) record_decode.ReadLong(); + oid_t tg_offset = (oid_t) record_decode.ReadLong(); + + ItemPointer location(tg_block, tg_offset); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_id, table_id); + auto schema = table->GetSchema(); + auto tg = table->GetTileGroupById(tg_block); + + // Create the tile group if it does not exist + if (tg.get() == nullptr) { + table->AddTileGroupWithOidForRecovery(tg_block); + tg = table->GetTileGroupById(tg_block); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + + std::unique_ptr tuple(new storage::Tuple(schema, true)); + for (oid_t oid = 0; oid < schema->GetColumns().size(); oid++) { + type::Value val = type::Value::DeserializeFrom( + record_decode, schema->GetColumn(oid).GetType()); + tuple->SetValue(oid, val, pool.get()); + } + if (schema_id == CATALOG_SCHEMA_OID) { // catalog schema oid + + + if (table_id == DATABASE_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_DATABASE"); + + // insert tuple into pg_database + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + + auto storage_manager = storage::StorageManager::GetInstance(); + auto pg_database = catalog::DatabaseCatalog::GetInstance(); + + auto database_oid = tuple->GetValue(0).GetAs(); + auto database_name = tuple->GetValue(1).ToString(); + + storage::Database *database = new storage::Database(database_oid); + database->setDBName(database_name); + + // register database with storage manager + storage_manager->AddDatabaseToStorageManager(database); + + // update database oid to prevent oid reuse after recovery + pg_database->GetNextOid(); // TODO(graghura): This is a hack, rewrite !!! + + } else if (table_id == TABLE_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_TABLE"); + + // insert table info into pg_table + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + + // update table oid to prevent oid reuse after recovery + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetTableCatalog()->GetNextOid(); + tuple_table_create = std::move(tuple); + pending_table_create = true; + + } else if (table_id == COLUMN_CATALOG_OID) { + + LOG_INFO("REPLAYING INSERT TO PG_COLUMN"); + + // insert tuple into pg_column - one for every attribute in the schema + InstallCatalogTuple(record_type, tuple.get(), table, commit_id, location); + + // update column oid to prevent oid reuse after recovery + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetColumnCatalog()->GetNextOid(); + + bool is_inline = true; + + std::string typeId = tuple->GetValue(4).ToString(); + type::TypeId column_type = StringToTypeId(typeId); + + if (column_type == type::TypeId::VARCHAR || + column_type == type::TypeId::VARBINARY) { + is_inline = false; + } + + catalog::Column tmp_col(column_type, + type::Type::GetTypeSize(column_type), + tuple->GetValue(1).ToString(), is_inline, + tuple->GetValue(1).GetAs()); + + uint index = stoi(tuple->GetValue(2).ToString()); + + if (index >= columns.size()) { + columns.resize(index + 1); + } + columns[index] = tmp_col; + + } else if (table_id == INDEX_CATALOG_OID) { + LOG_INFO("REPLAYING INSERT TO PG_INDEX"); + indexes.push_back(std::move(tuple)); + catalog::Catalog::GetInstance()->GetSystemCatalogs(CATALOG_DATABASE_OID)->GetIndexCatalog()->GetNextOid(); + } + } else { + + LOG_INFO("REPLAYING INSERT TO NON CATALOG TABLE"); + + // insert tuple + auto tuple_id = tg->InsertTupleFromRecovery(commit_id, tg_offset, tuple.get()); + table->IncreaseTupleCount(1); + if (tuple_id == tg->GetAllocatedTupleCount() - 1) { + // add a new tile group if the last slot has been taken + if (table->GetTileGroupById(tg->GetTileGroupId() + 1).get() == + nullptr) + table->AddTileGroupWithOidForRecovery(tg->GetTileGroupId() + 1); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + } + } + + else if(record_type==LogRecordType::TUPLE_DELETE) { + LOG_INFO("Tuple Delete"); + + oid_t database_id = (oid_t)record_decode.ReadLong(); + oid_t schema_id = (oid_t) record_decode.ReadLong(); + oid_t table_id = (oid_t)record_decode.ReadLong(); + + oid_t tg_block = (oid_t)record_decode.ReadLong(); + oid_t tg_offset = (oid_t)record_decode.ReadLong(); + + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_id, table_id); + auto tg = table->GetTileGroupById(tg_block); + + // This code might be useful on drop + if (schema_id == CATALOG_SCHEMA_OID) { // catalog schema oid + switch (table_id) { + + case DATABASE_CATALOG_OID:{ //pg_database + // TODO(graghura): delete database + } + + case TABLE_CATALOG_OID: { // pg_table + auto db_oid = tg->GetValue(tg_offset, 2).GetAs(); + auto table_oid = tg->GetValue(tg_offset, 0).GetAs(); + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + db_oid); // Getting database oid from pg_table + database->DropTableWithOid(table_oid); + break; + } + + case INDEX_CATALOG_OID: { // pg_index + // TODO(graghura): delete index + } + + case COLUMN_CATALOG_OID: { // pg_column + // TODO(graghura): delete column (I think schema updates are not supported now) + } + + } + } + + auto tuple_id = tg->DeleteTupleFromRecovery(commit_id, tg_offset); + table->IncreaseTupleCount(1); + if (tuple_id == tg->GetAllocatedTupleCount() - 1) { + if (table->GetTileGroupById(tg->GetTileGroupId() + 1).get() == + nullptr) + table->AddTileGroupWithOidForRecovery(tg->GetTileGroupId() + 1); + catalog::Manager::GetInstance().GetNextTileGroupId(); + } + break; + } + + else if(record_type==LogRecordType::TUPLE_UPDATE) { + LOG_INFO("Replaying TUPLE_UPDATE"); + } + + else if(record_type==LogRecordType::TRANSACTION_BEGIN) { + LOG_INFO("Replaying TXN_BEGIN"); + } + + else if(record_type==LogRecordType::TRANSACTION_COMMIT) { + LOG_INFO("Replaying TXN_COMMIT"); + } + + else if(record_type==LogRecordType::TRANSACTION_ABORT) { + LOG_ERROR("Shouldn't be replaying TXN_ABORT"); + PELOTON_ASSERT(false); + } + + curr_offset += record_len; + total_len -= record_len + sizeof(int); + } + + if(pending_table_create) { + CreateTableOnRecovery(tuple_table_create, columns); + } + + + std::set tables_with_indexes; + + // Index construction that was deferred from the read records phase + for (auto const &tup : indexes) { + LOG_INFO("Install Index"); + std::vector key_attrs; + oid_t key_attr; + auto txn = concurrency::TransactionManagerFactory::GetInstance().BeginTransaction( + IsolationLevelType::SERIALIZABLE); + + auto table_catalog = catalog::Catalog::GetInstance()-> + GetSystemCatalogs(CATALOG_DATABASE_OID)->GetTableCatalog(); + auto table_object = table_catalog->GetTableObject(tup->GetValue(2).GetAs(), txn); + auto database_oid = table_object->GetDatabaseOid(); + auto table = storage::StorageManager::GetInstance()->GetTableWithOid( + database_oid, table_object->GetTableOid()); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + auto tuple_schema = table->GetSchema(); + std::stringstream iss(tup->GetValue(6).ToString()); + while (iss >> key_attr) key_attrs.push_back(key_attr); + auto key_schema = catalog::Schema::CopySchema(tuple_schema, key_attrs); + key_schema->SetIndexedColumns(key_attrs); + + index::IndexMetadata *index_metadata = new index::IndexMetadata( + tup->GetValue(1).ToString(), tup->GetValue(0).GetAs(), + table->GetOid(), database_oid, + static_cast(tup->GetValue(3).GetAs()), + static_cast(tup->GetValue(4).GetAs()), + tuple_schema, key_schema, key_attrs, tup->GetValue(4).GetAs()); + std::shared_ptr index( + index::IndexFactory::GetIndex(index_metadata)); + table->AddIndex(index); + tables_with_indexes.insert(table); + // Attributes must be changed once we have arraytype + } + + // add tuples to index + for (storage::DataTable *table : tables_with_indexes) { + LOG_INFO("Install table_index"); + auto schema = table->GetSchema(); + size_t tg_count = table->GetTileGroupCount(); + for (oid_t tg = 0; tg < tg_count; tg++) { + auto tile_group = table->GetTileGroup(tg); + + for (int tuple_slot_id = START_OID; tuple_slot_id < DEFAULT_TUPLES_PER_TILEGROUP; + tuple_slot_id++) { + txn_id_t tuple_txn_id = tile_group->GetHeader()->GetTransactionId(tuple_slot_id); + if (tuple_txn_id != INVALID_TXN_ID) { + PELOTON_ASSERT(tuple_txn_id == INITIAL_TXN_ID); + std::unique_ptr t(new storage::Tuple(schema, true)); + for (size_t col = 0; col < schema->GetColumnCount(); col++) { + t->SetValue(col, tile_group->GetValue(tuple_slot_id, col), pool.get()); + } + ItemPointer p(tile_group->GetTileGroupId(), tuple_slot_id); + auto txn = concurrency::TransactionManagerFactory::GetInstance() + .BeginTransaction(IsolationLevelType::SERIALIZABLE); + ItemPointer *ip = nullptr; + table->InsertInIndexes(t.get(), p, txn, &ip); + tile_group->GetHeader()->SetIndirection(tuple_slot_id, ip); + concurrency::TransactionManagerFactory::GetInstance().CommitTransaction( + txn); + } + } + } + } + +} + +void WalRecovery::ReplayAllTxns(){ + + for(auto it = commited_txns_.begin(); it!=commited_txns_.end(); it++){ + ReplaySingleTxn(it->first); + } + + // reset epoch + if (max_epoch_id_ != INVALID_EID) { + auto &epoch_manager = concurrency::EpochManagerFactory::GetInstance(); + epoch_manager.Reset(max_epoch_id_+1); + } +} + + +void WalRecovery::ReplayLogFile(){ + + size_t curr_txn_offset = 0; + + // Pass 1 + ParseFromDisk(ReplayStage::PASS_1); + + + for(auto it = all_txns_.begin(); it!=all_txns_.end(); it++){ + + if(it->second.first != LogRecordType::TRANSACTION_COMMIT) + continue; + + auto offset_pair = std::make_pair(curr_txn_offset, curr_txn_offset); + commited_txns_.insert(std::make_pair(it->first, offset_pair)); + + curr_txn_offset += it->second.second; + } + + // Pass 2 + log_buffer_ = new char[log_buffer_size_]; + ParseFromDisk(ReplayStage::PASS_2); + + ReplayAllTxns(); +} + + + +} +} diff --git a/src/network/postgres_protocol_handler.cpp b/src/network/postgres_protocol_handler.cpp index ffbb786b88e..5dc1852d18f 100644 --- a/src/network/postgres_protocol_handler.cpp +++ b/src/network/postgres_protocol_handler.cpp @@ -862,7 +862,9 @@ void PostgresProtocolHandler::ExecExecuteMessageGetResult(ResultType status) { } void PostgresProtocolHandler::GetResult() { - traffic_cop_->ExecuteStatementPlanGetResult(); + +// traffic_cop_->ExecuteStatementPlanGetResult(); + auto status = traffic_cop_->ExecuteStatementGetResult(); switch (protocol_type_) { case NetworkProtocolType::POSTGRES_JDBC: diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 1f3d9195038..106d5edcd00 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -25,7 +25,6 @@ #include "executor/executor_context.h" #include "gc/gc_manager_factory.h" #include "index/index.h" -#include "logging/log_manager.h" #include "storage/abstract_table.h" #include "storage/data_table.h" #include "storage/database.h" @@ -721,9 +720,13 @@ bool DataTable::CheckForeignKeySrcAndCascade( return false; } - transaction_manager.PerformInsert(current_txn, location, - index_entry_ptr); - + std::vector values; + uint32_t num_columns = src_table->GetSchema()->GetColumnCount(); + for (uint32_t col_id = 0; col_id < num_columns; col_id++) { + values.push_back(src_new_tuple.GetValue(col_id)); + } + transaction_manager.PerformInsert(current_txn, location, index_entry_ptr, + reinterpret_cast(values.data(), values.size())); break; } } diff --git a/src/storage/tile_group_header.cpp b/src/storage/tile_group_header.cpp index 1e0b450144e..5942c12a2d1 100644 --- a/src/storage/tile_group_header.cpp +++ b/src/storage/tile_group_header.cpp @@ -22,7 +22,6 @@ #include "common/printable.h" #include "concurrency/transaction_manager_factory.h" #include "gc/gc_manager.h" -#include "logging/log_manager.h" #include "storage/backend_manager.h" #include "type/value.h" #include "storage/tuple.h" diff --git a/src/threadpool/logger_queue_pool.cpp b/src/threadpool/logger_queue_pool.cpp new file mode 100644 index 00000000000..a8a4cc67dd6 --- /dev/null +++ b/src/threadpool/logger_queue_pool.cpp @@ -0,0 +1,54 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logger_queue_pool.cpp +// +// Identification: src/threadpool/logger_queue_pool.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "threadpool/logger_queue_pool.h" +#include "common/container/lock_free_queue.h" +#include "logging/wal_logger.h" +#include + +namespace peloton{ +namespace threadpool{ + +void LoggerFunc(std::atomic_bool *is_running, LoggerQueue *logger_queue) { + constexpr auto kMinPauseTime = std::chrono::microseconds(1); + constexpr auto kMaxPauseTime = std::chrono::microseconds(1000); + auto pause_time = kMinPauseTime; + + logging::WalLogger logger; + + while (is_running->load() || !logger_queue->IsEmpty()) { + + logging::LogBuffer *log_buffer = nullptr; + + if (!logger_queue->Dequeue(log_buffer)) { + // Polling with exponential backoff + std::this_thread::sleep_for(pause_time); + pause_time = std::min(pause_time * 2, kMaxPauseTime); + } else { + + logger.PerformCompaction(log_buffer); + + if(logger.IsFlushNeeded(!logger_queue->IsEmpty())){ + logger.FlushToDisk(); + } + + // TODO(gandeevan): free log buffers + pause_time = kMinPauseTime; + } + } + + logger.FlushToDisk(); + +} + +} +} \ No newline at end of file diff --git a/src/traffic_cop/traffic_cop.cpp b/src/traffic_cop/traffic_cop.cpp index a87d99c0ac5..389f250050b 100644 --- a/src/traffic_cop/traffic_cop.cpp +++ b/src/traffic_cop/traffic_cop.cpp @@ -104,12 +104,13 @@ ResultType TrafficCop::CommitQueryHelper() { // If this exception is caused by a query in a transaction, // I will block following queries in that transaction until 'COMMIT' or // 'ROLLBACK' After receive 'COMMIT', see if it is rollback or really commit. + if (curr_state.second != ResultType::ABORTED) { // txn committed - return txn_manager.CommitTransaction(txn); + return txn_manager.CommitTransaction(txn, GetOnCompleteCallback()); } else { // otherwise, rollback - return txn_manager.AbortTransaction(txn); + return txn_manager.AbortTransaction(txn, GetOnCompleteCallback()); } } @@ -122,8 +123,7 @@ ResultType TrafficCop::AbortQueryHelper() { if (curr_state.second != ResultType::ABORTED) { auto txn = curr_state.first; auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto result = txn_manager.AbortTransaction(txn); - return result; + return txn_manager.AbortTransaction(txn, GetOnCompleteCallback()); } else { delete curr_state.first; // otherwise, the txn has already been aborted @@ -180,14 +180,22 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( return p_status_; } - auto on_complete = [&result, this](executor::ExecutionResult p_status, + auto on_complete = [&result, this, &txn](executor::ExecutionResult p_status, std::vector &&values) { this->p_status_ = p_status; + // TODO (Tianyi) I would make a decision on keeping one of p_status or // error_message in my next PR this->error_message_ = std::move(p_status.m_error_message); result = std::move(values); - task_callback_(task_callback_arg_); + + // COMMIT single statement transaction + ResultType result = this->ExecuteStatementPlanGetResult(); + + if(result != ResultType::QUEUING) { + p_status_.m_result = result; + task_callback_(task_callback_arg_); + } }; auto &pool = threadpool::MonoQueuePool::GetInstance(); @@ -200,36 +208,47 @@ executor::ExecutionResult TrafficCop::ExecuteHelper( LOG_TRACE("Check Tcop_txn_state Size After ExecuteHelper %lu", tcop_txn_state_.size()); + return p_status_; } -void TrafficCop::ExecuteStatementPlanGetResult() { - if (p_status_.m_result == ResultType::FAILURE) return; +ResultType TrafficCop::ExecuteStatementPlanGetResult() { + + + if (p_status_.m_result == ResultType::FAILURE) { + if(!tcop_txn_state_.empty()) { + tcop_txn_state_.pop(); + } + return ResultType::FAILURE; + } + + if(tcop_txn_state_.empty()) return ResultType::NOOP; auto txn_result = GetCurrentTxnState().first->GetResult(); + if (single_statement_txn_ || txn_result == ResultType::FAILURE) { LOG_TRACE("About to commit/abort: single stmt: %d,txn_result: %s", single_statement_txn_, ResultTypeToString(txn_result).c_str()); + switch (txn_result) { case ResultType::SUCCESS: // Commit single statement - LOG_TRACE("Commit Transaction"); - p_status_.m_result = CommitQueryHelper(); - break; + return CommitQueryHelper(); case ResultType::FAILURE: default: // Abort - LOG_TRACE("Abort Transaction"); if (single_statement_txn_) { LOG_TRACE("Tcop_txn_state size: %lu", tcop_txn_state_.size()); - p_status_.m_result = AbortQueryHelper(); + return AbortQueryHelper(); } else { tcop_txn_state_.top().second = ResultType::ABORTED; p_status_.m_result = ResultType::ABORTED; + return ResultType::ABORTED; } } } + return ResultType::SUCCESS; } /* @@ -275,6 +294,7 @@ std::shared_ptr TrafficCop::PrepareStatement( // --multi-statements except BEGIN in a transaction if (!tcop_txn_state_.empty()) { single_statement_txn_ = false; + tcop_txn_state_.top().first->SetSingleStatementTxn(single_statement_txn_); // multi-statment txn has been aborted, just skip this query, // and do not need to parse or execute this query anymore. // Do not return nullptr in case that 'COMMIT' cannot be execute, @@ -297,6 +317,8 @@ std::shared_ptr TrafficCop::PrepareStatement( single_statement_txn_ = true; } auto txn = txn_manager.BeginTransaction(thread_id); + txn->SetSingleStatementTxn(single_statement_txn_); + // this shouldn't happen if (txn == nullptr) { LOG_TRACE("Begin txn failed"); @@ -568,15 +590,40 @@ ResultType TrafficCop::ExecuteStatement( static_cast(statement->GetQueryType())); try { + + switch (statement->GetQueryType()) { case QueryType::QUERY_BEGIN: { return BeginQueryHelper(thread_id); } case QueryType::QUERY_COMMIT: { - return CommitQueryHelper(); + this->is_queuing_ = true; + auto &pool = threadpool::MonoQueuePool::GetInstance(); + + pool.SubmitTask([this] { + ResultType result = this->CommitQueryHelper(); + + if(result!=ResultType::QUEUING) + task_callback_(task_callback_arg_); + }); + + return ResultType::QUEUING; } + case QueryType::QUERY_ROLLBACK: { - return AbortQueryHelper(); + + this->is_queuing_ = true; + auto &pool = threadpool::MonoQueuePool::GetInstance(); + + pool.SubmitTask([this] { + ResultType result = this->AbortQueryHelper(); + + if(result!=ResultType::QUEUING) { + task_callback_(task_callback_arg_); + } + }); + + return ResultType::QUEUING; } default: // The statement may be out of date @@ -596,6 +643,7 @@ ResultType TrafficCop::ExecuteStatement( ExecuteHelper(statement->GetPlanTree(), params, result, result_format, thread_id); + if (GetQueuing()) { return ResultType::QUEUING; } else { diff --git a/test/brain/query_logger_test.cpp b/test/brain/query_logger_test.cpp index c3affa7fd40..0db2c8f8880 100644 --- a/test/brain/query_logger_test.cpp +++ b/test/brain/query_logger_test.cpp @@ -21,14 +21,22 @@ namespace test { class QueryLoggerTests : public PelotonTest { protected: + + bool SortByFingerprint(std::string x, std::string y){ + + std::string x_query = x.substr(0, x.find_first_of("|")); + std::string y_query = y.substr(0, y.find_first_of("|")); + + return x_query < y_query; + } + void SetUp() override { settings::SettingsManager::SetBool(settings::SettingId::brain, true); PelotonInit::Initialize(); // query to check that logging is done select_query_ = - "SELECT query_string, fingerprint FROM " - "peloton.pg_catalog.pg_query_history;"; + "SELECT query_string, fingerprint FROM pg_catalog.pg_query_history order by query_string;"; brain::QueryLogger::Fingerprint fingerprint{select_query_}; select_query_fingerprint_ = fingerprint.GetFingerprint(); @@ -52,6 +60,10 @@ class QueryLoggerTests : public PelotonTest { // give some time to actually log this query sleep(wait_time_); + //order by query_string + std::sort(expected_result.begin(), expected_result.end(), + [this] (std::string x, std::string y) { return SortByFingerprint(x, y); }); + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), expected_result, true); @@ -87,6 +99,11 @@ class QueryLoggerTests : public PelotonTest { temporary_expected_result.begin(), temporary_expected_result.end()); temporary_expected_result.clear(); + + //order by query_string + std::sort(expected_result.begin(), expected_result.end(), + [this] (std::string x, std::string y) { return SortByFingerprint(x, y); }); + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(select_query_.c_str(), expected_result, true); @@ -125,9 +142,10 @@ TEST_F(QueryLoggerTests, QueriesTest) { TestSimpleUtil("INSERT INTO test VALUES (1);", expected_result); TestSimpleUtil("INSERT INTO test VALUES (2);", expected_result); + // the select_query_ done at the end of above test + // won't be logged till the txn below commits expected_result - .pop_back(); // the select_query_ done at the end of above test - // won't be logged till the txn below commits + .pop_back(); // check if the queries are logged only when the transaction actually commits TestTransactionUtil("BEGIN;", expected_result, false); diff --git a/test/include/logging/testing_logging_util.h b/test/include/logging/testing_logging_util.h index c2e3eb2c4b6..346f3247d6b 100644 --- a/test/include/logging/testing_logging_util.h +++ b/test/include/logging/testing_logging_util.h @@ -12,17 +12,8 @@ #include "common/harness.h" - -#include "logging/log_manager.h" -#include "common/logger.h" -#include "logging/loggers/wal_frontend_logger.h" -#include "logging/loggers/wal_backend_logger.h" -#include "logging/records/tuple_record.h" -#include "logging/records/transaction_record.h" -#include "storage/data_table.h" -#include "storage/tile.h" -#include -#include "../executor/testing_executor_util.h" +#include "logging/wal_logger.h" +#include "logging/log_buffer.h" #define INVALID_LOGGER_IDX UINT_MAX @@ -32,246 +23,6 @@ namespace test { //===--------------------------------------------------------------------===// // LoggingTests Utility //===--------------------------------------------------------------------===// -enum logging_op_type { - LOGGING_OP_PREPARE, - LOGGING_OP_BEGIN, - LOGGING_OP_INSERT, - LOGGING_OP_UPDATE, - LOGGING_OP_DELETE, - LOGGING_OP_COMMIT, - LOGGING_OP_ABORT, - LOGGING_OP_DONE, - LOGGING_OP_COLLECT, - LOGGING_OP_FLUSH, -}; - -class TestingLoggingUtil { - public: - static std::vector BuildTupleRecords( - std::vector> &tuples, - size_t tile_group_size, size_t table_tile_group_count); - - static std::vector BuildTupleRecordsForRestartTest( - std::vector> &tuples, - size_t tile_group_size, size_t table_tile_group_count, - int out_of_range_tuples, int delete_tuples); - - static std::vector> BuildTuples( - storage::DataTable *table, int num_rows, bool mutate, bool random); -}; - -// Operation of the logger -struct LoggingOperation { - logging_op_type op; - cid_t cid; - LoggingOperation(logging_op_type op_, cid_t cid_ = INVALID_CID) - : op(op_), cid(cid_){}; -}; - -// The schedule for logging execution -struct LoggingSchedule { - std::vector operations; - LoggingSchedule() {} -}; - -class LoggerId { - public: - unsigned int front; - unsigned int back; - LoggerId(unsigned int front_, unsigned int back_ = INVALID_LOGGER_IDX) - : front(front_), back(back_) {} - - LoggerId() : front(INVALID_LOGGER_IDX), back(INVALID_LOGGER_IDX) {} -}; - -// A thread wrapper that runs a frontend logger -class AbstractLoggingThread { - public: - AbstractLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_) - : frontend_id(frontend_id_), - schedule(sched), - log_manager(log_manager_), - cur_seq(0), - go(false), - table(table_) {} - - virtual void RunLoop() = 0; - - void MainLoop(); - - std::thread Run() { - return std::thread(&AbstractLoggingThread::RunLoop, this); - } - - virtual void ExecuteNext() = 0; - - virtual ~AbstractLoggingThread() {} - - unsigned int frontend_id = INVALID_LOGGER_IDX; - LoggingSchedule *schedule; - logging::LogManager *log_manager; - int cur_seq; - bool go; - storage::DataTable *table; -}; - -class FrontendLoggingThread : public AbstractLoggingThread { - public: - FrontendLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_) - : AbstractLoggingThread(sched, log_manager_, frontend_id_, table_) {} - - void RunLoop(); - - void ExecuteNext(); - - ~FrontendLoggingThread() {} - - logging::WriteAheadFrontendLogger *frontend_logger = nullptr; - - // result of committed cid. used by front end logger only - std::vector results; -}; - -class BackendLoggingThread : public AbstractLoggingThread { - public: - BackendLoggingThread(LoggingSchedule *sched, - logging::LogManager *log_manager_, - unsigned int frontend_id_, storage::DataTable *table_, - unsigned int backend_id_) - : AbstractLoggingThread(sched, log_manager_, frontend_id_, table_), - backend_id(backend_id_) {} - - void RunLoop(); - - void ExecuteNext(); - - ~BackendLoggingThread() {} - - logging::WriteAheadBackendLogger *backend_logger = nullptr; - - unsigned int backend_id = INVALID_LOGGER_IDX; -}; - -// Logging scheduler, to make life easier writing logging test -class LoggingScheduler { - public: - LoggingScheduler(size_t num_backend_logger_per_frontend, - unsigned int num_frontend_logger, - logging::LogManager *log_manager_, - storage::DataTable *table_) - : log_manager(log_manager_), - num_frontend_logger(num_frontend_logger), - num_backend_logger_per_frontend(num_backend_logger_per_frontend), - frontend_schedules(num_frontend_logger), - backend_schedules(num_backend_logger_per_frontend * - num_frontend_logger), - table(table_) {} - - void Prepare() { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_PREPARE); - sequence[time++] = cur_id; - } - void Begin(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_BEGIN, - cid); - sequence[time++] = cur_id; - } - void Insert(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_INSERT, - cid); - sequence[time++] = cur_id; - } - void Delete(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_DELETE, - cid); - sequence[time++] = cur_id; - } - void Update(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_UPDATE, - cid); - sequence[time++] = cur_id; - } - void Abort(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_ABORT, - cid); - sequence[time++] = cur_id; - } - void Commit(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_COMMIT, - cid); - sequence[time++] = cur_id; - } - void Done(cid_t cid) { - backend_schedules[cur_id.back].operations.emplace_back(LOGGING_OP_DONE, - cid); - sequence[time++] = cur_id; - } - void Collect() { - frontend_schedules[cur_id.front].operations.emplace_back( - LOGGING_OP_COLLECT); - sequence[time++] = cur_id; - } - void Flush() { - frontend_schedules[cur_id.front].operations.emplace_back(LOGGING_OP_FLUSH); - sequence[time++] = cur_id; - } - - void Init(); - - void Cleanup(); - - void Run(); - - LoggingScheduler &BackendLogger(unsigned int frontend_idx, - unsigned int backend_idx) { - PELOTON_ASSERT(frontend_idx < frontend_schedules.size()); - PELOTON_ASSERT(backend_idx < num_backend_logger_per_frontend); - cur_id.front = frontend_idx; - cur_id.back = GetBackendLoggerId(frontend_idx, backend_idx); - return *this; - } - - LoggingScheduler &FrontendLogger(unsigned int frontend_idx) { - PELOTON_ASSERT(frontend_idx < frontend_schedules.size()); - cur_id.front = frontend_idx; - cur_id.back = INVALID_LOGGER_IDX; - return *this; - } - - int time = 0; - logging::LogManager *log_manager; - - unsigned int num_frontend_logger = 1; - unsigned int num_backend_logger_per_frontend = 2; - - // the logging schedules for frontend and backend loggers - std::vector frontend_schedules; - std::vector backend_schedules; - - // the logging threads for frontend and backend loggers - std::vector frontend_threads; - std::vector backend_threads; - - // the sequence of operation - std::map sequence; - - // current id of frontend & backend loggers - LoggerId cur_id = LoggerId(INVALID_LOGGER_IDX, INVALID_LOGGER_IDX); - - bool concurrent = false; - - storage::DataTable *table; - - private: - inline unsigned int GetBackendLoggerId(unsigned int frontend_idx, - unsigned int backend_idx) { - return frontend_idx * num_backend_logger_per_frontend + backend_idx; - } -}; } // namespace test } // namespace peloton diff --git a/test/logging/buffer_pool_test.cpp b/test/logging/buffer_pool_test.cpp deleted file mode 100644 index 89de8e6cfb8..00000000000 --- a/test/logging/buffer_pool_test.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // buffer_pool_test.cpp -// // -// // Identification: test/logging/buffer_pool_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "common/harness.h" -// #include "logging/circular_buffer_pool.h" -// #include - -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Buffer Pool Tests -// //===--------------------------------------------------------------------===// - -// class BufferPoolTests : public PelotonTest {}; - -// void EnqueueTest(logging::CircularBufferPool *buffer_pool, unsigned int count) { -// for (unsigned int i = 0; i < count; i++) { -// std::unique_ptr buf(new logging::LogBuffer(nullptr)); -// buf->SetSize(i); -// buffer_pool->Put(std::move(buf)); -// } -// } - -// void DequeueTest(logging::CircularBufferPool *buffer_pool, unsigned int count) { -// for (unsigned int i = 0; i < count; i++) { -// auto buf = std::move(buffer_pool->Get()); -// PELOTON_ASSERT(buf); -// EXPECT_EQ(buf->GetSize(), i); -// } -// } - -// void BackendThread(logging::WriteAheadBackendLogger *logger, -// unsigned int count) { -// for (unsigned int i = 1; i <= count; i++) { -// logging::TransactionRecord begin_record(LOGRECORD_TYPE_TRANSACTION_COMMIT, -// i); -// logger->Log(&begin_record); -// } -// } - -// void FrontendThread(logging::WriteAheadFrontendLogger *logger, -// unsigned int count) { -// srand(time(NULL)); -// while (true) { -// for (int i = 0; i < 10; i++) { -// logger->CollectLogRecordsFromBackendLoggers(); -// } -// logger->FlushLogRecords(); -// auto cid = logger->GetMaxFlushedCommitId(); -// if (cid == count) { -// break; -// } -// int rand_num = rand() % 5 + 1; -// std::chrono::milliseconds sleep_time(rand_num); -// std::this_thread::sleep_for(sleep_time); -// } -// } - -// TEST_F(BufferPoolTests, BufferPoolBasicTest) { -// logging::CircularBufferPool buffer_pool0; -// EnqueueTest(&buffer_pool0, 5); -// EXPECT_EQ(buffer_pool0.GetSize(), 5); - -// DequeueTest(&buffer_pool0, 5); -// EXPECT_EQ(buffer_pool0.GetSize(), 0); - -// EnqueueTest(&buffer_pool0, BUFFER_POOL_SIZE); -// EXPECT_EQ(buffer_pool0.GetSize(), BUFFER_POOL_SIZE); - -// for (int i = 0; i < 10; i++) { -// logging::CircularBufferPool buffer_pool1; -// std::thread enqueue_thread(EnqueueTest, &buffer_pool1, BUFFER_POOL_SIZE); -// std::thread dequeue_thread(DequeueTest, &buffer_pool1, BUFFER_POOL_SIZE); -// enqueue_thread.join(); -// dequeue_thread.join(); -// } -// } - -// TEST_F(BufferPoolTests, LogBufferBasicTest) { -// size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// size_t table_tile_group_count = 3; - -// std::unique_ptr recovery_table( -// TestingExecutorUtil::CreateTable(tile_group_size)); - -// // prepare tuples -// auto mutate = true; -// auto random = false; -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table.get(), num_rows, mutate, -// random); -// std::vector records = -// TestingLoggingUtil::BuildTupleRecords(tuples, tile_group_size, -// table_tile_group_count); -// logging::LogBuffer log_buffer(0); -// size_t total_length = 0; -// for (auto record : records) { -// PELOTON_ASSERT(record.GetTuple()->GetSchema()); -// CopySerializeOutput output_buffer; -// record.Serialize(output_buffer); -// size_t len = record.GetMessageLength(); -// total_length += len; -// log_buffer.WriteRecord(&record); -// } -// EXPECT_EQ(log_buffer.GetSize(), total_length); -// } - -// TEST_F(BufferPoolTests, LargeTupleRecordTest) { -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// std::vector columns; - -// catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), -// "A", true); -// catalog::Column column2(type::TypeId::VARCHAR, 1024 * 1024 * 20, "B", false); - -// columns.push_back(column1); -// columns.push_back(column2); - -// std::unique_ptr key_schema(new catalog::Schema(columns)); -// std::unique_ptr tuple( -// new storage::Tuple(key_schema.get(), true)); -// tuple->SetValue(0, type::ValueFactory::GetIntegerValue(1), testing_pool); -// tuple->SetValue(1, type::ValueFactory::GetVarcharValue( -// std::string(1024 * 1024 * 20, 'e').c_str()), -// testing_pool); - -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, INITIAL_TXN_ID, -// INVALID_OID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); - -// logging::LogBuffer log_buffer(0); -// size_t total_length = 0; -// PELOTON_ASSERT(record.GetTuple()->GetSchema()); -// CopySerializeOutput output_buffer; -// record.Serialize(output_buffer); -// size_t len = record.GetMessageLength(); -// total_length += len; -// auto success = log_buffer.WriteRecord(&record); -// EXPECT_EQ(log_buffer.GetSize(), total_length); -// EXPECT_TRUE(success); -// } - -// TEST_F(BufferPoolTests, BufferPoolConcurrentTest) { -// unsigned int txn_count = 9999; - -// auto &log_manager = logging::LogManager::GetInstance(); -// logging::LogManager::GetInstance().Configure(LoggingType::NVM_WAL, true); -// log_manager.SetLoggingStatus(LoggingStatusType::LOGGING); -// log_manager.InitFrontendLoggers(); - -// logging::WriteAheadFrontendLogger *frontend_logger = -// reinterpret_cast( -// log_manager.GetFrontendLogger(0)); -// logging::WriteAheadBackendLogger *backend_logger = -// reinterpret_cast( -// log_manager.GetBackendLogger()); - -// std::thread backend_thread(BackendThread, backend_logger, txn_count); -// std::thread frontend_thread(FrontendThread, frontend_logger, txn_count); -// backend_thread.join(); -// frontend_thread.join(); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/checkpoint_test.cpp b/test/logging/checkpoint_test.cpp deleted file mode 100644 index 656ec7cfaa5..00000000000 --- a/test/logging/checkpoint_test.cpp +++ /dev/null @@ -1,248 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // checkpoint_test.cpp -// // -// // Identification: test/logging/checkpoint_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include - -// #include "common/harness.h" -// #include "catalog/catalog.h" -// #include "logging/checkpoint.h" - -// #include "logging/testing_logging_util.h" -// #include "logging/logging_util.h" -// #include "logging/loggers/wal_backend_logger.h" -// #include "logging/checkpoint/simple_checkpoint.h" -// #include "logging/checkpoint_manager.h" -// #include "storage/database.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "index/index.h" - -// #include "executor/mock_executor.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Checkpoint Tests -// //===--------------------------------------------------------------------===// - -// class CheckpointTests : public PelotonTest {}; - -// oid_t GetTotalTupleCount(size_t table_tile_group_count, cid_t next_cid) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - -// txn_manager.SetNextCid(next_cid); - -// auto txn = txn_manager.BeginTransaction(); - -// auto &catalog_manager = catalog::Manager::GetInstance(); -// oid_t total_tuple_count = 0; -// for (size_t tile_group_id = 1; tile_group_id <= table_tile_group_count; -// tile_group_id++) { -// auto tile_group = catalog_manager.GetTileGroup(tile_group_id); -// total_tuple_count += tile_group->GetActiveTupleCount(); -// } -// txn_manager.CommitTransaction(txn); -// return total_tuple_count; -// } - -// // TEST_F(CheckpointTests, CheckpointIntegrationTest) { -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto txn = txn_manager.BeginTransaction(); - -// // // Create a table and wrap it in logical tile -// // size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// // size_t table_tile_group_count = 3; - -// // oid_t default_table_oid = 13; -// // // table has 3 tile groups -// // storage::DataTable *target_table = -// // TestingExecutorUtil::CreateTable(tile_group_size, true, default_table_oid); -// // TestingExecutorUtil::PopulateTable(target_table, -// // tile_group_size * table_tile_group_count, -// // false, false, false, txn); -// // txn_manager.CommitTransaction(txn); - -// // // add table to catalog -// // auto catalog = catalog::Catalog::GetInstance(); -// // storage::Database *db(new storage::Database(DEFAULT_DB_ID)); -// // db->AddTable(target_table); -// // catalog->AddDatabase(db); - -// // // create checkpoint -// // auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// // auto &log_manager = logging::LogManager::GetInstance(); -// // log_manager.SetGlobalMaxFlushedCommitId(txn_manager.GetNextCommitId()); -// // checkpoint_manager.Configure(CheckpointType::NORMAL, false, 1); -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); -// // auto checkpointer = checkpoint_manager.GetCheckpointer(0); - -// // checkpointer->DoCheckpoint(); - -// // auto most_recent_checkpoint_cid = checkpointer->GetMostRecentCheckpointCid(); -// // EXPECT_TRUE(most_recent_checkpoint_cid != INVALID_CID); - -// // // destroy and restart -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); - -// // // recovery from checkpoint -// // log_manager.PrepareRecovery(); -// // auto recovery_checkpointer = checkpoint_manager.GetCheckpointer(0); -// // recovery_checkpointer->DoRecovery(); - -// // EXPECT_EQ(db->GetTableCount(), 1); -// // EXPECT_EQ(db->GetTable(0)->GetTupleCount(), -// // tile_group_size * table_tile_group_count); -// // catalog->DropDatabaseWithOid(db->GetOid()); -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); -// // } - -// // TEST_F(CheckpointTests, CheckpointScanTest) { -// // logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto txn = txn_manager.BeginTransaction(); - -// // // Create a table and wrap it in logical tile -// // size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// // size_t table_tile_group_count = 3; - -// // // table has 3 tile groups -// // std::unique_ptr target_table( -// // TestingExecutorUtil::CreateTable(tile_group_size)); -// // TestingExecutorUtil::PopulateTable(target_table.get(), -// // tile_group_size * table_tile_group_count, -// // false, false, false, txn); -// // txn_manager.CommitTransaction(txn); - -// // auto cid = txn_manager.GetNextCommitId() - 1; -// // LOG_INFO("Scan with cid = %d. GetExpiredEpochIdCid = %d", (int)cid, -// // (int)txn_manager.GetExpiredCid()); -// // auto schema = target_table->GetSchema(); -// // std::vector column_ids; -// // column_ids.resize(schema->GetColumnCount()); -// // std::iota(column_ids.begin(), column_ids.end(), 0); - -// // // create checkpoint -// // auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// // checkpoint_manager.Configure(CheckpointType::NORMAL, true, 1); -// // checkpoint_manager.DestroyCheckpointers(); -// // checkpoint_manager.InitCheckpointers(); -// // auto checkpointer = checkpoint_manager.GetCheckpointer(0); - -// // auto simple_checkpointer = -// // reinterpret_cast(checkpointer); - -// // simple_checkpointer->SetLogger(new logging::WriteAheadBackendLogger()); -// // simple_checkpointer->SetStartCommitId(cid); -// // simple_checkpointer->Scan(target_table.get(), DEFAULT_DB_ID); - -// // // verify results -// // auto records = simple_checkpointer->GetRecords(); -// // EXPECT_EQ(records.size(), -// // TESTS_TUPLES_PER_TILEGROUP * table_tile_group_count); -// // for (unsigned int i = 0; i < records.size(); i++) { -// // EXPECT_EQ(records[i]->GetType(), LOGRECORD_TYPE_WAL_TUPLE_INSERT); -// // } -// // } - -// TEST_F(CheckpointTests, CheckpointRecoveryTest) { -// logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// size_t tile_group_size = TESTS_TUPLES_PER_TILEGROUP; -// size_t table_tile_group_count = 3; - -// std::unique_ptr recovery_table( -// TestingExecutorUtil::CreateTable(tile_group_size)); - -// // prepare tuples -// auto mutate = true; -// auto random = false; -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table.get(), num_rows, mutate, -// random); -// std::vector records = -// TestingLoggingUtil::BuildTupleRecords(tuples, tile_group_size, -// table_tile_group_count); - -// // recovery tuples from checkpoint -// logging::SimpleCheckpoint simple_checkpoint(true); -// for (auto record : records) { -// auto tuple = record.GetTuple(); -// auto target_location = record.GetInsertLocation(); -// // recovery checkpoint from these records -// simple_checkpoint.RecoverTuple(tuple, recovery_table.get(), target_location, -// DEFAULT_RECOVERY_CID); -// } - -// // recovered tuples are visible from DEFAULT_RECOVERY_CID -// auto total_tuple_count = -// GetTotalTupleCount(table_tile_group_count, DEFAULT_RECOVERY_CID); -// EXPECT_EQ(total_tuple_count, tile_group_size * table_tile_group_count); - -// // Clean up -// for (auto &tuple : tuples) { -// tuple.reset(); -// } -// } - -// TEST_F(CheckpointTests, CheckpointModeTransitionTest) { -// logging::LoggingUtil::RemoveDirectory("pl_checkpoint", false); - -// auto &log_manager = logging::LogManager::GetInstance(); -// auto &checkpoint_manager = logging::CheckpointManager::GetInstance(); -// checkpoint_manager.DestroyCheckpointers(); - -// checkpoint_manager.Configure(CheckpointType::NORMAL, true, 1); - -// // launch checkpoint thread, wait for standby mode -// auto thread = std::thread(&logging::CheckpointManager::StartStandbyMode, -// &checkpoint_manager); - -// checkpoint_manager.WaitForModeTransition(peloton::CheckpointStatus::STANDBY, -// true); - -// // Clean up table tile state before recovery from checkpoint -// log_manager.PrepareRecovery(); - -// // Do any recovery -// checkpoint_manager.StartRecoveryMode(); - -// // Wait for standby mode -// checkpoint_manager.WaitForModeTransition(CheckpointStatus::DONE_RECOVERY, -// true); - -// // Now, enter CHECKPOINTING mode -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::CHECKPOINTING); -// auto checkpointer = checkpoint_manager.GetCheckpointer(0); -// while (checkpointer->GetCheckpointStatus() != -// CheckpointStatus::CHECKPOINTING) { -// std::chrono::milliseconds sleep_time(10); -// std::this_thread::sleep_for(sleep_time); -// } -// checkpoint_manager.SetCheckpointStatus(CheckpointStatus::INVALID); -// thread.join(); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/log_buffer_pool_test.cpp b/test/logging/log_buffer_pool_test.cpp deleted file mode 100644 index a4c4bb56cc5..00000000000 --- a/test/logging/log_buffer_pool_test.cpp +++ /dev/null @@ -1,50 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_buffer_pool_test.cpp -// -// Identification: test/logging/log_buffer_pool_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_buffer_pool.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Pool Tests -//===--------------------------------------------------------------------===// - -class LogBufferPoolTests : public PelotonTest {}; - -TEST_F(LogBufferPoolTests, PoolTest) { - - logging::LogBufferPool log_buffer_pool(1); - - size_t thread_id = log_buffer_pool.GetThreadId(); - - EXPECT_EQ(thread_id, 1); - - std::unique_ptr log_buffer(new logging::LogBuffer(1, 1)); - - log_buffer_pool.GetBuffer(1); - - size_t slot_count = log_buffer_pool.GetEmptySlotCount(); - - EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount() - 1); - - log_buffer_pool.PutBuffer(std::move(log_buffer)); - - slot_count = log_buffer_pool.GetEmptySlotCount(); - - EXPECT_EQ(slot_count, log_buffer_pool.GetMaxSlotCount()); - -} - -} -} diff --git a/test/logging/log_buffer_test.cpp b/test/logging/log_buffer_test.cpp index 29b8c7a5a3d..0546b88b8f7 100644 --- a/test/logging/log_buffer_test.cpp +++ b/test/logging/log_buffer_test.cpp @@ -6,10 +6,13 @@ // // Identification: test/logging/log_buffer_test.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-18, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// +#include "storage/tile_group_factory.h" +#include "catalog/schema.h" +#include "catalog/column.h" #include "logging/log_buffer.h" #include "common/harness.h" @@ -22,57 +25,232 @@ namespace test { class LogBufferTests : public PelotonTest {}; -TEST_F(LogBufferTests, LogBufferTest) { - - logging::LogBuffer log_buffer(1, 1); +TEST_F(LogBufferTests, InsertLogBufferTest) { + oid_t block = 1, offset = 16; + ItemPointer location(block, offset); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + double value1 = 9.1, value2 = 9.2, value3 = 9.3; - int eid = log_buffer.GetEpochId(); + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); - EXPECT_EQ(eid, 1); +// TILES + std::vector tile_column_names; + std::vector> column_names; - int thread_id = log_buffer.GetThreadId(); + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); - EXPECT_EQ(thread_id, 1); + std::vector schemas; + std::vector columns; - log_buffer.Reset(); +// SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); - eid = log_buffer.GetEpochId(); + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); - EXPECT_EQ(eid, INVALID_EID); + std::map> column_map; + column_map[0] = std::make_pair(0, 0); - bool rt = log_buffer.Empty(); + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); - EXPECT_TRUE(rt); + logging::LogRecord insert_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_INSERT, location, epoch_id, txn_id, commit_id); - char *data = log_buffer.GetData(); + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); - int num = 99; + type::Value val1 = type::ValueFactory::GetDecimalValue(value1); + type::Value val2 = type::ValueFactory::GetDecimalValue(value2); + type::Value val3 = type::ValueFactory::GetDecimalValue(value3); + std::vector values; + values.push_back(val1); + values.push_back(val2); + values.push_back(val3); + insert_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(insert_record); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4 + + sizeof(double) * 3; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); - rt = log_buffer.WriteData((char*)(&num), sizeof(num)); + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); - EXPECT_TRUE(rt); + // Insert record + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_INSERT), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); +} + +TEST_F(LogBufferTests, UpdateLogBufferTest) { + oid_t block = 1, offset = 16, new_block = 1, new_offset = 24; + ItemPointer location(block, offset); + ItemPointer new_location(new_block, new_offset); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + double value1 = 9.1, value2 = 9.2, value3 = 9.3; + oid_t target_oid1 = 2, target_oid2 = 4, target_oid3 = 5; + + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + +// TILES + std::vector tile_column_names; + std::vector> column_names; + + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); - int num2; + std::vector schemas; + std::vector columns; - PELOTON_MEMCPY(&num2, data, sizeof(num)); +// SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); - EXPECT_EQ(num2, 99); + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); - size_t size = log_buffer.GetSize(); + std::map> column_map; + column_map[0] = std::make_pair(0, 0); - EXPECT_EQ(size, sizeof(num)); + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); - log_buffer.Reset(); + logging::LogRecord update_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_UPDATE, location, new_location, epoch_id, txn_id, + commit_id); - rt = log_buffer.Empty(); + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); - EXPECT_TRUE(rt); + type::Value val1 = type::ValueFactory::GetDecimalValue(value1); + type::Value val2 = type::ValueFactory::GetDecimalValue(value2); + type::Value val3 = type::ValueFactory::GetDecimalValue(value3); + std::vector values; + values.push_back(val1); + values.push_back(val2); + values.push_back(val3); + expression::AbstractExpression *expr = nullptr; + planner::DerivedAttribute tmp_att(expr); + Target target1 = std::make_pair(target_oid1, tmp_att); + Target target2 = std::make_pair(target_oid2, tmp_att); + Target target3 = std::make_pair(target_oid3, tmp_att); + TargetList *target_list = new TargetList; + target_list->push_back(target1); + target_list->push_back(target2); + target_list->push_back(target3); + update_record.SetOffsetsArray(target_list); + update_record.SetValuesArray(reinterpret_cast(values.data()), static_cast(values.size())); + log_buffer->WriteRecord(update_record); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 7 + + (sizeof(int32_t) + sizeof(double)) * 3; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); - size = log_buffer.GetSize(); + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); - EXPECT_EQ(size, 0); - + // Update Record + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_UPDATE), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(block, log_buffer_input.ReadLong()); + EXPECT_EQ(offset, log_buffer_input.ReadLong()); + EXPECT_EQ(new_block, log_buffer_input.ReadLong()); + EXPECT_EQ(new_offset, log_buffer_input.ReadLong()); + EXPECT_EQ(update_record.GetNumValues(), log_buffer_input.ReadLong()); + EXPECT_EQ(target_oid1, log_buffer_input.ReadInt()); + EXPECT_EQ(value1, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid2, log_buffer_input.ReadInt()); + EXPECT_EQ(value2, log_buffer_input.ReadDouble()); + EXPECT_EQ(target_oid3, log_buffer_input.ReadInt()); + EXPECT_EQ(value3, log_buffer_input.ReadDouble()); } +TEST_F(LogBufferTests, DeleteLogBufferTest) { + oid_t block = 1, old_block = 1, old_offset = 8; + ItemPointer old_location(old_block, old_offset); + eid_t epoch_id = 3; + txn_id_t txn_id = 99; + cid_t commit_id = 98; + oid_t database_id = 10; + oid_t table_id = 20; + oid_t tile_group_id = 30; + + logging::LogBuffer *log_buffer; + log_buffer = new logging::LogBuffer(logging::LogManager::GetInstance().GetTransactionBufferSize()); + +// TILES + std::vector tile_column_names; + std::vector> column_names; + + tile_column_names.push_back("INTEGER COL"); + column_names.push_back(tile_column_names); + + std::vector schemas; + std::vector columns; + +// SCHEMA + catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "A", true); + columns.push_back(column1); + + std::unique_ptr schema1(new catalog::Schema(columns)); + schemas.push_back(*schema1); + + std::map> column_map; + column_map[0] = std::make_pair(0, 0); + + std::shared_ptr tile_group(storage::TileGroupFactory::GetTileGroup( + database_id, table_id, tile_group_id, nullptr, schemas, column_map, 3)); + catalog::Manager::GetInstance().AddTileGroup(block, tile_group); + + logging::LogRecord delete_record = logging::LogRecordFactory::CreateTupleRecord( + LogRecordType::TUPLE_DELETE, old_location, epoch_id, txn_id, commit_id); + + EXPECT_EQ(log_buffer->GetThreshold(), logging::LogManager::GetInstance().GetTransactionBufferSize()); + + log_buffer->WriteRecord(delete_record); + size_t buffer_size = sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t) * 3 + sizeof(int64_t) * 4; + EXPECT_EQ(buffer_size, log_buffer->GetSize()); + + ReferenceSerializeInput log_buffer_input(log_buffer->GetData(), log_buffer->GetSize()); + + // Delete record + EXPECT_EQ(buffer_size - sizeof(int32_t), log_buffer_input.ReadInt()); + EXPECT_EQ(static_cast(LogRecordType::TUPLE_DELETE), log_buffer_input.ReadEnumInSingleByte()); + EXPECT_EQ(txn_id, log_buffer_input.ReadLong()); + EXPECT_EQ(epoch_id, log_buffer_input.ReadLong()); + EXPECT_EQ(commit_id, log_buffer_input.ReadLong()); + EXPECT_EQ(database_id, log_buffer_input.ReadLong()); + EXPECT_EQ(table_id, log_buffer_input.ReadLong()); + EXPECT_EQ(old_block, log_buffer_input.ReadLong()); + EXPECT_EQ(old_offset, log_buffer_input.ReadLong()); } } +} \ No newline at end of file diff --git a/test/logging/log_record_test.cpp b/test/logging/log_record_test.cpp deleted file mode 100644 index 7d674a0e922..00000000000 --- a/test/logging/log_record_test.cpp +++ /dev/null @@ -1,66 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// log_record_test.cpp -// -// Identification: test/logging/log_record_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_record.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Log Buffer Tests -//===--------------------------------------------------------------------===// - -class LogRecordTests : public PelotonTest {}; - -TEST_F(LogRecordTests, LogRecordTest) { - - std::vector tuple_type_list = { - LogRecordType::TUPLE_INSERT, - LogRecordType::TUPLE_DELETE, - LogRecordType::TUPLE_UPDATE - }; - - for (auto type : tuple_type_list) { - logging::LogRecord tuple_record = - logging::LogRecordFactory::CreateTupleRecord(type, ItemPointer(0, 0)); - - EXPECT_EQ(tuple_record.GetType(), type); - } - - std::vector txn_type_list = { - LogRecordType::TRANSACTION_BEGIN, - LogRecordType::TRANSACTION_COMMIT - }; - - for (auto type : txn_type_list) { - logging::LogRecord txn_record = - logging::LogRecordFactory::CreateTxnRecord(type, 50); - - EXPECT_EQ(txn_record.GetType(), type); - } - - std::vector epoch_type_list = { - LogRecordType::EPOCH_BEGIN, - LogRecordType::EPOCH_END - }; - - for (auto type : epoch_type_list) { - logging::LogRecord epoch_record = - logging::LogRecordFactory::CreateEpochRecord(type, 100); - - EXPECT_EQ(epoch_record.GetType(), type); - } -} - -} -} diff --git a/test/logging/logging_test.cpp b/test/logging/logging_test.cpp index e4bee1e32b7..ad771c946de 100644 --- a/test/logging/logging_test.cpp +++ b/test/logging/logging_test.cpp @@ -1,384 +1,64 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_test.cpp -// // -// // Identification: test/logging/logging_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include "catalog/catalog.h" -// #include "common/harness.h" -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/database.h" -// #include "storage/table_factory.h" -// #include "storage/tile.h" - -// #include "executor/mock_executor.h" - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// extern peloton::LoggingType peloton_logging_mode; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Logging Tests -// //===--------------------------------------------------------------------===// - -// class LoggingTests : public PelotonTest {}; - -// TEST_F(LoggingTests, BasicLoggingTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); - -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, AllCommittedTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, LaggardTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, FastLoggerTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, BothPreparingTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// // this prepare should still get a may commit of 3 -// scheduler.BackendLogger(0, 1).Prepare(); - -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 1).Begin(6); -// scheduler.BackendLogger(0, 1).Insert(6); -// scheduler.BackendLogger(0, 1).Commit(6); -// // this call should get a may commit of 4 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(3, results[0]); -// EXPECT_EQ(3, results[1]); -// EXPECT_EQ(4, results[2]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, TwoRoundTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Insert(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Insert(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Insert(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(5, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, InsertUpdateDeleteTest) { -// std::unique_ptr table( -// TestingExecutorUtil::CreateTable(1)); - -// auto &log_manager = logging::LogManager::GetInstance(); - -// LoggingScheduler scheduler(2, 1, &log_manager, table.get()); - -// scheduler.Init(); -// // Logger 0 is always the front end logger -// // The first txn to commit starts with cid 2 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(2); -// scheduler.BackendLogger(0, 0).Insert(2); -// scheduler.BackendLogger(0, 0).Commit(2); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(3); -// scheduler.BackendLogger(0, 1).Update(3); -// scheduler.BackendLogger(0, 1).Commit(3); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// // at this point everyone should be updated to 3 -// scheduler.BackendLogger(0, 0).Prepare(); -// scheduler.BackendLogger(0, 0).Begin(4); -// scheduler.BackendLogger(0, 0).Delete(4); -// scheduler.BackendLogger(0, 0).Commit(4); -// scheduler.BackendLogger(0, 1).Prepare(); -// scheduler.BackendLogger(0, 1).Begin(5); -// scheduler.BackendLogger(0, 1).Delete(5); -// scheduler.BackendLogger(0, 1).Commit(5); -// scheduler.FrontendLogger(0).Collect(); -// scheduler.FrontendLogger(0).Flush(); -// scheduler.BackendLogger(0, 0).Done(1); -// scheduler.BackendLogger(0, 1).Done(1); - -// scheduler.Run(); - -// auto results = scheduler.frontend_threads[0].results; -// EXPECT_EQ(5, results[1]); -// scheduler.Cleanup(); -// } - -// TEST_F(LoggingTests, BasicLogManagerTest) { -// peloton_logging_mode = LoggingType::INVALID; -// auto &log_manager = logging::LogManager::GetInstance(); -// log_manager.DropFrontendLoggers(); -// log_manager.SetLoggingStatus(LoggingStatusType::INVALID); -// // just start, write a few records and exit -// catalog::Schema *table_schema = -// new catalog::Schema({TestingExecutorUtil::GetColumnInfo(0), -// TestingExecutorUtil::GetColumnInfo(1), -// TestingExecutorUtil::GetColumnInfo(2), -// TestingExecutorUtil::GetColumnInfo(3)}); -// std::string table_name("TEST_TABLE"); - -// // Create table. -// bool own_schema = true; -// bool adapt_table = false; -// storage::DataTable *table = storage::TableFactory::GetDataTable( -// 12345, 123456, table_schema, table_name, 1, own_schema, adapt_table); - -// storage::Database *test_db = new storage::Database(12345); -// test_db->AddTable(table); -// auto catalog = catalog::Catalog::GetInstance(); -// catalog->AddDatabase(test_db); -// concurrency::TransactionManager &txn_manager = -// concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// TestingExecutorUtil::PopulateTable(table, 5, true, false, false, txn); -// txn_manager.CommitTransaction(txn); -// peloton_logging_mode = LoggingType::NVM_WAL; - -// log_manager.SetSyncCommit(true); -// EXPECT_FALSE(log_manager.ContainsFrontendLogger()); -// log_manager.StartStandbyMode(); -// log_manager.GetFrontendLogger(0)->SetTestMode(true); -// log_manager.StartRecoveryMode(); -// log_manager.WaitForModeTransition(LoggingStatusType::LOGGING, true); -// EXPECT_TRUE(log_manager.ContainsFrontendLogger()); -// log_manager.SetGlobalMaxFlushedCommitId(4); -// concurrency::TransactionContext test_txn; -// cid_t commit_id = 5; -// log_manager.PrepareLogging(); -// log_manager.LogBeginTransaction(commit_id); -// ItemPointer insert_loc(table->GetTileGroup(1)->GetTileGroupId(), 0); -// ItemPointer delete_loc(table->GetTileGroup(2)->GetTileGroupId(), 0); -// ItemPointer update_old(table->GetTileGroup(3)->GetTileGroupId(), 0); -// ItemPointer update_new(table->GetTileGroup(4)->GetTileGroupId(), 0); -// log_manager.LogInsert(commit_id, insert_loc); -// log_manager.LogUpdate(commit_id, update_old, update_new); -// log_manager.LogInsert(commit_id, delete_loc); -// log_manager.LogCommitTransaction(commit_id); - -// // since we are doing sync commit we should have reached 5 already -// EXPECT_EQ(commit_id, log_manager.GetPersistentFlushedCommitId()); -// log_manager.EndLogging(); -// } - -// } // namespace test -// } // namespace peloton -// >>>>>>> master +//===----------------------------------------------------------------------===// +// +// Peloton +// +// logging_test.cpp +// +// Identification: test/logging/logging_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include /* libpqxx is used to instantiate C++ client */ +#include "logging/log_buffer.h" +#include "common/harness.h" +#include "gtest/gtest.h" +#include "network/peloton_server.h" +#include "network/postgres_protocol_handler.h" +#include "util/string_util.h" +#include "network/connection_handle_factory.h" +#include "sql/testing_sql_util.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Logging Tests +//===--------------------------------------------------------------------===// + +class LoggingTests : public PelotonTest {}; + +TEST_F(LoggingTests, InsertLoggingTest) { + settings::SettingsManager::SetString(settings::SettingId::log_directory_name, + "./logging"); + settings::SettingsManager::SetString(settings::SettingId::log_file_name, + "wal.log"); + settings::SettingsManager::SetBool(settings::SettingId::enable_logging, true); + settings::SettingsManager::SetBool(settings::SettingId::enable_recovery, false); + PelotonInit::Initialize(); + + // Generate table and data for logging. + // TestingSQLUtil::ExecuteSQLQuery("BEGIN;"); + TestingSQLUtil::ExecuteSQLQuery( + "CREATE TABLE test_table (id INTEGER, value1 " + "REAL, value2 VARCHAR(32));"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (0, 1.2, 'Aaron');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (1, 12.34, 'loves');"); + TestingSQLUtil::ExecuteSQLQuery( + "INSERT INTO test_table VALUES (2, 12345.678912345, 'databases');"); + // TestingSQLUtil::ExecuteSQLQuery("COMMIT;"); + + // make sure the data in test_table is correct + std::string sql1 = "SELECT * FROM test_table;"; + std::vector expected1 = {"0|1.2|Aaron", "1|12.34|loves", + "2|12345.7|databases"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql1, expected1, false); + + PelotonInit::Shutdown(); +} + +} +} \ No newline at end of file diff --git a/test/logging/logging_util_test.cpp b/test/logging/logging_util_test.cpp deleted file mode 100644 index 91f30aa6fff..00000000000 --- a/test/logging/logging_util_test.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_util_test.cpp -// // -// // Identification: test/logging/logging_util_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "common/harness.h" - -// #include "logging/logging_util.h" - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Logging Tests -// //===--------------------------------------------------------------------===// -// class LoggingUtilTests : public PelotonTest {}; - -// TEST_F(LoggingUtilTests, BasicLoggingUtilTest) { -// auto status = logging::LoggingUtil::CreateDirectory("test_dir", 0700); -// EXPECT_TRUE(status); - -// status = logging::LoggingUtil::RemoveDirectory("test_dir", true); -// EXPECT_TRUE(status); -// } - -// } // namespace test -// } // namespace peloton diff --git a/test/logging/new_checkpointing_test.cpp b/test/logging/new_checkpointing_test.cpp deleted file mode 100644 index b16ea56471c..00000000000 --- a/test/logging/new_checkpointing_test.cpp +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// new_checkpointing_test.cpp -// -// Identification: test/logging/new_checkpointing_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/checkpoint_manager_factory.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Checkpointing Tests -//===--------------------------------------------------------------------===// - -class NewCheckpointingTests : public PelotonTest {}; - -TEST_F(NewCheckpointingTests, MyTest) { - auto &checkpoint_manager = logging::CheckpointManagerFactory::GetInstance(); - checkpoint_manager.Reset(); - - EXPECT_TRUE(true); -} - -} -} diff --git a/test/logging/new_logging_test.cpp b/test/logging/new_logging_test.cpp deleted file mode 100644 index 5973af7d475..00000000000 --- a/test/logging/new_logging_test.cpp +++ /dev/null @@ -1,34 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// Peloton -// -// new_logging_test.cpp -// -// Identification: test/logging/new_logging_test.cpp -// -// Copyright (c) 2015-16, Carnegie Mellon University Database Group -// -//===----------------------------------------------------------------------===// - -#include "logging/log_manager_factory.h" -#include "common/harness.h" - -namespace peloton { -namespace test { - -//===--------------------------------------------------------------------===// -// Logging Tests -//===--------------------------------------------------------------------===// - -class NewLoggingTests : public PelotonTest {}; - -TEST_F(NewLoggingTests, MyTest) { - auto &log_manager = logging::LogManagerFactory::GetInstance(); - log_manager.Reset(); - - EXPECT_TRUE(true); - -} - -} -} diff --git a/test/logging/recovery_test.cpp b/test/logging/recovery_test.cpp index 8e1cd5d2a2f..c69325d95f6 100644 --- a/test/logging/recovery_test.cpp +++ b/test/logging/recovery_test.cpp @@ -1,456 +1,61 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // recovery_test.cpp -// // -// // Identification: test/logging/recovery_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include -// #include -// #include -// #include - -// #include "catalog/catalog.h" -// #include "common/harness.h" -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "index/index.h" -// #include "logging/log_manager.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/database.h" -// #include "storage/table_factory.h" -// #include "storage/tile.h" - -// #include "executor/mock_executor.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // Recovery Tests -// //===--------------------------------------------------------------------===// - -// class RecoveryTests : public PelotonTest {}; - -// std::vector BuildLoggingTuples(storage::DataTable *table, -// int num_rows, bool mutate, -// bool random) { -// std::vector tuples; -// LOG_INFO("build a vector of %d tuples", num_rows); - -// // Random values -// std::srand(std::time(nullptr)); -// const catalog::Schema *schema = table->GetSchema(); -// // Ensure that the tile group is as expected. -// PELOTON_ASSERT(schema->GetColumnCount() == 4); - -// // Insert tuples into tile_group. -// const bool allocate = true; -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// for (int rowid = 0; rowid < num_rows; rowid++) { -// int populate_value = rowid; -// if (mutate) populate_value *= 3; - -// storage::Tuple *tuple = new storage::Tuple(schema, allocate); - -// // First column is unique in this case -// tuple->SetValue(0, -// type::ValueFactory::GetIntegerValue( -// TestingExecutorUtil::PopulatedValue(populate_value, 0)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// tuple->SetValue( -// 1, -// type::ValueFactory::GetIntegerValue(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 1)), -// testing_pool); - -// tuple->SetValue(2, type::ValueFactory::GetDecimalValue( -// TestingExecutorUtil::PopulatedValue( -// random ? std::rand() : populate_value, 2)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// auto string_value = type::ValueFactory::GetVarcharValue( -// std::to_string(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 3))); -// tuple->SetValue(3, string_value, testing_pool); -// tuples.push_back(tuple); -// } -// return tuples; -// } - -// TEST_F(RecoveryTests, RestartTest) { -// auto catalog = catalog::Catalog::GetInstance(); -// LOG_TRACE("Finish creating catalog"); -// LOG_TRACE("Creating recovery_table"); -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// LOG_TRACE("Finish creating recovery_table"); - -// size_t tile_group_size = 5; -// size_t table_tile_group_count = 3; -// int num_files = 3; - -// auto mutate = true; -// auto random = false; -// cid_t default_commit_id = INVALID_CID; -// cid_t default_delimiter = INVALID_CID; - -// // XXX: for now hardcode for one logger (suffix 0) -// std::string dir_name = logging::WriteAheadFrontendLogger::wal_directory_path; - -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// int num_rows = tile_group_size * table_tile_group_count; -// std::vector> tuples = -// TestingLoggingUtil::BuildTuples(recovery_table, num_rows + 2, mutate, -// random); - -// std::vector records = -// TestingLoggingUtil::BuildTupleRecordsForRestartTest( -// tuples, tile_group_size, table_tile_group_count, 1, 1); - -// logging::LoggingUtil::RemoveDirectory(dir_name.c_str(), false); - -// auto status = logging::LoggingUtil::CreateDirectory(dir_name.c_str(), 0700); -// EXPECT_TRUE(status); -// logging::LogManager::GetInstance().SetLogDirectoryName("./"); - -// for (int i = 0; i < num_files; i++) { -// std::string file_name = dir_name + "/" + std::string("peloton_log_") + -// std::to_string(i) + std::string(".log"); -// FILE *fp = fopen(file_name.c_str(), "wb"); - -// // now set the first 8 bytes to 0 - this is for the max_log id in this file -// fwrite((void *)&default_commit_id, sizeof(default_commit_id), 1, fp); - -// // now set the next 8 bytes to 0 - this is for the max delimiter in this -// // file -// fwrite((void *)&default_delimiter, sizeof(default_delimiter), 1, fp); - -// // First write a begin record -// CopySerializeOutput output_buffer_begin; -// logging::TransactionRecord record_begin(LOGRECORD_TYPE_TRANSACTION_BEGIN, -// i + 2); -// record_begin.Serialize(output_buffer_begin); - -// fwrite(record_begin.GetMessage(), sizeof(char), -// record_begin.GetMessageLength(), fp); - -// // Now write 5 insert tuple records into this file -// for (int j = 0; j < (int)tile_group_size; j++) { -// int num_record = i * tile_group_size + j; -// CopySerializeOutput output_buffer; -// records[num_record].Serialize(output_buffer); - -// fwrite(records[num_record].GetMessage(), sizeof(char), -// records[num_record].GetMessageLength(), fp); -// } - -// // Now write 1 extra out of range tuple, only in file 0, which -// // is present at the second-but-last position of this list -// if (i == 0) { -// CopySerializeOutput output_buffer_extra; -// records[num_files * tile_group_size].Serialize(output_buffer_extra); - -// fwrite(records[num_files * tile_group_size].GetMessage(), sizeof(char), -// records[num_files * tile_group_size].GetMessageLength(), fp); -// } - -// // // Now write 1 extra delete tuple, only in the last file, which -// // // is present at the end of this list -// // if (i == num_files - 1) { -// // CopySerializeOutput output_buffer_delete; -// // records[num_files * tile_group_size + 1].Serialize(output_buffer_delete); - -// // fwrite(records[num_files * tile_group_size + 1].GetMessage(), -// // sizeof(char), -// // records[num_files * tile_group_size + 1].GetMessageLength(), fp); -// // } - -// // // Now write commit -// // logging::TransactionRecord record_commit(LOGRECORD_TYPE_TRANSACTION_COMMIT, -// // i + 2); - -// // CopySerializeOutput output_buffer_commit; -// // record_commit.Serialize(output_buffer_commit); - -// // fwrite(record_commit.GetMessage(), sizeof(char), -// // record_commit.GetMessageLength(), fp); - -// // // Now write delimiter -// // CopySerializeOutput output_buffer_delim; -// // logging::TransactionRecord record_delim(LOGRECORD_TYPE_ITERATION_DELIMITER, -// // i + 2); - -// // record_delim.Serialize(output_buffer_delim); - -// // fwrite(record_delim.GetMessage(), sizeof(char), -// // record_delim.GetMessageLength(), fp); - -// fclose(fp); -// } - -// LOG_INFO("All files created and written to."); -// int index_count = recovery_table->GetIndexCount(); -// LOG_INFO("Number of indexes on this table: %d", (int)index_count); - -// for (int index_itr = index_count - 1; index_itr >= 0; --index_itr) { -// auto index = recovery_table->GetIndex(index_itr); -// EXPECT_EQ(index->GetNumberOfTuples(), 0); -// } - -// logging::WriteAheadFrontendLogger wal_fel; - -// EXPECT_EQ(wal_fel.GetMaxDelimiterForRecovery(), num_files + 1); -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); - -// LOG_TRACE("recovery_table tile group count before recovery: %ld", -// recovery_table->GetTileGroupCount()); - -// auto &log_manager = logging::LogManager::GetInstance(); -// log_manager.SetGlobalMaxFlushedIdForRecovery(num_files + 1); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - -// wal_fel.DoRecovery(); - -// LOG_TRACE("recovery_table tile group count after recovery: %ld", -// recovery_table->GetTileGroupCount()); - -// EXPECT_EQ(recovery_table->GetTupleCount(), -// tile_group_size * table_tile_group_count - 1); -// EXPECT_EQ(wal_fel.GetLogFileCursor(), num_files); - -// txn_manager.SetNextCid(5); -// wal_fel.RecoverIndex(); -// for (int index_itr = index_count - 1; index_itr >= 0; --index_itr) { -// auto index = recovery_table->GetIndex(index_itr); -// EXPECT_EQ(index->GetNumberOfTuples(), -// tile_group_size * table_tile_group_count - 1); -// } - -// // TODO check a few more invariants here -// wal_fel.CreateNewLogFile(false); - -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files + 1); - -// wal_fel.TruncateLog(4); - -// for (int i = 1; i <= 2; i++) { -// struct stat stat_buf; -// EXPECT_NE(stat((dir_name + "/" + std::string("peloton_log_") + -// std::to_string(i) + std::string(".log")).c_str(), -// &stat_buf), -// 0); -// } - -// wal_fel.CreateNewLogFile(true); - -// EXPECT_EQ(wal_fel.GetLogFileCounter(), num_files + 2); - -// status = logging::LoggingUtil::RemoveDirectory(dir_name.c_str(), false); -// EXPECT_TRUE(status); - -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// TEST_F(RecoveryTests, BasicInsertTest) { -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// auto catalog = catalog::Catalog::GetInstance(); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// type::Value val0 = (tuples[0]->GetValue(0)); -// type::Value val1 = (tuples[0]->GetValue(1)); -// type::Value val2 = (tuples[0]->GetValue(2)); -// type::Value val3 = (tuples[0]->GetValue(3)); -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_INSERT, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), INVALID_ITEMPOINTER, tuples[0], DEFAULT_DB_ID); -// curr_rec->SetTuple(tuples[0]); -// fel.InsertTuple(curr_rec); -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_TRUE(tg_header->GetBeginCommitId(5) <= test_commit_id); -// EXPECT_EQ(tg_header->GetEndCommitId(5), MAX_CID); - -// type::Value rval0 = (recovery_table->GetTileGroupById(100)->GetValue(5, 0)); -// CmpBool cmp0 = (val0.CompareEquals(rval0)); -// EXPECT_TRUE(cmp0 == CmpBool::CmpTrue); -// type::Value rval1 = (recovery_table->GetTileGroupById(100)->GetValue(5, 1)); -// CmpBool cmp1 = (val1.CompareEquals(rval1)); -// EXPECT_TRUE(cmp1 == CmpBool::CmpTrue); -// type::Value rval2 = (recovery_table->GetTileGroupById(100)->GetValue(5, 2)); -// CmpBool cmp2 = (val2.CompareEquals(rval2)); -// EXPECT_TRUE(cmp2 == CmpBool::CmpTrue); -// type::Value rval3 = (recovery_table->GetTileGroupById(100)->GetValue(5, 3)); -// CmpBool cmp3 = (val3.CompareEquals(rval3)); -// EXPECT_TRUE(cmp3 == CmpBool::CmpTrue); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 1); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// TEST_F(RecoveryTests, BasicUpdateTest) { -// auto catalog = catalog::Catalog::GetInstance(); -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// type::Value val0 = (tuples[0]->GetValue(0)); -// type::Value val1 = (tuples[0]->GetValue(1)); -// type::Value val2 = (tuples[0]->GetValue(2)); -// type::Value val3 = (tuples[0]->GetValue(3)); - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), ItemPointer(100, 4), tuples[0], DEFAULT_DB_ID); -// curr_rec->SetTuple(tuples[0]); -// fel.UpdateTuple(curr_rec); -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_TRUE(tg_header->GetBeginCommitId(5) <= test_commit_id); -// EXPECT_EQ(tg_header->GetEndCommitId(5), MAX_CID); -// EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id); - -// type::Value rval0 = (recovery_table->GetTileGroupById(100)->GetValue(5, 0)); -// CmpBool cmp0 = (val0.CompareEquals(rval0)); -// EXPECT_TRUE(cmp0 == CmpBool::CmpTrue); -// type::Value rval1 = (recovery_table->GetTileGroupById(100)->GetValue(5, 1)); -// CmpBool cmp1 = (val1.CompareEquals(rval1)); -// EXPECT_TRUE(cmp1 == CmpBool::CmpTrue); -// type::Value rval2 = (recovery_table->GetTileGroupById(100)->GetValue(5, 2)); -// CmpBool cmp2 = (val2.CompareEquals(rval2)); -// EXPECT_TRUE(cmp2 == CmpBool::CmpTrue); -// type::Value rval3 = (recovery_table->GetTileGroupById(100)->GetValue(5, 3)); -// CmpBool cmp3 = (val3.CompareEquals(rval3)); -// EXPECT_TRUE(cmp3 == CmpBool::CmpTrue); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// catalog->DropDatabaseWithOid(DEFAULT_DB_ID, txn); -// txn_manager.CommitTransaction(txn); -// } - -// /* (From Joy) TODO FIX this -// TEST_F(RecoveryTests, BasicDeleteTest) { -// auto recovery_table = ExecutorTestsUtil::CreateTable(1024); -// auto &manager = catalog::Manager::GetInstance(); -// storage::Database db(DEFAULT_DB_ID); -// manager.AddDatabase(&db); -// db.AddTable(recovery_table); - -// EXPECT_EQ(recovery_table->GetNumberOfTuples(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// logging::WriteAheadFrontendLogger fel(true); - -// cid_t test_commit_id = 10; - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id, recovery_table->GetOid(), -// INVALID_ITEMPOINTER, ItemPointer(100, 4), nullptr, DEFAULT_DB_ID); -// fel.DeleteTuple(curr_rec); - -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_EQ(tg_header->GetEndCommitId(4), test_commit_id); - -// // EXPECT_EQ(recovery_table->GetNumberOfTuples(), 1); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); -// }*/ - -// TEST_F(RecoveryTests, OutOfOrderCommitTest) { -// auto recovery_table = TestingExecutorUtil::CreateTable(1024); -// auto catalog = catalog::Catalog::GetInstance(); -// storage::Database *db = new storage::Database(DEFAULT_DB_ID); -// catalog->AddDatabase(db); -// db->AddTable(recovery_table); - -// auto tuples = BuildLoggingTuples(recovery_table, 1, false, false); -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 1); -// EXPECT_EQ(tuples.size(), 1); -// logging::WriteAheadFrontendLogger fel(true); -// // auto bel = logging::WriteAheadBackendLogger::GetInstance(); -// cid_t test_commit_id = 10; - -// auto curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_UPDATE, test_commit_id + 1, recovery_table->GetOid(), -// INVALID_ITEMPOINTER, ItemPointer(100, 5), nullptr, DEFAULT_DB_ID); -// fel.DeleteTuple(curr_rec); -// delete curr_rec; - -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); - -// curr_rec = new logging::TupleRecord( -// LOGRECORD_TYPE_TUPLE_INSERT, test_commit_id, recovery_table->GetOid(), -// ItemPointer(100, 5), INVALID_ITEMPOINTER, tuples[0], DEFAULT_DB_ID); - -// curr_rec->SetTuple(tuples[0]); -// fel.InsertTuple(curr_rec); - -// delete curr_rec; - -// auto tg_header = recovery_table->GetTileGroupById(100)->GetHeader(); -// EXPECT_EQ(tg_header->GetEndCommitId(5), test_commit_id + 1); - -// EXPECT_EQ(recovery_table->GetTupleCount(), 0); -// EXPECT_EQ(recovery_table->GetTileGroupCount(), 2); -// } - -// } // namespace test -// } // namespace peloton +//===----------------------------------------------------------------------===// +// +// Peloton +// +// recovery_test.cpp +// +// Identification: test/logging/recovery_test.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include /* libpqxx is used to instantiate C++ client */ +#include "logging/log_buffer.h" +#include "common/harness.h" +#include "gtest/gtest.h" +#include "network/peloton_server.h" +#include "network/postgres_protocol_handler.h" +#include "util/string_util.h" +#include "network/connection_handle_factory.h" +#include "sql/testing_sql_util.h" + +namespace peloton { +namespace test { + +//===--------------------------------------------------------------------===// +// Log Recovery Tests +//===--------------------------------------------------------------------===// + +class RecoveryTests : public PelotonTest {}; + +TEST_F(RecoveryTests, InsertRecoveryTest) { + LOG_INFO("start InsertRecoveryTest"); + + // Recover from the log file and test the results + settings::SettingsManager::SetString(settings::SettingId::log_directory_name, + "./logging"); + settings::SettingsManager::SetString(settings::SettingId::log_file_name, + "wal.log"); + settings::SettingsManager::SetBool(settings::SettingId::enable_logging, true); + settings::SettingsManager::SetBool(settings::SettingId::enable_recovery, true); + + LOG_INFO("before Initialize"); + + PelotonInit::Initialize(); + + LOG_INFO("after Initialize"); + + // make sure the data in test_table is correct + std::string sql1 = "SELECT * FROM test_table;"; + std::vector expected1 = {"0|1.2|Aaron", "1|12.34|loves", + "2|12345.7|databases"}; + TestingSQLUtil::ExecuteSQLQueryAndCheckResult(sql1, expected1, false); + + LOG_INFO("after ExecuteSQLQueryAndCheckResult"); + + PelotonInit::Shutdown(); +} + +} +} \ No newline at end of file diff --git a/test/logging/testing_logging_util.cpp b/test/logging/testing_logging_util.cpp index e495e32556b..0d84ada103f 100644 --- a/test/logging/testing_logging_util.cpp +++ b/test/logging/testing_logging_util.cpp @@ -1,357 +1,19 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // logging_tests_util.cpp -// // -// // Identification: test/logging/logging_tests_util.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - -// #include "logging/testing_logging_util.h" - -// #define DEFAULT_RECOVERY_CID 15 - -// namespace peloton { -// namespace test { - -// //===--------------------------------------------------------------------===// -// // LoggingTests Util -// //===--------------------------------------------------------------------===// - -// std::vector TestingLoggingUtil::BuildTupleRecords( -// std::vector> &tuples, -// size_t tile_group_size, size_t table_tile_group_count) { -// std::vector records; -// for (size_t block = 1; block <= table_tile_group_count; ++block) { -// for (size_t offset = 0; offset < tile_group_size; ++offset) { -// ItemPointer location(block, offset); -// auto &tuple = tuples[(block - 1) * tile_group_size + offset]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record( -// LOGRECORD_TYPE_WAL_TUPLE_INSERT, INITIAL_TXN_ID, INVALID_OID, -// location, INVALID_ITEMPOINTER, tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// } -// LOG_TRACE("Built a vector of %lu tuple WAL insert records", records.size()); -// return records; -// } - -// std::vector -// TestingLoggingUtil::BuildTupleRecordsForRestartTest( -// std::vector> &tuples, -// size_t tile_group_size, size_t table_tile_group_count, -// int out_of_range_tuples, int delete_tuples) { -// auto tile_group_start_oid = -// catalog::Manager::GetInstance().GetNextTileGroupId(); -// std::vector records; -// for (size_t block = 1; block <= table_tile_group_count; ++block) { -// for (size_t offset = 0; offset < tile_group_size; ++offset) { -// ItemPointer location(block + tile_group_start_oid, offset); -// auto &tuple = tuples[(block - 1) * tile_group_size + offset]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, block + 1, -// INVALID_OID, location, INVALID_ITEMPOINTER, -// tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// } -// for (int i = 0; i < out_of_range_tuples; i++) { -// ItemPointer location(tile_group_size + tile_group_start_oid, -// table_tile_group_count + i); -// auto &tuple = tuples[tile_group_size * table_tile_group_count + i]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_INSERT, 1000, -// INVALID_OID, location, INVALID_ITEMPOINTER, -// tuple.get(), DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } -// for (int i = 0; i < delete_tuples; i++) { -// ItemPointer location(tile_group_start_oid + 1, 0); -// auto &tuple = tuples[tile_group_size * table_tile_group_count + -// out_of_range_tuples + i]; -// PELOTON_ASSERT(tuple->GetSchema()); -// logging::TupleRecord record(LOGRECORD_TYPE_WAL_TUPLE_DELETE, 4, INVALID_OID, -// INVALID_ITEMPOINTER, location, nullptr, -// DEFAULT_DB_ID); -// record.SetTuple(tuple.get()); -// records.push_back(record); -// } - -// LOG_TRACE("Built a vector of %lu tuple WAL insert records", records.size()); -// return records; -// } - -// std::vector> TestingLoggingUtil::BuildTuples( -// storage::DataTable *table, int num_rows, bool mutate, bool random) { -// std::vector> tuples; -// LOG_TRACE("build a vector of %d tuples", num_rows); - -// // Random values -// std::srand(std::time(nullptr)); -// const catalog::Schema *schema = table->GetSchema(); -// // Ensure that the tile group is as expected. -// PELOTON_ASSERT(schema->GetColumnCount() == 4); - -// // Insert tuples into tile_group. -// const bool allocate = true; -// auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - -// for (int rowid = 0; rowid < num_rows; rowid++) { -// int populate_value = rowid; -// if (mutate) populate_value *= 3; - -// std::shared_ptr tuple(new storage::Tuple(schema, allocate)); - -// // First column is unique in this case -// tuple->SetValue(0, -// type::ValueFactory::GetIntegerValue( -// TestingExecutorUtil::PopulatedValue(populate_value, 0)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// tuple->SetValue( -// 1, -// type::ValueFactory::GetIntegerValue(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 1)), -// testing_pool); - -// tuple->SetValue(2, type::ValueFactory::GetDecimalValue( -// TestingExecutorUtil::PopulatedValue( -// random ? std::rand() : populate_value, 2)), -// testing_pool); - -// // In case of random, make sure this column has duplicated values -// auto string_value = type::ValueFactory::GetVarcharValue( -// std::to_string(TestingExecutorUtil::PopulatedValue( -// random ? std::rand() % (num_rows / 3) : populate_value, 3))); -// tuple->SetValue(3, string_value, testing_pool); -// PELOTON_ASSERT(tuple->GetSchema()); -// tuples.push_back(std::move(tuple)); -// } -// return tuples; -// } - -// // ======================================================================= -// // Abstract Logging Thread -// // ======================================================================= -// void AbstractLoggingThread::MainLoop() { -// while (true) { -// while (!go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// }; -// ExecuteNext(); -// if (cur_seq == (int)schedule->operations.size()) { -// go = false; -// return; -// } -// go = false; -// } -// } - -// // ======================================================================= -// // Frontend Logging Thread -// // ======================================================================= -// void FrontendLoggingThread::RunLoop() { -// frontend_logger = reinterpret_cast( -// log_manager->GetFrontendLogger(frontend_id)); - -// // Assume txns up to cid = 1 is committed -// frontend_logger->SetMaxFlushedCommitId(1); - -// MainLoop(); -// } - -// void FrontendLoggingThread::ExecuteNext() { -// // Prepare data for operation -// logging_op_type op = schedule->operations[cur_seq].op; - -// cur_seq++; - -// // Execute the operation -// switch (op) { -// case LOGGING_OP_COLLECT: { -// LOG_TRACE("Execute Collect"); -// PELOTON_ASSERT(frontend_logger); -// frontend_logger->CollectLogRecordsFromBackendLoggers(); -// break; -// } -// case LOGGING_OP_FLUSH: { -// LOG_TRACE("Execute Flush"); -// PELOTON_ASSERT(frontend_logger); -// frontend_logger->FlushLogRecords(); -// results.push_back(frontend_logger->GetMaxFlushedCommitId()); -// break; -// } -// default: { -// LOG_TRACE("Unsupported operation type!"); -// PELOTON_ASSERT(false); -// break; -// } -// } -// } - -// void BackendLoggingThread::RunLoop() { -// backend_logger = reinterpret_cast( -// log_manager->GetBackendLogger()); - -// MainLoop(); -// } - -// void BackendLoggingThread::ExecuteNext() { -// // Prepare data for operation -// logging_op_type op = schedule->operations[cur_seq].op; -// cid_t cid = schedule->operations[cur_seq].cid; - -// cur_seq++; - -// // Execute the operation -// switch (op) { -// case LOGGING_OP_PREPARE: { -// LOG_TRACE("Execute Prepare"); -// log_manager->PrepareLogging(); -// break; -// } -// case LOGGING_OP_BEGIN: { -// LOG_TRACE("Execute Begin txn %d", (int)cid); -// log_manager->LogBeginTransaction(cid); -// break; -// } -// case LOGGING_OP_INSERT: { -// LOG_TRACE("Execute Insert txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_INSERT, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_UPDATE: { -// LOG_TRACE("Execute Update txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_UPDATE, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_DELETE: { -// LOG_TRACE("Execute Delete txn %d", (int)cid); -// auto tuple = TestingLoggingUtil::BuildTuples(table, 1, false, false)[0]; -// std::unique_ptr tuple_record( -// backend_logger->GetTupleRecord(LOGRECORD_TYPE_TUPLE_DELETE, cid, 1, -// DEFAULT_DB_ID, INVALID_ITEMPOINTER, -// INVALID_ITEMPOINTER, tuple.get())); -// backend_logger->Log(tuple_record.get()); -// tuple.reset(); -// break; -// } -// case LOGGING_OP_DONE: { -// LOG_TRACE("Execute Done txn %d", (int)cid); -// log_manager->DoneLogging(); -// break; -// } -// case LOGGING_OP_COMMIT: { -// LOG_TRACE("Execute Commit txn %d", (int)cid); -// std::unique_ptr record(new logging::TransactionRecord( -// LOGRECORD_TYPE_TRANSACTION_COMMIT, cid)); -// PELOTON_ASSERT(backend_logger); -// backend_logger->Log(record.get()); -// break; -// } -// case LOGGING_OP_ABORT: { -// LOG_TRACE("Execute Abort txn %d", (int)cid); -// std::unique_ptr record(new logging::TransactionRecord( -// LOGRECORD_TYPE_TRANSACTION_ABORT, cid)); -// PELOTON_ASSERT(backend_logger); -// backend_logger->Log(record.get()); -// break; -// } -// default: { -// LOG_ERROR("Unsupported operation type!"); -// break; -// } -// } -// } - -// // ========================================================================= -// // Logging Scheduler -// // ========================================================================== -// void LoggingScheduler::Run() { -// // Run the txns according to the schedule -// if (!concurrent) { -// for (auto itr = sequence.begin(); itr != sequence.end(); itr++) { -// auto front_id = itr->second.front; -// auto backend_id = itr->second.back; -// PELOTON_ASSERT(front_id != INVALID_LOGGER_IDX); - -// // frontend logger's turn -// if (backend_id == INVALID_LOGGER_IDX) { -// LOG_TRACE("Execute Frontend Thread %d", (int)front_id); -// frontend_threads[front_id].go = true; -// while (frontend_threads[front_id].go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// } -// LOG_TRACE("Done Frontend Thread %d", (int)front_id); -// } else { -// // backend logger's turn -// LOG_TRACE("Execute Backend Thread (%d, %d)", (int)front_id, -// (int)backend_id % num_backend_logger_per_frontend); -// backend_threads[backend_id].go = true; -// while (backend_threads[backend_id].go) { -// std::chrono::milliseconds sleep_time(1); -// std::this_thread::sleep_for(sleep_time); -// } -// LOG_TRACE("Done Backend Thread (%d, %d)", (int)front_id, -// (int)backend_id % num_backend_logger_per_frontend); -// } -// } -// } -// } - -// void LoggingScheduler::Init() { -// logging::LogManager::GetInstance().Configure( -// LoggingType::NVM_WAL, true, num_frontend_logger, -// LoggerMappingStrategyType::MANUAL); -// log_manager->SetLoggingStatus(LoggingStatusType::LOGGING); -// log_manager->InitFrontendLoggers(); - -// for (unsigned int i = 0; i < num_frontend_logger; i++) { -// frontend_threads.emplace_back(&frontend_schedules[i], log_manager, i, -// table); -// } -// for (unsigned int i = 0; -// i < num_frontend_logger * num_backend_logger_per_frontend; i++) { -// backend_threads.emplace_back(&backend_schedules[i], log_manager, i, table, -// i % num_backend_logger_per_frontend); -// } - -// // Spawn frontend logger threads -// for (int i = 0; i < (int)frontend_schedules.size(); i++) { -// std::thread t = frontend_threads[i].Run(); -// t.detach(); -// } - -// // Spawn backend logger threads -// for (int i = 0; i < (int)backend_schedules.size(); i++) { -// std::thread t = backend_threads[i].Run(); -// t.detach(); -// } -// } - -// void LoggingScheduler::Cleanup() { log_manager->ResetFrontendLoggers(); } - -// } // namespace test -// } // namespace peloton +//===----------------------------------------------------------------------===// +// +// Peloton +// +// testing_logging_util.cpp +// +// Identification: test/index/testing_logging_util.cpp +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "logging/testing_logging_util.h" + +namespace peloton { + namespace test { + + } // namespace test +} // namespace peloton diff --git a/test/logging/write_behind_logging_test.cpp b/test/logging/write_behind_logging_test.cpp deleted file mode 100644 index 18fdca7d20d..00000000000 --- a/test/logging/write_behind_logging_test.cpp +++ /dev/null @@ -1,168 +0,0 @@ -// //===----------------------------------------------------------------------===// -// // -// // Peloton -// // -// // write_behind_logging_test.cpp -// // -// // Identification: test/logging/write_behind_logging_test.cpp -// // -// // Copyright (c) 2015-16, Carnegie Mellon University Database Group -// // -// //===----------------------------------------------------------------------===// - - -// #include "executor/testing_executor_util.h" -// #include "logging/testing_logging_util.h" -// #include "common/harness.h" - -// #include "concurrency/transaction_manager_factory.h" -// #include "executor/logical_tile_factory.h" -// #include "executor/executor_context.h" -// #include "logging/loggers/wal_frontend_logger.h" -// #include "logging/logging_util.h" -// #include "storage/data_table.h" -// #include "storage/tile.h" -// #include "storage/table_factory.h" - -// #include "executor/mock_executor.h" - -// using ::testing::NotNull; -// using ::testing::Return; -// using ::testing::InSequence; - -// namespace peloton { -// namespace test { -// class WriteBehindLoggingTests : public PelotonTest {}; - -// /* TODO: Disabled it due to arbitrary timing constraints -// void grant_thread(concurrency::TransactionManager &txn_manager){ -// for (long i = 6; i <= 20; i++){ -// std::this_thread::sleep_for(std::chrono::milliseconds(10)); -// txn_manager.SetMaxGrantCid(i); -// } -// } - -// // not sure the best way to test this, so I will spawn a new thread to bump up the grant every 10 ms -// // and ensure enough time has passed by the end of the test (we are not prematurely -// // allowing transactions to continue with unsanctioned cids -// TEST_F(WriteBehindLoggingTests, BasicGrantTest) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// txn_manager.SetMaxGrantCid(5); -// auto begin = std::chrono::high_resolution_clock::now(); -// std::thread granting_thread(grant_thread, std::ref(txn_manager)); -// for (int i = 0; i < 20; i++){ -// txn_manager.GetNextCommitId(); -// } -// auto end = std::chrono::high_resolution_clock::now(); -// std::chrono::milliseconds min_expected_dur(140); -// EXPECT_TRUE(end-begin > min_expected_dur); -// granting_thread.join(); - - -// } -// */ - -// int SeqScanCount(storage::DataTable *table, -// const std::vector &column_ids, -// expression::AbstractExpression *predicate) { -// auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// auto txn = txn_manager.BeginTransaction(); -// std::unique_ptr context( -// new executor::ExecutorContext(txn)); - -// planner::SeqScanPlan seq_scan_node(table, predicate, column_ids); -// executor::SeqScanExecutor seq_scan_executor(&seq_scan_node, context.get()); - -// EXPECT_TRUE(seq_scan_executor.Init()); -// auto tuple_cnt = 0; - -// while (seq_scan_executor.Execute()) { -// std::unique_ptr result_logical_tile( -// seq_scan_executor.GetOutput()); -// tuple_cnt += result_logical_tile->GetTupleCount(); -// } - -// txn_manager.CommitTransaction(txn); - -// return tuple_cnt; -// } - -// //check the visibility -// // TEST_F(WriteBehindLoggingTests, DirtyRangeVisibilityTest) { -// // auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); -// // auto &catalog_manager = catalog::Manager::GetInstance(); - -// // ItemPointer *index_entry_ptr = nullptr; - -// // std::unique_ptr table(TestingExecutorUtil::CreateTable()); -// // auto pool = TestingHarness::GetInstance().GetTestingPool(); - -// // txn_manager.SetNextCid(1); -// // auto txn = txn_manager.BeginTransaction(); -// // auto tuple = TestingExecutorUtil::GetTuple(table.get(), 1, pool); -// // index_entry_ptr = nullptr; -// // auto visible1 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible1, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 2 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 2, pool); -// // index_entry_ptr = nullptr; -// // auto visible2 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible2, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 3 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 3, pool); -// // index_entry_ptr = nullptr; -// // auto invisible1 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, invisible1, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 4 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 4, pool); -// // index_entry_ptr = nullptr; -// // auto invisible2 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, invisible2, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 5 -// // txn = txn_manager.BeginTransaction(); -// // tuple = TestingExecutorUtil::GetTuple(table.get(), 5, pool); -// // index_entry_ptr = nullptr; -// // auto visible3 = table->InsertTuple(tuple.get(), txn, &index_entry_ptr); -// // txn_manager.PerformInsert(txn, visible3, index_entry_ptr); -// // txn_manager.CommitTransaction(txn); - -// // // got cid 6 -// // std::vector column_ids; -// // column_ids.push_back(0); -// // column_ids.push_back(1); -// // column_ids.push_back(2); -// // column_ids.push_back(3); - -// // txn = txn_manager.BeginTransaction(); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible1.block)->GetHeader(), visible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible2.block)->GetHeader(), visible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible1.block)->GetHeader(), invisible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible2.block)->GetHeader(), invisible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible3.block)->GetHeader(), visible3.offset) == VisibilityType::OK); -// // txn_manager.AbortTransaction(txn); - -// // txn_manager.SetDirtyRange(std::make_pair(2, 4)); - -// // txn = txn_manager.BeginTransaction(); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible1.block)->GetHeader(), visible1.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible2.block)->GetHeader(), visible2.offset) == VisibilityType::OK); -// // EXPECT_FALSE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible1.block)->GetHeader(), invisible1.offset) == VisibilityType::OK); -// // EXPECT_FALSE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(invisible2.block)->GetHeader(), invisible2.offset) == VisibilityType::OK); -// // EXPECT_TRUE(txn_manager.IsVisible(txn, catalog_manager.GetTileGroup(visible3.block)->GetHeader(), visible3.offset) == VisibilityType::OK); -// // txn_manager.AbortTransaction(txn); -// // } - -// } // namespace test -// } // namespace peloton -