diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 99f58aeb53a..e8363f32549 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -28,6 +28,7 @@ #include "catalog/table_catalog.h" #include "catalog/table_metrics_catalog.h" #include "catalog/trigger_catalog.h" +#include "catalog/zone_map_catalog.h" #include "codegen/code_context.h" #include "concurrency/transaction_manager_factory.h" #include "function/date_functions.h" @@ -349,9 +350,13 @@ void Catalog::Bootstrap() { SettingsCatalog::GetInstance(txn); LanguageCatalog::GetInstance(txn); - // TODO: change pg_proc to per database + // TODO: change following catalogs to per database ProcCatalog::GetInstance(txn); + if (settings::SettingsManager::GetBool(settings::SettingId::zone_map)) { + ZoneMapCatalog::GetInstance(txn); + } + if (settings::SettingsManager::GetBool(settings::SettingId::brain)) { QueryHistoryCatalog::GetInstance(txn); } diff --git a/src/catalog/column_stats_catalog.cpp b/src/catalog/column_stats_catalog.cpp index 9aafff3bfb1..c148efa5b02 100644 --- a/src/catalog/column_stats_catalog.cpp +++ b/src/catalog/column_stats_catalog.cpp @@ -21,17 +21,11 @@ namespace peloton { namespace catalog { -ColumnStatsCatalog *ColumnStatsCatalog::GetInstance( - concurrency::TransactionContext *txn) { - static ColumnStatsCatalog column_stats_catalog{txn}; - return &column_stats_catalog; -} - -ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn) - : AbstractCatalog(txn, "CREATE TABLE " CATALOG_DATABASE_NAME +ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name) + : AbstractCatalog(txn, "CREATE TABLE " + database_name + "." CATALOG_SCHEMA_NAME "." COLUMN_STATS_CATALOG_NAME " (" - "database_id INT NOT NULL, " "table_id INT NOT NULL, " "column_id INT NOT NULL, " "num_rows INT NOT NULL, " @@ -44,20 +38,20 @@ ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn) "has_index BOOLEAN);") { // unique key: (database_id, table_id, column_id) Catalog::GetInstance()->CreateIndex(txn, - CATALOG_DATABASE_NAME, + database_name, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, COLUMN_STATS_CATALOG_NAME "_skey0", - {0, 1, 2}, + {ColumnId::TABLE_ID, ColumnId::COLUMN_ID}, true, IndexType::BWTREE); // non-unique key: (database_id, table_id) Catalog::GetInstance()->CreateIndex(txn, - CATALOG_DATABASE_NAME, + database_name, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME, COLUMN_STATS_CATALOG_NAME "_skey1", - {0, 1}, + {ColumnId::TABLE_ID}, false, IndexType::BWTREE); } @@ -65,7 +59,6 @@ ColumnStatsCatalog::ColumnStatsCatalog(concurrency::TransactionContext *txn) ColumnStatsCatalog::~ColumnStatsCatalog() {} bool ColumnStatsCatalog::InsertColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id, std::string column_name, @@ -80,7 +73,6 @@ bool ColumnStatsCatalog::InsertColumnStats(concurrency::TransactionContext *txn, std::unique_ptr tuple( new storage::Tuple(catalog_table_->GetSchema(), true)); - auto val_db_id = type::ValueFactory::GetIntegerValue(database_id); auto val_table_id = type::ValueFactory::GetIntegerValue(table_id); auto val_column_id = type::ValueFactory::GetIntegerValue(column_id); auto val_num_row = type::ValueFactory::GetIntegerValue(num_rows); @@ -110,7 +102,6 @@ bool ColumnStatsCatalog::InsertColumnStats(concurrency::TransactionContext *txn, type::ValueFactory::GetVarcharValue(column_name); type::Value val_has_index = type::ValueFactory::GetBooleanValue(has_index); - tuple->SetValue(ColumnId::DATABASE_ID, val_db_id, nullptr); tuple->SetValue(ColumnId::TABLE_ID, val_table_id, nullptr); tuple->SetValue(ColumnId::COLUMN_ID, val_column_id, nullptr); tuple->SetValue(ColumnId::NUM_ROWS, val_num_row, nullptr); @@ -127,13 +118,11 @@ bool ColumnStatsCatalog::InsertColumnStats(concurrency::TransactionContext *txn, } bool ColumnStatsCatalog::DeleteColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id) { oid_t index_offset = IndexId::SECONDARY_KEY_0; // Secondary key index std::vector values; - values.push_back(type::ValueFactory::GetIntegerValue(database_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(table_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(column_id).Copy()); @@ -141,7 +130,6 @@ bool ColumnStatsCatalog::DeleteColumnStats(concurrency::TransactionContext *txn, } std::unique_ptr> ColumnStatsCatalog::GetColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id) { std::vector column_ids( @@ -151,7 +139,6 @@ std::unique_ptr> ColumnStatsCatalog::GetColumnStats(con oid_t index_offset = IndexId::SECONDARY_KEY_0; // Secondary key index std::vector values; - values.push_back(type::ValueFactory::GetIntegerValue(database_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(table_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(column_id).Copy()); @@ -194,7 +181,6 @@ std::unique_ptr> ColumnStatsCatalog::GetColumnStats(con // Return value: number of column stats size_t ColumnStatsCatalog::GetTableStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, std::map>> &column_stats_map) { @@ -206,7 +192,6 @@ size_t ColumnStatsCatalog::GetTableStats(concurrency::TransactionContext *txn, oid_t index_offset = IndexId::SECONDARY_KEY_1; // Secondary key index std::vector values; - values.push_back(type::ValueFactory::GetIntegerValue(database_id).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(table_id).Copy()); auto result_tiles = diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index 7d9267abfa9..512f722e21d 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -33,7 +33,8 @@ SystemCatalogs::SystemCatalogs(concurrency::TransactionContext *txn, : pg_trigger_(nullptr), pg_table_metrics_(nullptr), pg_index_metrics_(nullptr), - pg_query_metrics_(nullptr) { + pg_query_metrics_(nullptr), + pg_column_stats_(nullptr) { oid_t database_oid = database->GetOid(); pg_attribute_ = new ColumnCatalog(txn, database, pool); @@ -92,6 +93,7 @@ SystemCatalogs::~SystemCatalogs() { if (pg_table_metrics_) delete pg_table_metrics_; if (pg_index_metrics_) delete pg_index_metrics_; if (pg_query_metrics_) delete pg_query_metrics_; + if (pg_column_stats_) delete pg_column_stats_; } /*@brief using sql create statement to create secondary catalog tables @@ -122,6 +124,11 @@ void SystemCatalogs::Bootstrap(concurrency::TransactionContext *txn, pg_query_metrics_ = new QueryMetricsCatalog(txn, database_name); } + if (!pg_column_stats_) { + pg_column_stats_ = new ColumnStatsCatalog(txn, database_name); + } + + // Reset oid of each catalog to avoid collisions between catalog // values added by system and users when checkpoint recovery. pg_attribute_->UpdateOid(OID_FOR_USER_OFFSET); diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index 4a3a5bcd1b2..311a392a966 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -38,7 +38,7 @@ namespace catalog { // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 #define OID_FOR_USER_OFFSET 10000 -#define CATALOG_TABLES_COUNT 10 +#define CATALOG_TABLES_COUNT 11 // Oid mask for each type #define DATABASE_OID_MASK (static_cast(catalog::CatalogType::DATABASE)) diff --git a/src/include/catalog/column_stats_catalog.h b/src/include/catalog/column_stats_catalog.h index fd643400db6..635a6d140c6 100644 --- a/src/include/catalog/column_stats_catalog.h +++ b/src/include/catalog/column_stats_catalog.h @@ -47,17 +47,15 @@ namespace catalog { class ColumnStatsCatalog : public AbstractCatalog { public: - ~ColumnStatsCatalog(); + ColumnStatsCatalog(concurrency::TransactionContext *txn, + const std::string &database_name); - // Global Singleton - static ColumnStatsCatalog *GetInstance( - concurrency::TransactionContext *txn = nullptr); + ~ColumnStatsCatalog(); //===--------------------------------------------------------------------===// // write Related API //===--------------------------------------------------------------------===// bool InsertColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id, std::string column_name, @@ -71,7 +69,6 @@ class ColumnStatsCatalog : public AbstractCatalog { type::AbstractPool *pool); bool DeleteColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id); @@ -79,32 +76,15 @@ class ColumnStatsCatalog : public AbstractCatalog { // Read-only Related API //===--------------------------------------------------------------------===// std::unique_ptr> GetColumnStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, oid_t column_id); size_t GetTableStats(concurrency::TransactionContext *txn, - oid_t database_id, oid_t table_id, std::map>> &column_stats_map); - // TODO: add more if needed - - enum ColumnId { - DATABASE_ID = 0, - TABLE_ID = 1, - COLUMN_ID = 2, - NUM_ROWS = 3, - CARDINALITY = 4, - FRAC_NULL = 5, - MOST_COMMON_VALS = 6, - MOST_COMMON_FREQS = 7, - HISTOGRAM_BOUNDS = 8, - COLUMN_NAME = 9, - HAS_INDEX = 10, - // Add new columns here in creation order - }; + // TODO: add more if needed enum ColumnStatsOffset { NUM_ROWS_OFF = 0, CARDINALITY_OFF = 1, @@ -117,7 +97,19 @@ class ColumnStatsCatalog : public AbstractCatalog { }; private: - ColumnStatsCatalog(concurrency::TransactionContext *txn); + enum ColumnId { + TABLE_ID = 0, + COLUMN_ID = 1, + NUM_ROWS = 2, + CARDINALITY = 3, + FRAC_NULL = 4, + MOST_COMMON_VALS = 5, + MOST_COMMON_FREQS = 6, + HISTOGRAM_BOUNDS = 7, + COLUMN_NAME = 8, + HAS_INDEX = 9, + // Add new columns here in creation order + }; enum IndexId { SECONDARY_KEY_0 = 0, diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index e143d4c227d..9adb335f0ab 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -14,6 +14,7 @@ #include +#include "catalog/column_stats_catalog.h" #include "catalog/constraint_catalog.h" #include "catalog/database_catalog.h" #include "catalog/index_metrics_catalog.h" @@ -125,6 +126,13 @@ class SystemCatalogs { return pg_query_metrics_; } + ColumnStatsCatalog *GetColumnStatsCatalog() { + if (!pg_column_stats_) { + throw CatalogException("Column stats catalog has not been initialized"); + } + return pg_column_stats_; + } + private: ColumnCatalog *pg_attribute_; SchemaCatalog *pg_namespace_; @@ -138,6 +146,7 @@ class SystemCatalogs { TableMetricsCatalog *pg_table_metrics_; IndexMetricsCatalog *pg_index_metrics_; QueryMetricsCatalog *pg_query_metrics_; + ColumnStatsCatalog * pg_column_stats_; }; } // namespace catalog diff --git a/src/include/optimizer/stats/stats_storage.h b/src/include/optimizer/stats/stats_storage.h index 85472c6a12d..9cd2f62b019 100644 --- a/src/include/optimizer/stats/stats_storage.h +++ b/src/include/optimizer/stats/stats_storage.h @@ -24,6 +24,7 @@ namespace peloton { namespace storage { +class Database; class DataTable; } @@ -41,10 +42,6 @@ class StatsStorage { StatsStorage(); - /* Functions for managing stats table and schema */ - - void CreateStatsTableInCatalog(); - /* Functions for adding, updating and quering stats */ void InsertOrUpdateTableStats(storage::DataTable *table, @@ -72,6 +69,7 @@ class StatsStorage { /* Functions for triggerring stats collection */ ResultType AnalyzeStatsForAllTables( + storage::Database *database, concurrency::TransactionContext *txn = nullptr); ResultType AnalyzeStatsForTable( diff --git a/src/include/settings/settings.h b/src/include/settings/settings.h index a442fd151cf..729bb7618fd 100644 --- a/src/include/settings/settings.h +++ b/src/include/settings/settings.h @@ -19,17 +19,17 @@ //===----------------------------------------------------------------------===// // Peloton port SETTING_int(port, - "Peloton port (default: 15721)", - 15721, - 1024, 65535, - false, false) + "Peloton port (default: 15721)", + 15721, + 1024, 65535, + false, false) // Maximum number of connections SETTING_int(max_connections, - "Maximum number of connections (default: 64)", - 64, - 1, 512, - true, true) + "Maximum number of connections (default: 64)", + 64, + 1, 512, + true, true) SETTING_int(rpc_port, "Peloton rpc port (default: 15445)", @@ -83,11 +83,11 @@ SETTING_string(root_cert_file, //===----------------------------------------------------------------------===// SETTING_double(bnlj_buffer_size, - "The default buffer size to use for blockwise nested loop joins (default: 1 MB)", - 1.0 * 1024.0 * 1024.0, - 1.0 * 1024, - 1.0 * 1024.0 * 1024.0 * 1024, - true, true) + "The default buffer size to use for blockwise nested loop joins (default: 1 MB)", + 1.0 * 1024.0 * 1024.0, + 1.0 * 1024, + 1.0 * 1024.0 * 1024.0 * 1024, + true, true) // Size of the MonoQueue task queue SETTING_int(monoqueue_task_queue_size, @@ -214,6 +214,10 @@ SETTING_bool(dump_ir, false, true, true) +SETTING_bool(zone_map, + "Enable zone map (default: false)", + false, false, false) + //===----------------------------------------------------------------------===// // Optimizer //===----------------------------------------------------------------------===// @@ -232,8 +236,8 @@ SETTING_int(task_execution_timeout, "execution step of optimizer, " "assuming one plan has been found (default 5000)", 5000, - 1000, 60000, - true, true) + 1000, 60000, + true, true) //===----------------------------------------------------------------------===// // GENERAL diff --git a/src/include/storage/zone_map_manager.h b/src/include/storage/zone_map_manager.h index d7ccd989e1a..33af2c9d1d7 100644 --- a/src/include/storage/zone_map_manager.h +++ b/src/include/storage/zone_map_manager.h @@ -48,8 +48,6 @@ class ZoneMapManager { ZoneMapManager(); - void CreateZoneMapTableInCatalog(); - void CreateZoneMapsForTable(storage::DataTable *table, concurrency::TransactionContext *txn); @@ -119,8 +117,6 @@ class ZoneMapManager { // Data Members //===--------------------------------------------------------------------===// std::unique_ptr pool_; - - bool zone_map_table_exists; }; } // namespace storage diff --git a/src/optimizer/stats/stats_storage.cpp b/src/optimizer/stats/stats_storage.cpp index 0824a978b21..95b22970aa9 100644 --- a/src/optimizer/stats/stats_storage.cpp +++ b/src/optimizer/stats/stats_storage.cpp @@ -14,6 +14,7 @@ #include "catalog/catalog.h" #include "catalog/column_stats_catalog.h" +#include "catalog/system_catalogs.h" #include "concurrency/transaction_manager_factory.h" #include "optimizer/stats/column_stats.h" #include "optimizer/stats/table_stats.h" @@ -31,22 +32,9 @@ StatsStorage *StatsStorage::GetInstance() { /** * StatsStorage - Constructor of StatsStorage. - * In the construcotr, `pg_column_stats` table and `samples_db` database are - * created. */ StatsStorage::StatsStorage() { pool_.reset(new type::EphemeralPool()); - CreateStatsTableInCatalog(); -} - -/** - * CreateStatsCatalog - Create 'pg_column_stats' table in the catalog database. - */ -void StatsStorage::CreateStatsTableInCatalog() { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - catalog::ColumnStatsCatalog::GetInstance(txn); - txn_manager.CommitTransaction(txn); } /** @@ -109,7 +97,9 @@ void StatsStorage::InsertOrUpdateColumnStats( LOG_TRACE("InsertOrUpdateColumnStats, %d, %lf, %lf, %s, %s, %s", num_rows, cardinality, frac_null, most_common_vals.c_str(), most_common_freqs.c_str(), histogram_bounds.c_str()); - auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); + auto column_stats_catalog = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_id) + ->GetColumnStatsCatalog(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); bool single_statement_txn = false; @@ -117,9 +107,8 @@ void StatsStorage::InsertOrUpdateColumnStats( single_statement_txn = true; txn = txn_manager.BeginTransaction(); } - column_stats_catalog->DeleteColumnStats(txn, database_id, table_id, column_id); + column_stats_catalog->DeleteColumnStats(txn, table_id, column_id); column_stats_catalog->InsertColumnStats(txn, - database_id, table_id, column_id, column_name, @@ -144,12 +133,13 @@ void StatsStorage::InsertOrUpdateColumnStats( std::shared_ptr StatsStorage::GetColumnStatsByID(oid_t database_id, oid_t table_id, oid_t column_id) { - auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); + auto column_stats_catalog = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_id) + ->GetColumnStatsCatalog(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // std::unique_ptr> column_stats_vector auto column_stats_vector = column_stats_catalog->GetColumnStats(txn, - database_id, table_id, column_id); txn_manager.CommitTransaction(txn); @@ -228,10 +218,11 @@ std::shared_ptr StatsStorage::ConvertVectorToColumnStats( */ std::shared_ptr StatsStorage::GetTableStats( oid_t database_id, oid_t table_id, concurrency::TransactionContext *txn) { - auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); + auto column_stats_catalog = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_id) + ->GetColumnStatsCatalog(); std::map>> column_stats_map; column_stats_catalog->GetTableStats(txn, - database_id, table_id, column_stats_map); @@ -254,10 +245,11 @@ std::shared_ptr StatsStorage::GetTableStats( std::shared_ptr StatsStorage::GetTableStats( oid_t database_id, oid_t table_id, std::vector column_ids, concurrency::TransactionContext *txn) { - auto column_stats_catalog = catalog::ColumnStatsCatalog::GetInstance(nullptr); + auto column_stats_catalog = + catalog::Catalog::GetInstance()->GetSystemCatalogs(database_id) + ->GetColumnStatsCatalog(); std::map>> column_stats_map; column_stats_catalog->GetTableStats(txn, - database_id, table_id, column_stats_map); @@ -274,35 +266,33 @@ std::shared_ptr StatsStorage::GetTableStats( } /** - * AnalyzeStatsForAllTables - This function iterates all databases and - * datatables to collect their stats and store them in the column_stats_catalog. + * AnalyzeStatsForAllTables - This function iterates all tables in a database + * to collect their stats and store them in the column_stats_catalog. */ ResultType StatsStorage::AnalyzeStatsForAllTables( + storage::Database *database, concurrency::TransactionContext *txn) { if (txn == nullptr) { LOG_TRACE("Do not have transaction to analyze all tables' stats."); return ResultType::FAILURE; } - auto storage_manager = storage::StorageManager::GetInstance(); + auto catalog = catalog::Catalog::GetInstance(); + auto table_catalog_entries = + catalog->GetDatabaseCatalogEntry(txn, database->GetOid()) + ->GetTableCatalogEntries(); + for (auto table_catalog_entry_pair : table_catalog_entries) { + auto table_oid = table_catalog_entry_pair.first; + auto table_catalog_entry = table_catalog_entry_pair.second; - oid_t database_count = storage_manager->GetDatabaseCount(); - LOG_TRACE("Database count: %u", database_count); - for (oid_t db_offset = 0; db_offset < database_count; db_offset++) { - auto database = storage_manager->GetDatabaseWithOffset(db_offset); - if (database->GetOid() == CATALOG_DATABASE_OID) { + if (table_catalog_entry->GetSchemaName() == CATALOG_SCHEMA_NAME) continue; - } - oid_t table_count = database->GetTableCount(); - for (oid_t table_offset = 0; table_offset < table_count; table_offset++) { - auto table = database->GetTable(table_offset); - LOG_DEBUG("Analyzing table: %s", table->GetName().c_str()); - std::unique_ptr table_stats_collector( - new TableStatsCollector(table)); - table_stats_collector->CollectColumnStats(); - InsertOrUpdateTableStats(table, table_stats_collector.get(), txn); - } + + auto table = database->GetTableWithOid(table_oid); + LOG_DEBUG("Analyzing table: %s", table_catalog_entry->GetTableName().c_str()); + AnalyzeStatsForTable(table, txn); } + return ResultType::SUCCESS; } diff --git a/src/storage/zone_map_manager.cpp b/src/storage/zone_map_manager.cpp index 01f75070aea..25cecda5566 100644 --- a/src/storage/zone_map_manager.cpp +++ b/src/storage/zone_map_manager.cpp @@ -17,6 +17,7 @@ #include "catalog/database_catalog.h" #include "concurrency/transaction_context.h" #include "concurrency/transaction_manager_factory.h" +#include "settings/settings_manager.h" #include "storage/storage_manager.h" #include "storage/data_table.h" #include "type/ephemeral_pool.h" @@ -33,22 +34,9 @@ ZoneMapManager *ZoneMapManager::GetInstance() { } ZoneMapManager::ZoneMapManager() { - zone_map_table_exists = false; pool_.reset(new type::EphemeralPool()); } -/** - * @brief The function creates the zone map table in catalog - */ -void ZoneMapManager::CreateZoneMapTableInCatalog() { - LOG_DEBUG("Create the Zone Map table"); - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - catalog::ZoneMapCatalog::GetInstance(txn); - txn_manager.CommitTransaction(txn); - zone_map_table_exists = true; -} - /** * @brief The function creates zone maps for all tile groups for a given table * @@ -283,7 +271,9 @@ bool ZoneMapManager::ShouldScanTileGroup( * * @return True if the zone map table in catalog exists and vice versa */ -bool ZoneMapManager::ZoneMapTableExists() { return zone_map_table_exists; } +bool ZoneMapManager::ZoneMapTableExists() { + return settings::SettingsManager::GetBool(settings::SettingId::zone_map); +} } // namespace storage -} // namespace peloton \ No newline at end of file +} // namespace peloton diff --git a/test/codegen/zone_map_scan_test.cpp b/test/codegen/zone_map_scan_test.cpp index f3b24ac7e53..56bc73d41a0 100644 --- a/test/codegen/zone_map_scan_test.cpp +++ b/test/codegen/zone_map_scan_test.cpp @@ -19,6 +19,7 @@ #include "expression/operator_expression.h" #include "planner/seq_scan_plan.h" #include "codegen/counting_consumer.h" +#include "settings/settings_manager.h" #include "codegen/testing_codegen_util.h" @@ -31,6 +32,10 @@ class ZoneMapScanTest : public PelotonCodeGenTest { public: ZoneMapScanTest() : PelotonCodeGenTest(TEST_TUPLES_PER_TILEGROUP), num_rows_to_insert(20) { + settings::SettingsManager::SetBool(settings::SettingId::zone_map, true); + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + // Load test table LoadTestTable(TestTableId(), num_rows_to_insert); @@ -52,11 +57,8 @@ class ZoneMapScanTest : public PelotonCodeGenTest { tile_group_header->SetImmutability(); } // Create Zone Maps. - auto catalog = catalog::Catalog::GetInstance(); - (void)catalog; storage::ZoneMapManager *zone_map_manager = storage::ZoneMapManager::GetInstance(); - zone_map_manager->CreateZoneMapTableInCatalog(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); zone_map_manager->CreateZoneMapsForTable(&table, txn); @@ -151,4 +153,4 @@ TEST_F(ZoneMapScanTest, ScanwithConjunctionPredicate) { type::ValueFactory::GetIntegerValue(21))); } } -} \ No newline at end of file +} diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 4adb7827475..f02fff3184b 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -440,7 +440,7 @@ storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( std::move(table_schema), table_name, is_catalog, - tuples_per_tilegroup_count); + (uint32_t)tuples_per_tilegroup_count); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/optimizer/selectivity_test.cpp b/test/optimizer/selectivity_test.cpp index 6794fa29b9f..5115513af7b 100644 --- a/test/optimizer/selectivity_test.cpp +++ b/test/optimizer/selectivity_test.cpp @@ -14,6 +14,8 @@ #include "catalog/catalog.h" #include "catalog/column_catalog.h" +#include "catalog/schema.h" +#include "catalog/system_catalogs.h" #include "common/harness.h" #include "common/logger.h" #include "concurrency/transaction_manager_factory.h" @@ -54,9 +56,12 @@ void ExpectSelectivityEqual(double actual, double expected, // Test range selectivity including >, <, >= and <= using uniform dataset TEST_F(SelectivityTests, RangeSelectivityTest) { + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); CreateAndLoadTable(); @@ -70,7 +75,6 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { } txn = txn_manager.BeginTransaction(); - auto catalog = catalog::Catalog::GetInstance(); auto database = catalog->GetDatabaseWithName(txn, DEFAULT_DB_NAME); auto table = catalog->GetTableWithName(txn, DEFAULT_DB_NAME, @@ -120,32 +124,52 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { // checks if getting column stats and sampling works. TEST_F(SelectivityTests, LikeSelectivityTest) { const int tuple_count = 1000; - const int tuple_per_tilegroup = 100; + const uint32_t tuple_per_tilegroup = 100; + auto catalog = catalog::Catalog::GetInstance(); - // Create a table + // Create a database auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - std::unique_ptr data_table( - TestingExecutorUtil::CreateTable(tuple_per_tilegroup, false)); - TestingExecutorUtil::PopulateTable(data_table.get(), tuple_count, false, + catalog->CreateDatabase(txn, DEFAULT_DB_NAME); + txn_manager.CommitTransaction(txn); + + // Create a table + txn = txn_manager.BeginTransaction(); + std::unique_ptr table_schema( + new catalog::Schema({TestingExecutorUtil::GetColumnInfo(0), + TestingExecutorUtil::GetColumnInfo(1), + TestingExecutorUtil::GetColumnInfo(2), + TestingExecutorUtil::GetColumnInfo(3)})); + catalog->CreateTable(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + "test_table", + false, + tuple_per_tilegroup); + auto data_table = catalog->GetTableWithName(txn, + DEFAULT_DB_NAME, + DEFAULT_SCHEMA_NAME, + "test_table"); + TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, false, true, txn); txn_manager.CommitTransaction(txn); // Run analyze txn = txn_manager.BeginTransaction(); - optimizer::StatsStorage::GetInstance()->AnalyzeStatsForTable(data_table.get(), + optimizer::StatsStorage::GetInstance()->AnalyzeStatsForTable(data_table, txn); txn_manager.CommitTransaction(txn); oid_t db_id = data_table->GetDatabaseOid(); oid_t table_id = data_table->GetOid(); - auto stats_storage = StatsStorage::GetInstance(); txn = txn_manager.BeginTransaction(); + auto stats_storage = StatsStorage::GetInstance(); auto table_stats = stats_storage->GetTableStats(db_id, table_id, txn); txn_manager.CommitTransaction(txn); table_stats->SetTupleSampler( - std::make_shared(data_table.get())); + std::make_shared(data_table)); type::Value value = type::ValueFactory::GetVarcharValue("%3"); ValueCondition condition1{"test_table.COL_D", ExpressionType::COMPARE_LIKE, @@ -162,6 +186,10 @@ TEST_F(SelectivityTests, LikeSelectivityTest) { EXPECT_EQ(like_than_sel_1, 1); EXPECT_EQ(like_than_sel_2, 0); + + txn = txn_manager.BeginTransaction(); + catalog->DropDatabaseWithName(txn, DEFAULT_DB_NAME); + txn_manager.CommitTransaction(txn); } TEST_F(SelectivityTests, EqualSelectivityTest) { diff --git a/test/optimizer/stats_storage_test.cpp b/test/optimizer/stats_storage_test.cpp index 131bfbdb6ac..40f3abde152 100644 --- a/test/optimizer/stats_storage_test.cpp +++ b/test/optimizer/stats_storage_test.cpp @@ -17,10 +17,12 @@ #include "optimizer/stats/table_stats.h" #include "storage/data_table.h" #include "storage/database.h" +#include "storage/storage_manager.h" #include "storage/tile.h" -#include "catalog/schema.h" #include "catalog/catalog.h" #include "catalog/column_stats_catalog.h" +#include "catalog/schema.h" +#include "catalog/system_catalogs.h" #include "executor/testing_executor_util.h" #include "concurrency/transaction_manager_factory.h" @@ -36,34 +38,43 @@ using namespace optimizer; class StatsStorageTests : public PelotonTest {}; const int tuple_count = 100; -const int tuple_per_tilegroup = 100; - -std::unique_ptr InitializeTestTable() { - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - auto txn = txn_manager.BeginTransaction(); - std::unique_ptr data_table( - TestingExecutorUtil::CreateTable(tuple_per_tilegroup, false)); - TestingExecutorUtil::PopulateTable(data_table.get(), tuple_count, false, - false, true, txn); - txn_manager.CommitTransaction(txn); - return data_table; -} +const uint32_t tuple_per_tilegroup = 100; storage::DataTable *CreateTestDBAndTable() { const std::string test_db_name = "test_db"; - auto database = TestingExecutorUtil::InitializeDatabase(test_db_name); + const std::string test_table_name = "test_table"; + auto catalog = catalog::Catalog::GetInstance(); + + TestingExecutorUtil::InitializeDatabase(test_db_name); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - storage::DataTable *data_table = - TestingExecutorUtil::CreateTable(tuple_per_tilegroup, false); + std::unique_ptr table_schema( + new catalog::Schema({TestingExecutorUtil::GetColumnInfo(0), + TestingExecutorUtil::GetColumnInfo(1), + TestingExecutorUtil::GetColumnInfo(2), + TestingExecutorUtil::GetColumnInfo(3)})); + catalog->CreateTable(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + std::move(table_schema), + test_table_name, + false, + tuple_per_tilegroup); + auto data_table = catalog->GetTableWithName(txn, + test_db_name, + DEFAULT_SCHEMA_NAME, + test_table_name); TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, false, true, txn); - database->AddTable(data_table); txn_manager.CommitTransaction(txn); return data_table; } +void DropTestDB() { + TestingExecutorUtil::DeleteDatabase("test_db"); +} + /** * VerifyAndPrintColumnStats - Verify whether the stats of the test table are * correctly stored in catalog and print them out. @@ -93,30 +104,32 @@ void VerifyAndPrintColumnStats(storage::DataTable *data_table, } TEST_F(StatsStorageTests, InsertAndGetTableStatsTest) { - auto data_table = InitializeTestTable(); + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + + auto data_table = CreateTestDBAndTable(); // Collect stats. std::unique_ptr table_stats_collector( - new TableStatsCollector(data_table.get())); + new TableStatsCollector(data_table)); table_stats_collector->CollectColumnStats(); // Insert stats. - auto catalog = catalog::Catalog::GetInstance(); - (void)catalog; StatsStorage *stats_storage = StatsStorage::GetInstance(); - stats_storage->InsertOrUpdateTableStats(data_table.get(), + stats_storage->InsertOrUpdateTableStats(data_table, table_stats_collector.get()); - VerifyAndPrintColumnStats(data_table.get(), 4); + VerifyAndPrintColumnStats(data_table, 4); + + DropTestDB(); } TEST_F(StatsStorageTests, InsertAndGetColumnStatsTest) { - auto catalog = catalog::Catalog::GetInstance(); - (void)catalog; StatsStorage *stats_storage = StatsStorage::GetInstance(); + auto data_table = CreateTestDBAndTable(); - oid_t database_id = 1; - oid_t table_id = 2; + oid_t database_id = data_table->GetDatabaseOid(); + oid_t table_id = data_table->GetOid(); oid_t column_id = 3; int num_rows = 10; double cardinality = 8; @@ -146,15 +159,16 @@ TEST_F(StatsStorageTests, InsertAndGetColumnStatsTest) { auto column_stats_ptr2 = stats_storage->GetColumnStatsByID(database_id, table_id, column_id + 1); EXPECT_EQ(column_stats_ptr2, nullptr); + + DropTestDB(); } TEST_F(StatsStorageTests, UpdateColumnStatsTest) { - auto catalog = catalog::Catalog::GetInstance(); - (void)catalog; StatsStorage *stats_storage = StatsStorage::GetInstance(); + auto data_table = CreateTestDBAndTable(); - oid_t database_id = 1; - oid_t table_id = 2; + oid_t database_id = data_table->GetDatabaseOid(); + oid_t table_id = data_table->GetOid(); oid_t column_id = 3; int num_row_0 = 10; @@ -193,56 +207,68 @@ TEST_F(StatsStorageTests, UpdateColumnStatsTest) { EXPECT_EQ(column_stats_ptr->frac_null, frac_null_1); EXPECT_EQ(column_stats_ptr->column_name, column_name_1); + + DropTestDB(); } TEST_F(StatsStorageTests, AnalyzeStatsForTableTest) { - auto data_table = InitializeTestTable(); + auto data_table = CreateTestDBAndTable(); // Analyze table. StatsStorage *stats_storage = StatsStorage::GetInstance(); // Must pass in the transaction. - ResultType result = stats_storage->AnalyzeStatsForTable(data_table.get()); + ResultType result = stats_storage->AnalyzeStatsForTable(data_table); EXPECT_EQ(result, ResultType::FAILURE); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - result = stats_storage->AnalyzeStatsForTable(data_table.get(), txn); + result = stats_storage->AnalyzeStatsForTable(data_table, txn); EXPECT_EQ(result, ResultType::SUCCESS); txn_manager.CommitTransaction(txn); // Check the correctness of the stats. - VerifyAndPrintColumnStats(data_table.get(), 4); + VerifyAndPrintColumnStats(data_table, 4); + + DropTestDB(); } // TODO: Add more tables. TEST_F(StatsStorageTests, AnalyzeStatsForAllTablesTest) { auto data_table = CreateTestDBAndTable(); + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + data_table->GetDatabaseOid()); StatsStorage *stats_storage = StatsStorage::GetInstance(); // Must pass in the transaction. - ResultType result = stats_storage->AnalyzeStatsForAllTables(); + ResultType result = stats_storage->AnalyzeStatsForAllTables(database); EXPECT_EQ(result, ResultType::FAILURE); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - result = stats_storage->AnalyzeStatsForAllTables(txn); + result = stats_storage->AnalyzeStatsForAllTables(database, txn); EXPECT_EQ(result, ResultType::SUCCESS); txn_manager.CommitTransaction(txn); // Check the correctness of the stats. VerifyAndPrintColumnStats(data_table, 4); + + DropTestDB(); } TEST_F(StatsStorageTests, GetTableStatsTest) { - auto data_table = InitializeTestTable(); + auto data_table = CreateTestDBAndTable(); + auto database = + storage::StorageManager::GetInstance()->GetDatabaseWithOid( + data_table->GetDatabaseOid()); StatsStorage *stats_storage = StatsStorage::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - stats_storage->AnalyzeStatsForAllTables(txn); + stats_storage->AnalyzeStatsForAllTables(database, txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); @@ -250,6 +276,8 @@ TEST_F(StatsStorageTests, GetTableStatsTest) { data_table->GetDatabaseOid(), data_table->GetOid(), txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(table_stats->num_rows, tuple_count); + + DropTestDB(); } } // namespace test diff --git a/test/sql/analyze_sql_test.cpp b/test/sql/analyze_sql_test.cpp index 2504b839342..6b2b8446495 100644 --- a/test/sql/analyze_sql_test.cpp +++ b/test/sql/analyze_sql_test.cpp @@ -42,6 +42,9 @@ void CreateAndLoadTable() { } TEST_F(AnalyzeSQLTests, AnalyzeAllTablesTest) { + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->CreateDatabase(txn, DEFAULT_DB_NAME); @@ -54,7 +57,7 @@ TEST_F(AnalyzeSQLTests, AnalyzeAllTablesTest) { // Free the database txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropDatabaseWithName(txn, DEFAULT_DB_NAME); + catalog->DropDatabaseWithName(txn, DEFAULT_DB_NAME); txn_manager.CommitTransaction(txn); } @@ -75,7 +78,7 @@ TEST_F(AnalyzeSQLTests, AnalyzeSingleTableTest) { auto catalog = catalog::Catalog::GetInstance(); storage::DataTable *db_column_stats_collector_table = catalog->GetTableWithName(txn, - CATALOG_DATABASE_NAME, + DEFAULT_DB_NAME, CATALOG_SCHEMA_NAME, COLUMN_STATS_CATALOG_NAME); EXPECT_NE(db_column_stats_collector_table, nullptr); diff --git a/test/storage/zone_map_test.cpp b/test/storage/zone_map_test.cpp index f075eda2f5a..d68938d3735 100644 --- a/test/storage/zone_map_test.cpp +++ b/test/storage/zone_map_test.cpp @@ -18,6 +18,7 @@ #include "storage/data_table.h" #include "executor/testing_executor_util.h" +#include "settings/settings_manager.h" #include "storage/tile_group.h" #include "storage/database.h" #include "storage/tile.h" @@ -120,11 +121,8 @@ storage::DataTable *CreateTestTable() { auto tile_group_header = tile_group_ptr->GetHeader(); tile_group_header->SetImmutability(); } - auto catalog = catalog::Catalog::GetInstance(); - (void)catalog; storage::ZoneMapManager *zone_map_manager = storage::ZoneMapManager::GetInstance(); - zone_map_manager->CreateZoneMapTableInCatalog(); txn = txn_manager.BeginTransaction(); zone_map_manager->CreateZoneMapsForTable((data_table.get()), txn); txn_manager.CommitTransaction(txn); @@ -155,6 +153,10 @@ expression::AbstractExpression *CreateConjunctionPredicate( } TEST_F(ZoneMapTests, ZoneMapContentsTest) { + settings::SettingsManager::SetBool(settings::SettingId::zone_map, true); + auto catalog = catalog::Catalog::GetInstance(); + catalog->Bootstrap(); + std::unique_ptr data_table(CreateTestTable()); oid_t database_id = (data_table.get())->GetDatabaseOid(); oid_t table_id = (data_table.get())->GetOid(); @@ -420,4 +422,4 @@ TEST_F(ZoneMapTests, ZoneMapMultiColumnConjunctionPredicateTest) { } } } // End test namespace -} // End peloton namespace \ No newline at end of file +} // End peloton namespace