From 5d93f4ed3ae97e4ed2d6d37ffb6bbe4ba8b21a08 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 25 Feb 2018 22:52:44 -0500 Subject: [PATCH 01/42] Added layout.h. Modified Runtime functions to handle row stores. --- src/codegen/proxy/runtime_functions_proxy.cpp | 4 +- src/codegen/runtime_functions.cpp | 15 +++++ src/codegen/tile_group.cpp | 8 ++- .../codegen/proxy/runtime_functions_proxy.h | 7 ++ src/include/codegen/runtime_functions.h | 5 ++ src/include/storage/layout.h | 66 +++++++++++++++++++ src/include/storage/tile_group.h | 9 ++- src/storage/data_table.cpp | 2 +- src/storage/layout.cpp | 43 ++++++++++++ src/storage/tile_group.cpp | 15 +++-- 10 files changed, 161 insertions(+), 13 deletions(-) create mode 100644 src/include/storage/layout.h create mode 100644 src/storage/layout.cpp diff --git a/src/codegen/proxy/runtime_functions_proxy.cpp b/src/codegen/proxy/runtime_functions_proxy.cpp index 700e948d85c..5dd34f7bae2 100644 --- a/src/codegen/proxy/runtime_functions_proxy.cpp +++ b/src/codegen/proxy/runtime_functions_proxy.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "codegen/proxy/runtime_functions_proxy.h" - #include "codegen/proxy/data_table_proxy.h" #include "codegen/proxy/tile_group_proxy.h" #include "codegen/proxy/zone_map_proxy.h" @@ -25,6 +24,9 @@ DEFINE_TYPE(ColumnLayoutInfo, "peloton::ColumnLayoutInfo", DEFINE_TYPE(AbstractExpression, "peloton::expression::AbstractExpression", MEMBER(opaque)); +DEFINE_TYPE(Schema, "peloton::catalog::Schema", + MEMBER(opaque)); + DEFINE_METHOD(peloton::codegen, RuntimeFunctions, HashCrc64); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, GetTileGroup); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, GetTileGroupLayout); diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index 466478fc55d..78d9623261b 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -19,6 +19,7 @@ #include "expression/abstract_expression.h" #include "expression/expression_util.h" #include "storage/data_table.h" +#include "storage/layout.h" #include "storage/tile_group.h" #include "storage/tile.h" #include "storage/zone_map_manager.h" @@ -101,8 +102,22 @@ void RuntimeFunctions::FillPredicateArray( // to skip over to find successive values of the column. //===----------------------------------------------------------------------===// void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, + const catalog::Schema *schema, ColumnLayoutInfo *infos, uint32_t num_cols) { + const auto& layout = tile_group->GetLayout(); + // For ROW_STORE_OID, the tile group contains a single tile + // and all the columns are in the same order as the table schema. + if (layout.IsRowStore()) { + auto tuple_location = tile_group->GetTile(0)->GetTupleLocation(0); + auto stride = schema->GetLength(); + for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { + infos[col_idx].column = tuple_location + schema->GetOffset(col_idx); + infos[col_idx].stride = stride; + infos[col_idx].is_columnar = false; + } + return; + } for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { // Map the current column to a tile and a column offset in the tile oid_t tile_offset, tile_column_offset; diff --git a/src/codegen/tile_group.cpp b/src/codegen/tile_group.cpp index 4831cee3437..8ec29f2e987 100644 --- a/src/codegen/tile_group.cpp +++ b/src/codegen/tile_group.cpp @@ -88,11 +88,17 @@ llvm::Value *TileGroup::GetTileGroupId(CodeGen &codegen, std::vector TileGroup::GetColumnLayouts( CodeGen &codegen, llvm::Value *tile_group_ptr, llvm::Value *column_layout_infos) const { + + llvm::Value *schema_ptr = codegen->CreateIntToPtr( + codegen.Const64((int64_t)&schema_), + SchemaProxy::GetType(codegen)->getPointerTo()); + // Call RuntimeFunctions::GetTileGroupLayout() uint32_t num_cols = schema_.GetColumnCount(); codegen.Call( RuntimeFunctionsProxy::GetTileGroupLayout, - {tile_group_ptr, column_layout_infos, codegen.Const32(num_cols)}); + {tile_group_ptr, schema_ptr, column_layout_infos, + codegen.Const32(num_cols)}); // Collect triplets of all columns std::vector layouts; diff --git a/src/include/codegen/proxy/runtime_functions_proxy.h b/src/include/codegen/proxy/runtime_functions_proxy.h index e012501efc0..d264b160963 100644 --- a/src/include/codegen/proxy/runtime_functions_proxy.h +++ b/src/include/codegen/proxy/runtime_functions_proxy.h @@ -14,6 +14,7 @@ #include #include +#include "catalog/schema.h" #include "codegen/proxy/proxy.h" #include "codegen/proxy/type_builder.h" #include "codegen/runtime_functions.h" @@ -34,6 +35,11 @@ PROXY(AbstractExpression) { DECLARE_TYPE; }; +PROXY(Schema) { + DECLARE_MEMBER(0, char[sizeof(catalog::Schema)], opaque); + DECLARE_TYPE; +}; + PROXY(RuntimeFunctions) { DECLARE_METHOD(HashCrc64); DECLARE_METHOD(GetTileGroup); @@ -45,6 +51,7 @@ PROXY(RuntimeFunctions) { TYPE_BUILDER(ColumnLayoutInfo, codegen::RuntimeFunctions::ColumnLayoutInfo); TYPE_BUILDER(AbstractExpression, expression::AbstractExpression); +TYPE_BUILDER(Schema, catalog::Schema); } // namespace codegen } // namespace peloton \ No newline at end of file diff --git a/src/include/codegen/runtime_functions.h b/src/include/codegen/runtime_functions.h index 4a7eafac1eb..c67e5aaae0b 100644 --- a/src/include/codegen/runtime_functions.h +++ b/src/include/codegen/runtime_functions.h @@ -17,6 +17,10 @@ namespace peloton { +namespace catalog { +class Schema; +} + namespace storage { class DataTable; class TileGroup; @@ -66,6 +70,7 @@ class RuntimeFunctions { // Get the column configuration for every column in the tile group static void GetTileGroupLayout(const storage::TileGroup *tile_group, + const catalog::Schema *schema, ColumnLayoutInfo *infos, uint32_t num_cols); static void ThrowDivideByZeroException(); diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h new file mode 100644 index 00000000000..692a0437f6e --- /dev/null +++ b/src/include/storage/layout.h @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tuple.h +// +// Identification: src/include/storage/layout.h +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include + +#include "common/internal_types.h" + +#define ROW_STORE_OID 0 + +namespace peloton { +namespace storage { + +typedef std::map> column_map_type; + +class Layout { + public: + + Layout(const column_map_type& column_map) + : layout_id_(ROW_STORE_OID), + num_columns_(column_map.size()), + tile_group_layout_(column_map) { + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + if (tile_group_layout_[column_id].first != 0) { + layout_id_ = INVALID_OID; + return; + } + } + // Since this is a row store, we don't need the map + tile_group_layout_.clear(); + } + + // Check whether the Layout is a RowStore. + inline bool IsRowStore() const { return (layout_id_ == ROW_STORE_OID); } + + oid_t GetLayoutId() const { return layout_id_; } + + void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, + oid_t &tile_column_offset) const; + + private: + + // Layout Id of the tile + oid_t layout_id_; + + // Number of columns in the layout + oid_t num_columns_; + + // to + column_map_type tile_group_layout_; + +}; + + +} //namespace storage +} //namespace peloton \ No newline at end of file diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 946c65f5f8c..b0c03ab79a5 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -21,6 +21,7 @@ #include "common/item_pointer.h" #include "common/printable.h" #include "planner/project_info.h" +#include "storage/layout.h" #include "type/abstract_pool.h" #include "common/internal_types.h" #include "type/value.h" @@ -53,8 +54,6 @@ class AbstractTable; class TileGroupIterator; class RollbackSegment; -typedef std::map> column_map_type; - /** * Represents a group of tiles logically horizontally contiguous. * @@ -180,11 +179,13 @@ class TileGroup : public Printable { void SetValue(type::Value &value, oid_t tuple_id, oid_t column_id); - double GetSchemaDifference(const storage::column_map_type &new_column_map); + double GetLayoutDifference(const storage::column_map_type &new_column_map); // Sync the contents void Sync(); + const storage::Layout& GetLayout() const { return tile_group_layout_; } + protected: //===--------------------------------------------------------------------===// // Data members @@ -221,6 +222,8 @@ class TileGroup : public Printable { // column to tile mapping : // to column_map_type column_map; + + Layout tile_group_layout_; }; } // namespace storage diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 1f3d9195038..7e54322ef21 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1284,7 +1284,7 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get orig tile group from catalog auto &catalog_manager = catalog::Manager::GetInstance(); auto tile_group = catalog_manager.GetTileGroup(tile_group_id); - auto diff = tile_group->GetSchemaDifference(default_partition_); + auto diff = tile_group->GetLayoutDifference(default_partition_); // Check threshold for transformation if (diff < theta) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp new file mode 100644 index 00000000000..02eaee0bf3b --- /dev/null +++ b/src/storage/layout.cpp @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// tuple.cpp +// +// Identification: src/storage/layout.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "storage/layout.h" + +namespace peloton { +namespace storage { + +// Sets the tile id and column id w.r.t that tile corresponding to +// the specified tile group column id. +void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, + oid_t &tile_column_offset) const { + + // Ensure that the column_offset is not out of bound + PL_ASSERT(num_columns_ > column_offset); + + // For row store layout, tile id is always 0 and the tile + // column_id and tile_group column_id is the same. + if (layout_id_ == ROW_STORE_OID) { + tile_offset = 0; + tile_column_offset = column_offset; + return; + } + + // For other layouts, fetch the layout and + // get the entry in the column map + auto entry = tile_group_layout_.at(column_offset); + tile_offset = entry.first; + tile_column_offset = entry.second; +} + + +} // namespace storage +} // namespace peloton \ No newline at end of file diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 5a40f4ad5fd..ab6db21c02e 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -20,6 +20,7 @@ #include "common/platform.h" #include "common/internal_types.h" #include "storage/abstract_table.h" +#include "storage/layout.h" #include "storage/tile.h" #include "storage/tile_group_header.h" #include "storage/tuple.h" @@ -40,9 +41,9 @@ TileGroup::TileGroup(BackendType backend_type, tile_group_header(tile_group_header), table(table), num_tuple_slots(tuple_count), - column_map(column_map) { + column_map(column_map), + tile_group_layout_(column_map){ tile_count = tile_schemas.size(); - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); oid_t tile_id = manager.GetNextTileId(); @@ -321,20 +322,20 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, oid_t TileGroup::GetTileIdFromColumnId(oid_t column_id) { oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); return tile_offset; } oid_t TileGroup::GetTileColumnId(oid_t column_id) { oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); return tile_column_id; } type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } @@ -342,7 +343,7 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } @@ -353,7 +354,7 @@ std::shared_ptr TileGroup::GetTileReference( return tiles[tile_offset]; } -double TileGroup::GetSchemaDifference( +double TileGroup::GetLayoutDifference( const storage::column_map_type &new_column_map) { double theta = 0; size_t capacity = column_map.size(); From e909f824a9e4cb2b1b389cb06ace0815a5de1b09 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 25 Feb 2018 23:08:52 -0500 Subject: [PATCH 02/42] Modify TileGroup api --- src/include/storage/layout.h | 20 ++++++-------------- src/include/storage/tile_group.h | 4 ---- src/storage/layout.cpp | 30 ++++++++++++++++++++++++++++-- src/storage/tile_group.cpp | 12 ------------ 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 692a0437f6e..49342a989e2 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -26,28 +26,20 @@ typedef std::map> column_map_type; class Layout { public: - Layout(const column_map_type& column_map) - : layout_id_(ROW_STORE_OID), - num_columns_(column_map.size()), - tile_group_layout_(column_map) { - for (oid_t column_id = 0; column_id < num_columns_; column_id++) { - if (tile_group_layout_[column_id].first != 0) { - layout_id_ = INVALID_OID; - return; - } - } - // Since this is a row store, we don't need the map - tile_group_layout_.clear(); - } + Layout(const column_map_type& column_map); // Check whether the Layout is a RowStore. inline bool IsRowStore() const { return (layout_id_ == ROW_STORE_OID); } oid_t GetLayoutId() const { return layout_id_; } - void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, + inline void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const; + oid_t GetTileIdFromColumnId(oid_t column_id); + + oid_t GetTileColumnId(oid_t column_id); + private: // Layout Id of the tile diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index b0c03ab79a5..a771cb59e06 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -171,10 +171,6 @@ class TileGroup : public Printable { tile_column_offset = entry.second; } - oid_t GetTileIdFromColumnId(oid_t column_id); - - oid_t GetTileColumnId(oid_t column_id); - type::Value GetValue(oid_t tuple_id, oid_t column_id); void SetValue(type::Value &value, oid_t tuple_id, oid_t column_id); diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 02eaee0bf3b..b14e30b18c6 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -15,10 +15,25 @@ namespace peloton { namespace storage { +// Constructor for the Layout class +Layout::Layout(const column_map_type& column_map) + : layout_id_(ROW_STORE_OID), + num_columns_(column_map.size()), + tile_group_layout_(column_map) { + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + if (tile_group_layout_[column_id].first != 0) { + layout_id_ = INVALID_OID; + return; + } + } + // Since this is a row store, we don't need the map + tile_group_layout_.clear(); +} + // Sets the tile id and column id w.r.t that tile corresponding to // the specified tile group column id. -void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, - oid_t &tile_column_offset) const { +inline void Layout::LocateTileAndColumn(oid_t column_offset, + oid_t &tile_offset, oid_t &tile_column_offset) const { // Ensure that the column_offset is not out of bound PL_ASSERT(num_columns_ > column_offset); @@ -38,6 +53,17 @@ void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, tile_column_offset = entry.second; } +oid_t Layout::GetTileIdFromColumnId(oid_t column_id) { + oid_t tile_column_id, tile_offset; + LocateTileAndColumn(column_id, tile_offset, tile_column_id); + return tile_offset; +} + +oid_t Layout::GetTileColumnId(oid_t column_id) { + oid_t tile_column_id, tile_offset; + LocateTileAndColumn(column_id, tile_offset, tile_column_id); + return tile_column_id; +} } // namespace storage } // namespace peloton \ No newline at end of file diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index ab6db21c02e..702965901ff 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -320,18 +320,6 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, return tuple_slot_id; } -oid_t TileGroup::GetTileIdFromColumnId(oid_t column_id) { - oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); - return tile_offset; -} - -oid_t TileGroup::GetTileColumnId(oid_t column_id) { - oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); - return tile_column_id; -} - type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; From 48bfd04fcc4abf6872810c03c6e0c9907bd4c4a4 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 25 Feb 2018 23:33:25 -0500 Subject: [PATCH 03/42] Changed Updater & Inserter to use Layout object --- src/codegen/inserter.cpp | 9 +++++---- src/codegen/runtime_functions.cpp | 2 +- src/codegen/updater.cpp | 9 +++++---- src/include/storage/layout.h | 2 +- src/storage/layout.cpp | 2 +- src/storage/tile_group.cpp | 6 ++++-- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index ce652cf746e..b885d92e6e0 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -34,11 +34,12 @@ void Inserter::Init(storage::DataTable *table, char *Inserter::AllocateTupleStorage() { location_ = table_->GetEmptyTupleSlot(nullptr); - // Get the tile offset assuming that it is in a tuple format + // Get the tile offset assuming that it is a row store auto tile_group = table_->GetTileGroupById(location_.block); - oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(0, tile_offset, tile_column_offset); - tile_ = tile_group->GetTileReference(tile_offset); + auto layout = tile_group->GetLayout(); + PL_ASSERT(layout.IsRowStore()); + // layout is still a row store. Hence tile offset it 0 + tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(location_.offset); } diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index 78d9623261b..1f46e03eaca 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -121,7 +121,7 @@ void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { // Map the current column to a tile and a column offset in the tile oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(col_idx, tile_offset, tile_column_offset); + layout.LocateTileAndColumn(col_idx, tile_offset, tile_column_offset); // Now grab the column information auto *tile = tile_group->GetTile(tile_offset); diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 88ef489aed1..020dd4a534d 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -44,10 +44,11 @@ void Updater::Init(storage::DataTable *table, char *Updater::GetDataPtr(uint32_t tile_group_id, uint32_t tuple_offset) { auto tile_group = table_->GetTileGroupById(tile_group_id); - // Get the tile offset assuming that it is still in a tuple format - oid_t tile_offset, tile_column_offset; - tile_group->LocateTileAndColumn(0, tile_offset, tile_column_offset); - tile_ = tile_group->GetTileReference(tile_offset); + // Get the tile offset assuming that it is still a row store + // Hence the Tile offset is 0. + auto layout = tile_group->GetLayout(); + PL_ASSERT(layout.IsRowStore()); + tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(tuple_offset); } diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 49342a989e2..6b35671222e 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -33,7 +33,7 @@ class Layout { oid_t GetLayoutId() const { return layout_id_; } - inline void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, + void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const; oid_t GetTileIdFromColumnId(oid_t column_id); diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index b14e30b18c6..5d4a36fd2ba 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -32,7 +32,7 @@ Layout::Layout(const column_map_type& column_map) // Sets the tile id and column id w.r.t that tile corresponding to // the specified tile group column id. -inline void Layout::LocateTileAndColumn(oid_t column_offset, +void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const { // Ensure that the column_offset is not out of bound diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 702965901ff..47641d5b957 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -323,7 +323,8 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, + tile_offset, tile_column_id); return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } @@ -331,7 +332,8 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, tile_offset, tile_column_id); + tile_group_layout_.LocateTileAndColumn(column_id, + tile_offset, tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } From 2697ea6eeb2407d360a06ad76dc4cd47836a6e54 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 11 Mar 2018 21:46:37 -0400 Subject: [PATCH 04/42] Remove calls from optimizer and executor --- src/executor/logical_tile_factory.cpp | 8 ++++---- src/optimizer/stats/tuple_sampler.cpp | 5 ++--- test/executor/testing_executor_util.cpp | 8 +++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/executor/logical_tile_factory.cpp b/src/executor/logical_tile_factory.cpp index b1278a2b274..b53cf934cd1 100644 --- a/src/executor/logical_tile_factory.cpp +++ b/src/executor/logical_tile_factory.cpp @@ -89,11 +89,11 @@ LogicalTile *LogicalTileFactory::WrapTileGroup( CreateIdentityPositionList(tile_group->GetActiveTupleCount())); // Construct schema. - std::vector &schemas = tile_group->GetTileSchemas(); - PELOTON_ASSERT(schemas.size() == tile_group->NumTiles()); - for (unsigned int i = 0; i < schemas.size(); i++) { + unsigned int num_tiles = tile_group->NumTiles(); + for (unsigned int i = 0; i < num_tiles; i++) { auto base_tile_ref = tile_group->GetTileReference(i); - for (oid_t col_id = 0; col_id < schemas[i].GetColumnCount(); col_id++) { + auto schema = base_tile_ref->GetSchema(); + for (oid_t col_id = 0; col_id < schema->GetColumnCount(); col_id++) { new_tile->AddColumn(base_tile_ref, col_id, position_list_idx); } } diff --git a/src/optimizer/stats/tuple_sampler.cpp b/src/optimizer/stats/tuple_sampler.cpp index 470a5e4392d..9bb91540c9c 100644 --- a/src/optimizer/stats/tuple_sampler.cpp +++ b/src/optimizer/stats/tuple_sampler.cpp @@ -91,15 +91,14 @@ bool TupleSampler::GetTupleInTileGroup(storage::TileGroup *tile_group, } size_t tuple_column_itr = 0; - auto tile_schemas = tile_group->GetTileSchemas(); size_t tile_count = tile_group->GetTileCount(); LOG_TRACE("tile_count: %lu", tile_count); for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - oid_t tile_column_count = schema.GetColumnCount(); storage::Tile *tile = tile_group->GetTile(tile_itr); + const catalog::Schema &schema = *(tile->GetSchema()); + oid_t tile_column_count = schema.GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_offset); storage::Tuple tile_tuple(&schema, tile_tuple_location); diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index b9aff4693e6..be2b381c1f2 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -30,6 +30,7 @@ #include "storage/data_table.h" #include "storage/database.h" #include "storage/table_factory.h" +#include "storage/tile.h" #include "storage/tile_group.h" #include "storage/tile_group_factory.h" #include "storage/tuple.h" @@ -252,8 +253,13 @@ void TestingExecutorUtil::PopulateTable(storage::DataTable *table, int num_rows, */ void TestingExecutorUtil::PopulateTiles( std::shared_ptr tile_group, int num_rows) { + size_t tile_count = tile_group->GetTileCount(); // Create tuple schema from tile schemas. - std::vector &tile_schemas = tile_group->GetTileSchemas(); + std::vector tile_schemas; + for (oid_t tile_id = 0; tile_id < tile_count; tile_id++) { + tile_schemas.push_back(*(tile_group->GetTile(tile_id)->GetSchema())); + } + std::unique_ptr schema( catalog::Schema::AppendSchemaList(tile_schemas)); From 81db08dc67d6e1a1487070fa355c75dd2632edf6 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 26 Mar 2018 13:35:06 -0400 Subject: [PATCH 05/42] Removed all calls to schemas vector from TileGroup --- src/gc/gc_manager.cpp | 18 ++++++++---------- src/include/gc/gc_manager.h | 2 +- src/include/storage/tile_group.h | 5 ----- src/storage/data_table.cpp | 9 +++++---- src/storage/tile_group.cpp | 25 +++++++++++-------------- 5 files changed, 25 insertions(+), 34 deletions(-) diff --git a/src/gc/gc_manager.cpp b/src/gc/gc_manager.cpp index 7db811f197d..070ab6374b9 100644 --- a/src/gc/gc_manager.cpp +++ b/src/gc/gc_manager.cpp @@ -23,8 +23,8 @@ namespace peloton { namespace gc { // Check a tuple and reclaim all varlen field -void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tg, oid_t tuple_id) { - oid_t tile_count = tg->tile_count; +void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id) { + oid_t tile_count = tile_group->tile_count; oid_t tile_col_count; type::TypeId type_id; char *tuple_location; @@ -32,23 +32,21 @@ void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tg, oid_t tuple char *varlen_ptr; for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tg->tile_schemas[tile_itr]; - - tile_col_count = schema.GetColumnCount(); - - storage::Tile *tile = tg->GetTile(tile_itr); + storage::Tile *tile = tile_group->GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_col_count = schema->GetColumnCount(); for (oid_t tile_col_itr = 0; tile_col_itr < tile_col_count; ++tile_col_itr) { - type_id = schema.GetType(tile_col_itr); + type_id = schema->GetType(tile_col_itr); if ((type_id != type::TypeId::VARCHAR && type_id != type::TypeId::VARBINARY) - || (schema.IsInlined(tile_col_itr) == true)) { + || (schema->IsInlined(tile_col_itr) == true)) { // Not of varlen type, or is inlined, skip continue; } // Get the raw varlen pointer tuple_location = tile->GetTupleLocation(tuple_id); - field_location = tuple_location + schema.GetOffset(tile_col_itr); + field_location = tuple_location + schema->GetOffset(tile_col_itr); varlen_ptr = type::Value::GetDataFromStorage(type_id, field_location); // Call the corresponding varlen pool free if (varlen_ptr != nullptr) { diff --git a/src/include/gc/gc_manager.h b/src/include/gc/gc_manager.h index 2697629f3e2..213ab82fa49 100644 --- a/src/include/gc/gc_manager.h +++ b/src/include/gc/gc_manager.h @@ -79,7 +79,7 @@ class GCManager { concurrency::TransactionContext *txn UNUSED_ATTRIBUTE) {} protected: - void CheckAndReclaimVarlenColumns(storage::TileGroup *tg, oid_t tuple_id); + void CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id); protected: volatile bool is_running_; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index a771cb59e06..9fa31c1b155 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -156,8 +156,6 @@ class TileGroup : public Printable { void SetTileGroupId(oid_t tile_group_id_) { tile_group_id = tile_group_id_; } - std::vector &GetTileSchemas() { return tile_schemas; } - size_t GetTileCount() const { return tile_count; } // Sets the tile id and column id w.r.t that tile corresponding to @@ -195,9 +193,6 @@ class TileGroup : public Printable { // Backend type BackendType backend_type; - // mapping to tile schemas - std::vector tile_schemas; - // set of tiles std::vector> tiles; diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 7e54322ef21..7541957c1c6 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1203,7 +1203,6 @@ std::vector TransformTileGroupSchema( // First, get info from the original tile group's schema std::map> schemas; - auto orig_schemas = tile_group->GetTileSchemas(); for (auto column_map_entry : column_map) { new_tile_offset = column_map_entry.second.first; new_tile_column_offset = column_map_entry.second.second; @@ -1212,9 +1211,11 @@ std::vector TransformTileGroupSchema( tile_group->LocateTileAndColumn(column_offset, orig_tile_offset, orig_tile_column_offset); - // Get the column info from original schema - auto orig_schema = orig_schemas[orig_tile_offset]; - auto column_info = orig_schema.GetColumn(orig_tile_column_offset); + // Get the column info from original tile + auto tile = tile_group->GetTile(orig_tile_offset); + PL_ASSERT(tile != nullptr); + auto orig_schema = tile->GetSchema(); + auto column_info = orig_schema->GetColumn(orig_tile_column_offset); schemas[new_tile_offset][new_tile_column_offset] = column_info; } diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 47641d5b957..ca170c8a90b 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -37,20 +37,19 @@ TileGroup::TileGroup(BackendType backend_type, table_id(INVALID_OID), tile_group_id(INVALID_OID), backend_type(backend_type), - tile_schemas(schemas), tile_group_header(tile_group_header), table(table), num_tuple_slots(tuple_count), column_map(column_map), tile_group_layout_(column_map){ - tile_count = tile_schemas.size(); + tile_count = schemas.size(); for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); oid_t tile_id = manager.GetNextTileId(); std::shared_ptr tile(storage::TileFactory::GetTile( backend_type, database_id, table_id, tile_group_id, tile_id, - tile_group_header, tile_schemas[tile_itr], this, tuple_count)); + tile_group_header, schemas[tile_itr], this, tuple_count)); // Add a reference to the tile in the tile group tiles.push_back(tile); @@ -110,16 +109,15 @@ void TileGroup::CopyTuple(const Tuple *tuple, const oid_t &tuple_slot_id) { oid_t column_itr = 0; for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); - storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { @@ -192,16 +190,15 @@ oid_t TileGroup::InsertTupleFromRecovery(cid_t commit_id, oid_t tuple_slot_id, oid_t column_itr = 0; for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); - storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { @@ -292,16 +289,16 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, oid_t column_itr = 0; for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - const catalog::Schema &schema = tile_schemas[tile_itr]; - tile_column_count = schema.GetColumnCount(); storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_column_count = schema->GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_slot_id); PELOTON_ASSERT(tile_tuple_location); // NOTE:: Only a tuple wrapper - storage::Tuple tile_tuple(&schema, tile_tuple_location); + storage::Tuple tile_tuple(schema, tile_tuple_location); for (oid_t tile_column_itr = 0; tile_column_itr < tile_column_count; tile_column_itr++) { From f4525b56396abf4a72880fc3fc32d6a0e11cd88d Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 26 Mar 2018 14:03:04 -0400 Subject: [PATCH 06/42] Removed Calls to LocateTileAndColumn for TileGroup --- src/executor/logical_tile.cpp | 6 ++++-- src/include/storage/layout.h | 2 ++ src/include/storage/tile_group.h | 11 ----------- src/main/sdbench/sdbench_workload.cpp | 5 +++-- src/storage/data_table.cpp | 13 +++++++++---- 5 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/executor/logical_tile.cpp b/src/executor/logical_tile.cpp index 3bdeec9b129..f3857748ecf 100644 --- a/src/executor/logical_tile.cpp +++ b/src/executor/logical_tile.cpp @@ -16,6 +16,7 @@ #include "catalog/schema.h" #include "common/macros.h" #include "storage/data_table.h" +#include "storage/layout.h" #include "storage/tile.h" #include "storage/tile_group.h" #include "type/value.h" @@ -396,11 +397,12 @@ void LogicalTile::AddColumns( const std::shared_ptr &tile_group, const std::vector &column_ids) { const int position_list_idx = 0; + auto tile_group_layout = tile_group->GetLayout(); for (oid_t origin_column_id : column_ids) { oid_t base_tile_offset, tile_column_id; - tile_group->LocateTileAndColumn(origin_column_id, base_tile_offset, - tile_column_id); + tile_group_layout.LocateTileAndColumn(origin_column_id, base_tile_offset, + tile_column_id); AddColumn(tile_group->GetTileReference(base_tile_offset), tile_column_id, position_list_idx); diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 6b35671222e..28bc540931b 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -33,6 +33,8 @@ class Layout { oid_t GetLayoutId() const { return layout_id_; } + // Sets the tile id and column id w.r.t that tile corresponding to + // the specified tile group column id. void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 9fa31c1b155..5911a214a60 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -158,17 +158,6 @@ class TileGroup : public Printable { size_t GetTileCount() const { return tile_count; } - // Sets the tile id and column id w.r.t that tile corresponding to - // the specified tile group column id. - inline void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, - oid_t &tile_column_offset) const { - PELOTON_ASSERT(column_map.count(column_offset) != 0); - // get the entry in the column map - auto entry = column_map.at(column_offset); - tile_offset = entry.first; - tile_column_offset = entry.second; - } - type::Value GetValue(oid_t tuple_id, oid_t column_id); void SetValue(type::Value &value, oid_t tuple_id, oid_t column_id); diff --git a/src/main/sdbench/sdbench_workload.cpp b/src/main/sdbench/sdbench_workload.cpp index 8468d88ffe3..3d50b037e8c 100644 --- a/src/main/sdbench/sdbench_workload.cpp +++ b/src/main/sdbench/sdbench_workload.cpp @@ -561,8 +561,9 @@ static void CopyColumn(oid_t col_itr) { // Begin copy oid_t orig_tile_offset, orig_tile_column_offset; auto orig_tile_group = sdbench_table->GetTileGroup(tile_group_itr); - orig_tile_group->LocateTileAndColumn(col_itr, orig_tile_offset, - orig_tile_column_offset); + auto orig_layout = orig_tile_group->GetLayout(); + orig_layout.LocateTileAndColumn(col_itr, orig_tile_offset, + orig_tile_column_offset); auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); oid_t tuple_count = state.tuples_per_tilegroup; for (oid_t tuple_itr = 0; tuple_itr < tuple_count; tuple_itr++) { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 7541957c1c6..ef8f7d345a9 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -29,6 +29,7 @@ #include "storage/abstract_table.h" #include "storage/data_table.h" #include "storage/database.h" +#include "storage/layout.h" #include "storage/storage_manager.h" #include "storage/tile.h" #include "storage/tile_group.h" @@ -1200,6 +1201,7 @@ std::vector TransformTileGroupSchema( std::vector new_schema; oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; + auto tile_group_layout = tile_group->GetLayout(); // First, get info from the original tile group's schema std::map> schemas; @@ -1208,8 +1210,8 @@ std::vector TransformTileGroupSchema( new_tile_column_offset = column_map_entry.second.second; oid_t column_offset = column_map_entry.first; - tile_group->LocateTileAndColumn(column_offset, orig_tile_offset, - orig_tile_column_offset); + tile_group_layout.LocateTileAndColumn(column_offset, orig_tile_offset, + orig_tile_column_offset); // Get the column info from original tile auto tile = tile_group->GetTile(orig_tile_offset); @@ -1240,6 +1242,9 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, auto orig_column_map = orig_tile_group->GetColumnMap(); PELOTON_ASSERT(new_column_map.size() == orig_column_map.size()); + auto new_layout = new_tile_group->GetLayout(); + auto orig_layout = orig_tile_group->GetLayout(); + oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; @@ -1248,10 +1253,10 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, // Go over each column copying onto the new tile group for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // Locate the original base tile and tile column offset - orig_tile_group->LocateTileAndColumn(column_itr, orig_tile_offset, + orig_layout.LocateTileAndColumn(column_itr, orig_tile_offset, orig_tile_column_offset); - new_tile_group->LocateTileAndColumn(column_itr, new_tile_offset, + new_layout.LocateTileAndColumn(column_itr, new_tile_offset, new_tile_column_offset); auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); From bc3834f2535e0427d6a3930e11b4dd4755014b6e Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 26 Mar 2018 16:22:31 -0400 Subject: [PATCH 07/42] Moved definition of column_map_type. Removed column_map from tile_group --- src/include/common/internal_types.h | 10 +++ src/include/storage/abstract_table.h | 2 - src/include/storage/data_table.h | 4 ++ src/include/storage/layout.h | 20 ++++-- src/include/storage/tile_group.h | 10 --- src/include/tuning/clusterer.h | 5 -- src/storage/data_table.cpp | 16 +++-- src/storage/layout.cpp | 91 +++++++++++++++++++++++++--- src/storage/tile_group.cpp | 21 ------- 9 files changed, 122 insertions(+), 57 deletions(-) diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 17020512944..0f12d14a49b 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -1396,6 +1397,15 @@ typedef std::unordered_set ExprSet; +//===--------------------------------------------------------------------===// +// Storage +//===--------------------------------------------------------------------===// + +/* column_map_type used to store the layout of a tile_group + * to + */ +typedef std::map> column_map_type; + //===--------------------------------------------------------------------===// // Wire protocol typedefs //===--------------------------------------------------------------------===// diff --git a/src/include/storage/abstract_table.h b/src/include/storage/abstract_table.h index 0d41173a9b6..a85676dec01 100644 --- a/src/include/storage/abstract_table.h +++ b/src/include/storage/abstract_table.h @@ -29,8 +29,6 @@ namespace peloton { -typedef std::map> column_map_type; - namespace concurrency { class TransactionContext; } diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index c8348485115..d02cf0deeae 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -23,6 +23,7 @@ #include "common/container/lock_free_array.h" #include "storage/abstract_table.h" #include "storage/indirection_array.h" +#include "storage/layout.h" #include "trigger/trigger.h" //===--------------------------------------------------------------------===// @@ -421,6 +422,9 @@ class DataTable : public AbstractTable { // adapt table bool adapt_table_ = true; + // default layout for the table + Layout default_layout_; + // default partition map for table column_map_type default_partition_; diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 28bc540931b..7f95363d157 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -21,13 +21,15 @@ namespace peloton { namespace storage { -typedef std::map> column_map_type; - class Layout { public: + Layout (const oid_t num_columns); + Layout(const column_map_type& column_map); + Layout(const column_map_type& column_map, oid_t layout_id); + // Check whether the Layout is a RowStore. inline bool IsRowStore() const { return (layout_id_ == ROW_STORE_OID); } @@ -38,9 +40,15 @@ class Layout { void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const; - oid_t GetTileIdFromColumnId(oid_t column_id); + double GetLayoutDifference(const storage::Layout &other) const; + + oid_t GetTileIdFromColumnId(oid_t column_id) const; + + oid_t GetTileColumnId(oid_t column_id) const; + + oid_t GetColumnCount() const { return num_columns_;} - oid_t GetTileColumnId(oid_t column_id); + std::string GetMapInfo() const; private: @@ -50,8 +58,8 @@ class Layout { // Number of columns in the layout oid_t num_columns_; - // to - column_map_type tile_group_layout_; + // Layout of the columns. + column_map_type column_layout_; }; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 5911a214a60..bf764962069 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -142,10 +142,6 @@ class TileGroup : public Printable { peloton::type::AbstractPool *GetTilePool(const oid_t tile_id) const; - const std::map> &GetColumnMap() const { - return column_map; - } - oid_t GetTileGroupId() const; oid_t GetDatabaseId() const { return database_id; } @@ -162,8 +158,6 @@ class TileGroup : public Printable { void SetValue(type::Value &value, oid_t tuple_id, oid_t column_id); - double GetLayoutDifference(const storage::column_map_type &new_column_map); - // Sync the contents void Sync(); @@ -199,10 +193,6 @@ class TileGroup : public Printable { std::mutex tile_group_mutex; - // column to tile mapping : - // to - column_map_type column_map; - Layout tile_group_layout_; }; diff --git a/src/include/tuning/clusterer.h b/src/include/tuning/clusterer.h index 1bdc8d1ee15..85b57696055 100644 --- a/src/include/tuning/clusterer.h +++ b/src/include/tuning/clusterer.h @@ -30,11 +30,6 @@ namespace tuning { // Clusterer //===--------------------------------------------------------------------===// -/** - * Column Id to < Tile Id, Tile Column Id > - */ -typedef std::map> column_map_type; - /** * Sequential k-Means Clustering */ diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index ef8f7d345a9..725c60137f4 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -67,6 +67,7 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, table_name(table_name), tuples_per_tilegroup_(tuples_per_tilegroup), adapt_table_(adapt_table), + default_layout_(schema->GetColumnCount()), trigger_list_(new trigger::TriggerList()) { // Init default partition auto col_count = schema->GetColumnCount(); @@ -1237,18 +1238,21 @@ std::vector TransformTileGroupSchema( // Set the transformed tile group column-at-a-time void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, storage::TileGroup *new_tile_group) { - // Check the schema of the two tile groups - auto new_column_map = new_tile_group->GetColumnMap(); - auto orig_column_map = orig_tile_group->GetColumnMap(); - PELOTON_ASSERT(new_column_map.size() == orig_column_map.size()); auto new_layout = new_tile_group->GetLayout(); auto orig_layout = orig_tile_group->GetLayout(); + // Check that both tile groups have the same schema + // Currently done by checking that the number of columns are equal + // TODO Pooja: Handle schena equality for multiple schema versions. + auto new_column_count = new_layout.GetColumnCount(); + auto orig_column_count = orig_layout.GetColumnCount(); + PL_ASSERT(new_column_count == orig_column_count); + oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; - auto column_count = new_column_map.size(); + auto column_count = new_column_count; auto tuple_count = orig_tile_group->GetAllocatedTupleCount(); // Go over each column copying onto the new tile group for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { @@ -1290,7 +1294,7 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get orig tile group from catalog auto &catalog_manager = catalog::Manager::GetInstance(); auto tile_group = catalog_manager.GetTileGroup(tile_group_id); - auto diff = tile_group->GetLayoutDifference(default_partition_); + auto diff = tile_group->GetLayout().GetLayoutDifference(default_layout_); // Check threshold for transformation if (diff < theta) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 5d4a36fd2ba..1789af8e0c8 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -10,26 +10,41 @@ // //===----------------------------------------------------------------------===// +#include +#include + #include "storage/layout.h" namespace peloton { namespace storage { -// Constructor for the Layout class +// Constructor for the layout class with column_count +Layout::Layout(const oid_t num_columns) + : layout_id_(ROW_STORE_OID), + num_columns_(num_columns) {} + +// Constructor for the Layout class with column_map Layout::Layout(const column_map_type& column_map) : layout_id_(ROW_STORE_OID), num_columns_(column_map.size()), - tile_group_layout_(column_map) { + column_layout_(column_map) { for (oid_t column_id = 0; column_id < num_columns_; column_id++) { - if (tile_group_layout_[column_id].first != 0) { + if (column_layout_[column_id].first != 0) { layout_id_ = INVALID_OID; return; } } // Since this is a row store, we don't need the map - tile_group_layout_.clear(); + column_layout_.clear(); } +// Constructor for Layout class with predefined layout_id +Layout::Layout(const column_map_type &column_map, oid_t layout_id) + : layout_id_(layout_id), + num_columns_(column_map.size()), + column_layout_(column_map) {} + + // Sets the tile id and column id w.r.t that tile corresponding to // the specified tile group column id. void Layout::LocateTileAndColumn(oid_t column_offset, @@ -48,22 +63,84 @@ void Layout::LocateTileAndColumn(oid_t column_offset, // For other layouts, fetch the layout and // get the entry in the column map - auto entry = tile_group_layout_.at(column_offset); + auto entry = column_layout_.at(column_offset); tile_offset = entry.first; tile_column_offset = entry.second; } -oid_t Layout::GetTileIdFromColumnId(oid_t column_id) { +double Layout::GetLayoutDifference(const storage::Layout &other) const { + double theta = 0; + double diff = 0; + + // Calculate theta only for TileGroups with the same schema. + // TODO Pooja: Handle the case where two Layouts with same + // number of columns have a different schema. + PL_ASSERT(this->num_columns_ == other.num_columns_); + + if ((this->layout_id_ != other.layout_id_)) { + + for (oid_t col_itr = 0; col_itr < num_columns_; col_itr++) { + + if (this->GetTileIdFromColumnId(col_itr) != + other.GetTileIdFromColumnId(col_itr)) { + diff++; + } + } + // compute theta + theta = diff / num_columns_; + } + + return theta; +} + +oid_t Layout::GetTileIdFromColumnId(oid_t column_id) const { oid_t tile_column_id, tile_offset; LocateTileAndColumn(column_id, tile_offset, tile_column_id); return tile_offset; } -oid_t Layout::GetTileColumnId(oid_t column_id) { +oid_t Layout::GetTileColumnId(oid_t column_id) const { oid_t tile_column_id, tile_offset; LocateTileAndColumn(column_id, tile_offset, tile_column_id); return tile_column_id; } +std::string Layout::GetMapInfo() const { + std::stringstream ss; + std::map> tile_column_map; + + if (layout_id_ == ROW_STORE_OID) { + // Row store always contains only 1 tile. The tile_id is always 0. + oid_t tile_id = 0; + tile_column_map[tile_id] = {}; + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + tile_column_map[tile_id].push_back(col_id); + } + } else { + for (auto column_info : column_layout_) { + oid_t col_id = column_info.first; + oid_t tile_id = column_info.second.first; + + if(tile_column_map.find(tile_id) == tile_column_map.end()) { + tile_column_map[tile_id] = {}; + } + tile_column_map[tile_id].push_back(col_id); + } + } + + // Construct a string from tile_col_map + for (auto tile_info : tile_column_map) { + ss << tile_info.first << ": "; + for (auto col_id : tile_info.second) { + ss << col_id << " "; + } + ss << " :: "; + + } + + return ss.str(); +} + + } // namespace storage } // namespace peloton \ No newline at end of file diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index ca170c8a90b..80351c6a09b 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -40,7 +40,6 @@ TileGroup::TileGroup(BackendType backend_type, tile_group_header(tile_group_header), table(table), num_tuple_slots(tuple_count), - column_map(column_map), tile_group_layout_(column_map){ tile_count = schemas.size(); for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { @@ -341,26 +340,6 @@ std::shared_ptr TileGroup::GetTileReference( return tiles[tile_offset]; } -double TileGroup::GetLayoutDifference( - const storage::column_map_type &new_column_map) { - double theta = 0; - size_t capacity = column_map.size(); - double diff = 0; - - for (oid_t col_itr = 0; col_itr < capacity; col_itr++) { - auto &old_col = column_map.at(col_itr); - auto &new_col = new_column_map.at(col_itr); - - // The tile don't match - if (old_col.first != new_col.first) diff++; - } - - // compute diff - theta = diff / capacity; - - return theta; -} - void TileGroup::Sync() { // Sync the tile group data by syncing all the underlying tiles for (auto tile : tiles) { From 0d7dfdea6640adc87c44e9dadb95122ae9357b28 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 26 Mar 2018 21:08:52 -0400 Subject: [PATCH 08/42] Added GetInfo function --- src/include/storage/layout.h | 9 ++++++--- src/storage/layout.cpp | 18 +++++++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 7f95363d157..b9a9ee8c98a 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -15,13 +15,14 @@ #include #include "common/internal_types.h" +#include "common/printable.h" #define ROW_STORE_OID 0 namespace peloton { namespace storage { -class Layout { +class Layout : public Printable { public: Layout (const oid_t num_columns); @@ -48,7 +49,10 @@ class Layout { oid_t GetColumnCount() const { return num_columns_;} - std::string GetMapInfo() const; + std::string GetColumnMapInfo() const; + + // Get a string representation for debugging + const std::string GetInfo() const; private: @@ -63,6 +67,5 @@ class Layout { }; - } //namespace storage } //namespace peloton \ No newline at end of file diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 1789af8e0c8..a7cec16d5f5 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -14,6 +14,7 @@ #include #include "storage/layout.h" +#include "util/stringbox_util.h" namespace peloton { namespace storage { @@ -105,7 +106,7 @@ oid_t Layout::GetTileColumnId(oid_t column_id) const { return tile_column_id; } -std::string Layout::GetMapInfo() const { +std::string Layout::GetColumnMapInfo() const { std::stringstream ss; std::map> tile_column_map; @@ -120,7 +121,6 @@ std::string Layout::GetMapInfo() const { for (auto column_info : column_layout_) { oid_t col_id = column_info.first; oid_t tile_id = column_info.second.first; - if(tile_column_map.find(tile_id) == tile_column_map.end()) { tile_column_map[tile_id] = {}; } @@ -135,12 +135,24 @@ std::string Layout::GetMapInfo() const { ss << col_id << " "; } ss << " :: "; - } return ss.str(); } +const std::string Layout::GetInfo() const { + std::ostringstream os; + + os << peloton::GETINFO_DOUBLE_STAR << " Layout[#" << layout_id_ << "] " + << peloton::GETINFO_DOUBLE_STAR << std::endl; + os << "Number of columns[" << num_columns_ << "] " << std::endl; + + os << "ColumnMap[ " << GetColumnMapInfo() << "] " << std::endl; + + return peloton::StringUtil::Prefix(peloton::StringBoxUtil::Box(os.str()), + GETINFO_SPACER); +} + } // namespace storage } // namespace peloton \ No newline at end of file From 36cdad0b833f3cf5f5dc1b91ad29243f999167ec Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Fri, 30 Mar 2018 18:47:03 -0400 Subject: [PATCH 09/42] make check pases. Need to change a few more function calls --- src/catalog/schema.cpp | 18 +++++++++--------- src/include/catalog/schema.h | 12 ++++++------ test/executor/logical_tile_test.cpp | 18 ++++++++++-------- test/executor/testing_executor_util.cpp | 6 +++--- test/storage/data_table_test.cpp | 2 +- test/storage/tile_group_test.cpp | 2 +- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index e3e088e607a..df71b1315a9 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -205,23 +205,23 @@ Schema *Schema::FilterSchema(const Schema *schema, } // Append two schema objects -Schema *Schema::AppendSchema(Schema *first, Schema *second) { +Schema *Schema::AppendSchema(const Schema *first, const Schema *second) { return AppendSchemaPtrList({first, second}); } // Append subset of columns in the two given schemas -Schema *Schema::AppendSchema(Schema *first, std::vector &first_set, - Schema *second, std::vector &second_set) { - const std::vector schema_list({first, second}); +Schema *Schema::AppendSchema(const Schema *first, std::vector &first_set, + const Schema *second, std::vector &second_set) { + const std::vector schema_list({first, second}); const std::vector> subsets({first_set, second_set}); return AppendSchemaPtrList(schema_list, subsets); } // Append given schemas. -Schema *Schema::AppendSchemaList(std::vector &schema_list) { +Schema *Schema::AppendSchemaList(std::vector &schema_list) { // All we do here is convert vector to vector. // This is a convenience function. - std::vector schema_ptr_list; + std::vector schema_ptr_list; for (unsigned int i = 0; i < schema_list.size(); i++) { schema_ptr_list.push_back(&schema_list[i]); } @@ -229,7 +229,7 @@ Schema *Schema::AppendSchemaList(std::vector &schema_list) { } // Append given schemas. -Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { +Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { std::vector> subsets; for (unsigned int i = 0; i < schema_list.size(); i++) { @@ -246,13 +246,13 @@ Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { // Append subsets of columns in the given schemas. Schema *Schema::AppendSchemaPtrList( - const std::vector &schema_list, + const std::vector &schema_list, const std::vector> &subsets) { PELOTON_ASSERT(schema_list.size() == subsets.size()); std::vector columns; for (unsigned int i = 0; i < schema_list.size(); i++) { - Schema *schema = schema_list[i]; + const Schema *schema = schema_list[i]; const std::vector &subset = subsets[i]; unsigned int column_count = schema->GetColumnCount(); diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index 8a2feec5dcd..97de588d62c 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -62,21 +62,21 @@ class Schema : public Printable { const std::vector &set); // Append two schema objects - static Schema *AppendSchema(Schema *first, Schema *second); + static Schema *AppendSchema(const Schema *first, const Schema *second); // Append subset of columns in the two given schemas - static Schema *AppendSchema(Schema *first, std::vector &first_set, - Schema *second, std::vector &second_set); + static Schema *AppendSchema(const Schema *first, std::vector &first_set, + const Schema *second, std::vector &second_set); // Append given schemas. - static Schema *AppendSchemaList(std::vector &schema_list); + static Schema *AppendSchemaList(std::vector &schema_list); // Append given schemas. - static Schema *AppendSchemaPtrList(const std::vector &schema_list); + static Schema *AppendSchemaPtrList(const std::vector &schema_list); // Append subsets of columns in the given schemas. static Schema *AppendSchemaPtrList( - const std::vector &schema_list, + const std::vector &schema_list, const std::vector> &subsets); // Compare two schemas diff --git a/test/executor/logical_tile_test.cpp b/test/executor/logical_tile_test.cpp index baf3d342262..ee2e47d26fe 100644 --- a/test/executor/logical_tile_test.cpp +++ b/test/executor/logical_tile_test.cpp @@ -98,10 +98,15 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { std::shared_ptr tile_group( TestingExecutorUtil::CreateTileGroup(tuple_count)); + std::vector tile_schemas; + for (unsigned int i = 0; i < tile_group->NumTiles(); i++) { + tile_schemas.push_back(tile_group->GetTile(i)->GetSchema()); + } + + // Create tuple schema from tile schemas. - std::vector &tile_schemas = tile_group->GetTileSchemas(); std::unique_ptr schema( - catalog::Schema::AppendSchemaList(tile_schemas)); + catalog::Schema::AppendSchemaPtrList(tile_schemas)); // Create tuples and insert them into tile group. const bool allocate = true; @@ -156,10 +161,7 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { logical_tile->AddPositionList(std::move(position_list1)); logical_tile->AddPositionList(std::move(position_list2)); - PELOTON_ASSERT(tile_schemas.size() == 2); - catalog::Schema *schema1 = &tile_schemas[0]; - catalog::Schema *schema2 = &tile_schemas[1]; - oid_t column_count = schema2->GetColumnCount(); + oid_t column_count = tile_schemas[1]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { logical_tile->AddColumn(base_tile_ref, column_itr, column_itr); } @@ -185,12 +187,12 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { logical_tile->AddPositionList(std::move(position_list3)); logical_tile->AddPositionList(std::move(position_list4)); - oid_t column_count1 = schema1->GetColumnCount(); + oid_t column_count1 = tile_schemas[0]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count1; column_itr++) { logical_tile->AddColumn(base_tile_ref1, column_itr, column_itr); } - oid_t column_count2 = schema2->GetColumnCount(); + oid_t column_count2 = tile_schemas[1]->GetColumnCount(); for (oid_t column_itr = 0; column_itr < column_count2; column_itr++) { logical_tile->AddColumn(base_tile_ref2, column_itr, column_count1 + column_itr); diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index be2b381c1f2..0021f90d6fd 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -255,13 +255,13 @@ void TestingExecutorUtil::PopulateTiles( std::shared_ptr tile_group, int num_rows) { size_t tile_count = tile_group->GetTileCount(); // Create tuple schema from tile schemas. - std::vector tile_schemas; + std::vector tile_schemas; for (oid_t tile_id = 0; tile_id < tile_count; tile_id++) { - tile_schemas.push_back(*(tile_group->GetTile(tile_id)->GetSchema())); + tile_schemas.push_back(tile_group->GetTile(tile_id)->GetSchema()); } std::unique_ptr schema( - catalog::Schema::AppendSchemaList(tile_schemas)); + catalog::Schema::AppendSchemaPtrList(tile_schemas)); // Ensure that the tile group is as expected. PELOTON_ASSERT(schema->GetColumnCount() == 4); diff --git a/test/storage/data_table_test.cpp b/test/storage/data_table_test.cpp index e85337fecda..d64ec8bcb90 100644 --- a/test/storage/data_table_test.cpp +++ b/test/storage/data_table_test.cpp @@ -42,7 +42,7 @@ TEST_F(DataTableTests, TransformTileGroupTest) { txn_manager.CommitTransaction(txn); // Create the new column map - storage::column_map_type column_map; + column_map_type column_map; column_map[0] = std::make_pair(0, 0); column_map[1] = std::make_pair(0, 1); column_map[2] = std::make_pair(1, 0); diff --git a/test/storage/tile_group_test.cpp b/test/storage/tile_group_test.cpp index 5411d1be2de..d42fe4e3f8e 100644 --- a/test/storage/tile_group_test.cpp +++ b/test/storage/tile_group_test.cpp @@ -371,7 +371,7 @@ TEST_F(TileGroupTests, TileCopyTest) { column_names.push_back(tile_column_names); const int tuple_count = 4; - storage::column_map_type column_map; + column_map_type column_map; // default column map auto col_count = schema->GetColumnCount(); From 57145723dd1068bba19f172d3a4b0bd31d02428c Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 4 Apr 2018 22:46:47 -0400 Subject: [PATCH 10/42] Convert PL_ASSERT -> PELOTON_ASSERT --- src/codegen/inserter.cpp | 2 +- src/codegen/updater.cpp | 2 +- src/storage/data_table.cpp | 4 ++-- src/storage/layout.cpp | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/codegen/inserter.cpp b/src/codegen/inserter.cpp index b885d92e6e0..abe00ccbb07 100644 --- a/src/codegen/inserter.cpp +++ b/src/codegen/inserter.cpp @@ -37,7 +37,7 @@ char *Inserter::AllocateTupleStorage() { // Get the tile offset assuming that it is a row store auto tile_group = table_->GetTileGroupById(location_.block); auto layout = tile_group->GetLayout(); - PL_ASSERT(layout.IsRowStore()); + PELOTON_ASSERT(layout.IsRowStore()); // layout is still a row store. Hence tile offset it 0 tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(location_.offset); diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 020dd4a534d..5e595bf922f 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -47,7 +47,7 @@ char *Updater::GetDataPtr(uint32_t tile_group_id, uint32_t tuple_offset) { // Get the tile offset assuming that it is still a row store // Hence the Tile offset is 0. auto layout = tile_group->GetLayout(); - PL_ASSERT(layout.IsRowStore()); + PELOTON_ASSERT(layout.IsRowStore()); tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(tuple_offset); } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 725c60137f4..cbc022b3139 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1216,7 +1216,7 @@ std::vector TransformTileGroupSchema( // Get the column info from original tile auto tile = tile_group->GetTile(orig_tile_offset); - PL_ASSERT(tile != nullptr); + PELOTON_ASSERT(tile != nullptr); auto orig_schema = tile->GetSchema(); auto column_info = orig_schema->GetColumn(orig_tile_column_offset); schemas[new_tile_offset][new_tile_column_offset] = column_info; @@ -1247,7 +1247,7 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, // TODO Pooja: Handle schena equality for multiple schema versions. auto new_column_count = new_layout.GetColumnCount(); auto orig_column_count = orig_layout.GetColumnCount(); - PL_ASSERT(new_column_count == orig_column_count); + PELOTON_ASSERT(new_column_count == orig_column_count); oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index a7cec16d5f5..6a0ae288035 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -52,7 +52,7 @@ void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const { // Ensure that the column_offset is not out of bound - PL_ASSERT(num_columns_ > column_offset); + PELOTON_ASSERT(num_columns_ > column_offset); // For row store layout, tile id is always 0 and the tile // column_id and tile_group column_id is the same. @@ -76,7 +76,7 @@ double Layout::GetLayoutDifference(const storage::Layout &other) const { // Calculate theta only for TileGroups with the same schema. // TODO Pooja: Handle the case where two Layouts with same // number of columns have a different schema. - PL_ASSERT(this->num_columns_ == other.num_columns_); + PELOTON_ASSERT(this->num_columns_ == other.num_columns_); if ((this->layout_id_ != other.layout_id_)) { From f9130af559e2ab48e4cd46ed8cfbd8270f03663c Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 9 Apr 2018 12:18:23 -0400 Subject: [PATCH 11/42] Modified DataTable and LayoutTunerTest --- src/include/storage/data_table.h | 2 +- src/include/storage/layout.h | 3 +++ src/storage/data_table.cpp | 5 +++-- src/storage/layout.cpp | 20 ++++++++++++++++++++ test/tuning/layout_tuner_test.cpp | 23 +++++++++-------------- 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index d02cf0deeae..e3499501c37 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -250,7 +250,7 @@ class DataTable : public AbstractTable { void SetDefaultLayout(const column_map_type &layout); - column_map_type GetDefaultLayout() const; + Layout GetDefaultLayout() const; //===--------------------------------------------------------------------===// // INDEX TUNER diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index b9a9ee8c98a..e3f1f0f2a1f 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -54,6 +54,9 @@ class Layout : public Printable { // Get a string representation for debugging const std::string GetInfo() const; + friend bool operator==(const Layout& lhs, const Layout& rhs); + friend bool operator!=(const Layout& lhs, const Layout& rhs); + private: // Layout Id of the tile diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index cbc022b3139..98bd656321e 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1388,11 +1388,12 @@ std::map DataTable::GetColumnMapStats() { } void DataTable::SetDefaultLayout(const column_map_type &layout) { + default_layout_ = Layout(layout); default_partition_ = layout; } -column_map_type DataTable::GetDefaultLayout() const { - return default_partition_; +Layout DataTable::GetDefaultLayout() const { + return default_layout_; } void DataTable::AddTrigger(trigger::Trigger new_trigger) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 6a0ae288035..083e1e5029b 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -20,6 +20,7 @@ namespace peloton { namespace storage { // Constructor for the layout class with column_count +// The default layout is always a ROW_STORE Layout::Layout(const oid_t num_columns) : layout_id_(ROW_STORE_OID), num_columns_(num_columns) {} @@ -153,6 +154,25 @@ const std::string Layout::GetInfo() const { GETINFO_SPACER); } +bool operator==(const Layout& lhs, const Layout& rhs) { + + // Check the equality of layout_oid_ + if (lhs.GetLayoutId() != rhs.GetLayoutId()) { + return false; + } + // Check the equality of column_count_ + if (lhs.GetColumnCount() != rhs.GetColumnCount()) { + return false; + } + + // Check for the equality of the column_layout_ + return (lhs.column_layout_ == rhs.column_layout_); +} + +bool operator!=(const Layout& lhs, const Layout& rhs) { + return !(lhs == rhs); +} + } // namespace storage } // namespace peloton \ No newline at end of file diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index 108bfe41ae6..b550aa52957 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -23,7 +23,7 @@ #include "concurrency/transaction_manager_factory.h" #include "storage/data_table.h" -#include "storage/tile_group.h" +#include "storage/layout.h" namespace peloton { namespace test { @@ -60,8 +60,7 @@ TEST_F(LayoutTunerTests, BasicTest) { // Check old default tile group layout auto old_default_layout = data_table->GetDefaultLayout(); - LOG_INFO("Layout: %s", - layout_tuner.GetColumnMapInfo(old_default_layout).c_str()); + LOG_INFO("Layout: %s",old_default_layout.GetColumnMapInfo().c_str()); // Start layout tuner layout_tuner.Start(); @@ -107,25 +106,21 @@ TEST_F(LayoutTunerTests, BasicTest) { // Check new default tile group layout auto new_default_layout = data_table->GetDefaultLayout(); - LOG_INFO("Layout: %s", - layout_tuner.GetColumnMapInfo(new_default_layout).c_str()); + LOG_INFO("Layout: %s", new_default_layout.GetColumnMapInfo().c_str()); // Ensure that the layout has been changed EXPECT_NE(new_default_layout, old_default_layout); // Check the new default table layout - column_count = new_default_layout.size(); + column_count = new_default_layout.GetColumnCount(); EXPECT_EQ(column_count, 4); - auto first_column_tile = new_default_layout[0].first; - auto second_column_tile = new_default_layout[1].first; - auto third_column_tile = new_default_layout[2].first; - auto fourth_column_tile = new_default_layout[3].first; - EXPECT_EQ(first_column_tile, 0); - EXPECT_EQ(second_column_tile, 0); - EXPECT_EQ(third_column_tile, 0); - EXPECT_EQ(fourth_column_tile, 1); + // Check the tile corresponding to each column. + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(0),0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(1),0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2),0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3),1); } } // namespace test From 422cf95af1cc2e9258418e8f866b7e02a2a878b6 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 16 Apr 2018 17:56:17 -0400 Subject: [PATCH 12/42] Added Layout Test for codegen --- src/catalog/catalog.cpp | 5 +- src/include/catalog/catalog.h | 3 +- test/codegen/table_scan_translator_test.cpp | 135 +++++++++++++++++++- 3 files changed, 139 insertions(+), 4 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 90b9d13a62a..f056de0b0d3 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -295,7 +295,8 @@ ResultType Catalog::CreateTable(const std::string &database_name, const std::string &table_name, std::unique_ptr schema, concurrency::TransactionContext *txn, - bool is_catalog, oid_t tuples_per_tilegroup) { + bool is_catalog, oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { if (txn == nullptr) throw CatalogException("Do not have transaction to create table " + table_name); @@ -348,7 +349,7 @@ ResultType Catalog::CreateTable(const std::string &database_name, bool adapt_table = false; auto table = storage::TableFactory::GetDataTable( database_object->GetDatabaseOid(), table_oid, schema.release(), - table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog); + table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog, layout_type); database->AddTable(table, is_catalog); // put data table object into rw_object_set txn->RecordCreate(database_object->GetDatabaseOid(), table_oid, INVALID_OID); diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index cd19d7fe72b..90905e7066e 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -99,7 +99,8 @@ class Catalog { const std::string &database_name, const std::string &schema_name, const std::string &table_name, std::unique_ptr, concurrency::TransactionContext *txn, bool is_catalog = false, - oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + peloton::LayoutType layout_type = LayoutType::ROW); // Create index for a table ResultType CreateIndex(const std::string &database_name, diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 881e24a6996..66404dd6e58 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -18,6 +18,7 @@ #include "expression/operator_expression.h" #include "planner/seq_scan_plan.h" #include "storage/storage_manager.h" +#include "storage/table_factory.h" #include "codegen/testing_codegen_util.h" @@ -27,7 +28,7 @@ namespace test { class TableScanTranslatorTest : public PelotonCodeGenTest { std::string all_cols_table_name = "crazy_table"; - public: +public: TableScanTranslatorTest() : PelotonCodeGenTest(), num_rows_to_insert(64) { // Load test table LoadTestTable(TestTableId(), num_rows_to_insert); @@ -35,6 +36,122 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { CreateAndLoadAllColsTable(); } + void ExecuteTileGroupTest(peloton::LayoutType layout_type) { + + const int tuples_per_tilegroup= 100; + const int tile_group_count = 5; + const int tuple_count = tuples_per_tilegroup * tile_group_count; + const oid_t col_count = 128; + const bool is_inlined = true; + + + ///////////////////////////////////////////////////////// + // Define the schema. + ///////////////////////////////////////////////////////// + + std::vector columns; + + for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { + auto column = + catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); + + columns.push_back(column); + } + + std::unique_ptr table_schema = std::unique_ptr(new catalog::Schema(columns)); + std::string table_name("TEST_TABLE"); + + ///////////////////////////////////////////////////////// + // Create table. + ///////////////////////////////////////////////////////// + + // bool own_schema = true; + // bool adapt_table = true; + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + const bool allocate = true; + auto txn = txn_manager.BeginTransaction(); + + // Insert table in catalog + catalog->CreateTable(test_db_name, table_name, std::move(table_schema), + txn, is_catalog, tuples_per_tilegroup, layout_type); + txn_manager.EndTransaction(txn); + + auto table = GetDatabase().GetTableWithName(table_name); + + ///////////////////////////////////////////////////////// + // Load in the data + ///////////////////////////////////////////////////////// + + // Insert tuples into tile_group. + + txn = txn_manager.BeginTransaction(); + auto table_schema_ptr = table->GetSchema(); + auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); + + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id <= col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Do a seq scan with predicate on top of the table + ///////////////////////////////////////////////////////// + + // Column ids to be scanned. + std::vector column_ids; + for(oid_t col_id = 0 ; col_id < col_count; col_id++) { + column_ids.push_back(col_id); + } + + + // Setup the scan plan node + planner::SeqScanPlan scan(table, nullptr, column_ids); + + // Do binding + planner::BindingContext context; + scan.PerformBinding(context); + + // Printing consumer + codegen::BufferingConsumer buffer{column_ids, context}; + + + // COMPILE and execute + CompileAndExecute(scan, buffer); + + // Check that we got all the results + const auto &results = buffer.GetOutputTuples(); + EXPECT_EQ(results.size(), tuple_count); + + for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { + auto &tuple = results[tuple_id]; + int tuple_id_value = tuple_id; + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); + EXPECT_EQ(CmpBool::CmpTrue, tuple.GetValue(col_id).CompareEquals(value)); + } + } + } + uint32_t NumRowsInTestTable() const { return num_rows_to_insert; } oid_t TestTableId() { return test_table_oids[0]; } @@ -584,5 +701,21 @@ TEST_F(TableScanTranslatorTest, ScanWithModuloPredicate) { type::ValueFactory::GetIntegerValue(1))); } +TEST_F(TableScanTranslatorTest, ScanRowLayout) { + // + // Creates a table with LayoutType::ROW and + // invokes the TableScanTranslator + // + ExecuteTileGroupTest(LayoutType::ROW); +} + +TEST_F(TableScanTranslatorTest, ScanColumnLayout) { + // + // Creates a table with LayoutType::COLUMN and + // invokes the TableScanTranslator + // + ExecuteTileGroupTest(LayoutType::COLUMN); +} + } // namespace test } // namespace peloton From 0500c9eb5d04f659d8e29f367c61997b6cc990da Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 16 Apr 2018 18:27:15 -0400 Subject: [PATCH 13/42] Get the tests to actually use different layouts --- src/codegen/runtime_functions.cpp | 2 +- src/include/storage/layout.h | 10 ++++- src/storage/data_table.cpp | 13 ++++-- src/storage/layout.cpp | 71 +++++++++++++++++++++++++------ 4 files changed, 78 insertions(+), 18 deletions(-) diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index 1f46e03eaca..d26da38b88a 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -106,7 +106,7 @@ void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, ColumnLayoutInfo *infos, uint32_t num_cols) { const auto& layout = tile_group->GetLayout(); - // For ROW_STORE_OID, the tile group contains a single tile + // For LayoutType::ROW, the tile group contains a single tile // and all the columns are in the same order as the table schema. if (layout.IsRowStore()) { auto tuple_location = tile_group->GetTile(0)->GetTupleLocation(0); diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index e3f1f0f2a1f..eb89866fd28 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -18,6 +18,7 @@ #include "common/printable.h" #define ROW_STORE_OID 0 +#define COLUMN_STORE_OID 1 namespace peloton { namespace storage { @@ -25,14 +26,17 @@ namespace storage { class Layout : public Printable { public: - Layout (const oid_t num_columns); + Layout (const oid_t num_columns, LayoutType layout_type = LayoutType::ROW); Layout(const column_map_type& column_map); Layout(const column_map_type& column_map, oid_t layout_id); // Check whether the Layout is a RowStore. - inline bool IsRowStore() const { return (layout_id_ == ROW_STORE_OID); } + inline bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } + + // Check whether the Layout is ColumnStore. + inline bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } oid_t GetLayoutId() const { return layout_id_; } @@ -68,6 +72,8 @@ class Layout : public Printable { // Layout of the columns. column_map_type column_layout_; + LayoutType layout_type_; + }; } //namespace storage diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 98bd656321e..2ad08b3a22f 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -70,9 +70,16 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, default_layout_(schema->GetColumnCount()), trigger_list_(new trigger::TriggerList()) { // Init default partition - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - default_partition_[col_itr] = std::make_pair(0, col_itr); + if (layout_type == LayoutType::ROW) { + auto col_count = schema->GetColumnCount(); + for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { + default_partition_[col_itr] = std::make_pair(0, col_itr); + } + } else if (layout_type == LayoutType::COLUMN) { + auto col_count = schema->GetColumnCount(); + for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { + default_partition_[col_itr] = std::make_pair(col_itr, 0); + } } if (is_catalog == true) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 083e1e5029b..483dcd5ecad 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -21,30 +21,65 @@ namespace storage { // Constructor for the layout class with column_count // The default layout is always a ROW_STORE -Layout::Layout(const oid_t num_columns) - : layout_id_(ROW_STORE_OID), - num_columns_(num_columns) {} +Layout::Layout(const oid_t num_columns, LayoutType layout_type) + : num_columns_(num_columns), + layout_type_(layout_type){} // Constructor for the Layout class with column_map Layout::Layout(const column_map_type& column_map) : layout_id_(ROW_STORE_OID), num_columns_(column_map.size()), column_layout_(column_map) { + // Figure out the layout type if not present + bool row_layout = true, column_layout = true; for (oid_t column_id = 0; column_id < num_columns_; column_id++) { if (column_layout_[column_id].first != 0) { - layout_id_ = INVALID_OID; - return; + row_layout = false; + } + if (column_layout_[column_id].first != column_id) { + column_layout = false; } } - // Since this is a row store, we don't need the map - column_layout_.clear(); + + // Assign the layout_type_ + // If a table has one column, it would be LayoutType::Row + if (row_layout) { + layout_type_ = LayoutType::ROW; + layout_id_ = ROW_STORE_OID; + } else if (column_layout) { + layout_type_ = LayoutType::COLUMN; + layout_id_ = COLUMN_STORE_OID; + } else { + // TODO Pooja - Handle this by making an entry into the catalog + layout_type_ = LayoutType::HYBRID; + layout_id_ = INVALID_OID; + } + + if (layout_type_ != LayoutType::HYBRID) { + // Since this is a predefined layout, we don't need the map + column_layout_.clear(); + } } // Constructor for Layout class with predefined layout_id Layout::Layout(const column_map_type &column_map, oid_t layout_id) : layout_id_(layout_id), num_columns_(column_map.size()), - column_layout_(column_map) {} + column_layout_(column_map) { + + if (layout_id_ == ROW_STORE_OID) { + layout_type_ = LayoutType::ROW; + } else if (layout_id == COLUMN_STORE_OID) { + layout_type_ = LayoutType::COLUMN; + } else { + layout_type_ = LayoutType::HYBRID; + } + + if (layout_type_ != LayoutType::HYBRID) { + // Since this is a predefined layout, we don't need the map + column_layout_.clear(); + } +} // Sets the tile id and column id w.r.t that tile corresponding to @@ -56,13 +91,21 @@ void Layout::LocateTileAndColumn(oid_t column_offset, PELOTON_ASSERT(num_columns_ > column_offset); // For row store layout, tile id is always 0 and the tile - // column_id and tile_group column_id is the same. - if (layout_id_ == ROW_STORE_OID) { + // column_id and tile column_id is the same. + if (layout_type_ == LayoutType::ROW) { tile_offset = 0; tile_column_offset = column_offset; return; } + // For column store layout, tile_id is always same as column_id + // and the tile column_id is always 0. + if (layout_type_ == LayoutType::COLUMN) { + tile_offset = column_offset; + tile_column_offset = 0; + return; + } + // For other layouts, fetch the layout and // get the entry in the column map auto entry = column_layout_.at(column_offset); @@ -75,8 +118,7 @@ double Layout::GetLayoutDifference(const storage::Layout &other) const { double diff = 0; // Calculate theta only for TileGroups with the same schema. - // TODO Pooja: Handle the case where two Layouts with same - // number of columns have a different schema. + // Check the schema before invoking this function. PELOTON_ASSERT(this->num_columns_ == other.num_columns_); if ((this->layout_id_ != other.layout_id_)) { @@ -156,6 +198,11 @@ const std::string Layout::GetInfo() const { bool operator==(const Layout& lhs, const Layout& rhs) { + // Check the equality of layout_type_ + if (lhs.layout_type_ != rhs.layout_type_) { + return false; + } + // Check the equality of layout_oid_ if (lhs.GetLayoutId() != rhs.GetLayoutId()) { return false; From cdb9b47c66152efbf9617c9ffc72372ed7b8bcd9 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 08:56:14 -0400 Subject: [PATCH 14/42] Modified storage layer, builds successfully. Need to fix tests --- src/include/storage/abstract_table.h | 16 +++---- src/include/storage/data_table.h | 9 ++-- src/include/storage/layout.h | 7 +++ src/include/storage/tile_group.h | 7 +-- src/include/storage/tile_group_factory.h | 2 +- src/storage/abstract_table.cpp | 60 +++++------------------- src/storage/data_table.cpp | 39 +++++++-------- src/storage/layout.cpp | 25 ++++++++++ src/storage/temp_table.cpp | 8 +--- src/storage/tile_group.cpp | 8 ++-- src/storage/tile_group_factory.cpp | 4 +- 11 files changed, 84 insertions(+), 101 deletions(-) diff --git a/src/include/storage/abstract_table.h b/src/include/storage/abstract_table.h index a85676dec01..b13a08d64df 100644 --- a/src/include/storage/abstract_table.h +++ b/src/include/storage/abstract_table.h @@ -22,6 +22,7 @@ #include "common/item_pointer.h" #include "common/printable.h" #include "common/internal_types.h" +#include "storage/layout.h" //===--------------------------------------------------------------------===// // GUC Variables @@ -75,12 +76,12 @@ class AbstractTable : public Printable { // LAYOUT TYPE //===--------------------------------------------------------------------===// - void SetLayoutType(peloton::LayoutType layout) { - layout_type = layout; + void SetDefaultLayout(std::shared_ptr layout) { + default_layout_ = layout; } - peloton::LayoutType GetLayoutType() { - return layout_type; + std::shared_ptr GetDefaultLayout() { + return default_layout_; } //===--------------------------------------------------------------------===// // TILE GROUP @@ -140,11 +141,9 @@ class AbstractTable : public Printable { //===--------------------------------------------------------------------===// TileGroup *GetTileGroupWithLayout(oid_t database_id, oid_t tile_group_id, - const column_map_type &partitioning, + std::shared_ptr layout, const size_t num_tuples); - column_map_type GetTileGroupLayout() const; - //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// @@ -162,7 +161,8 @@ class AbstractTable : public Printable { */ bool own_schema_; - peloton::LayoutType layout_type; + // Default layout of the table + std::shared_ptr default_layout_; }; } // namespace storage diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index e3499501c37..1f986bc5e4e 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -149,7 +149,7 @@ class DataTable : public AbstractTable { size_t GetTileGroupCount() const; // Get a tile group with given layout - TileGroup *GetTileGroupWithLayout(const column_map_type &partitioning); + TileGroup *GetTileGroupWithLayout(std::shared_ptr layout); //===--------------------------------------------------------------------===// // TRIGGER @@ -248,9 +248,9 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); - void SetDefaultLayout(const column_map_type &layout); + void SetDefaultLayout(const column_map_type &column_map); - Layout GetDefaultLayout() const; + const Layout& GetDefaultLayout() const; //===--------------------------------------------------------------------===// // INDEX TUNER @@ -422,9 +422,6 @@ class DataTable : public AbstractTable { // adapt table bool adapt_table_ = true; - // default layout for the table - Layout default_layout_; - // default partition map for table column_map_type default_partition_; diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index eb89866fd28..cd619830bb4 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -21,6 +21,11 @@ #define COLUMN_STORE_OID 1 namespace peloton { + +namespace catalog { +class Schema; +} + namespace storage { class Layout : public Printable { @@ -53,6 +58,8 @@ class Layout : public Printable { oid_t GetColumnCount() const { return num_columns_;} + std::vector GetLayoutSchemas(catalog::Schema* const schema) const; + std::string GetColumnMapInfo() const; // Get a string representation for debugging diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index bf764962069..526bc41718c 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -75,7 +75,7 @@ class TileGroup : public Printable { // Tile group constructor TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count); + std::shared_ptr layout, int tuple_count); ~TileGroup(); @@ -161,7 +161,7 @@ class TileGroup : public Printable { // Sync the contents void Sync(); - const storage::Layout& GetLayout() const { return tile_group_layout_; } + const storage::Layout& GetLayout() const { return *tile_group_layout_; } protected: //===--------------------------------------------------------------------===// @@ -193,7 +193,8 @@ class TileGroup : public Printable { std::mutex tile_group_mutex; - Layout tile_group_layout_; + // Refernce to the layout of the TileGroup + std::shared_ptr tile_group_layout_; }; } // namespace storage diff --git a/src/include/storage/tile_group_factory.h b/src/include/storage/tile_group_factory.h index e6d99dd907d..3e615aba479 100644 --- a/src/include/storage/tile_group_factory.h +++ b/src/include/storage/tile_group_factory.h @@ -32,7 +32,7 @@ class TileGroupFactory { static TileGroup *GetTileGroup(oid_t database_id, oid_t table_id, oid_t tile_group_id, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, + std::shared_ptr layout, int tuple_count); }; diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index ee0607252d7..ada7c892149 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -26,67 +26,29 @@ namespace storage { AbstractTable::AbstractTable(oid_t table_oid, catalog::Schema *schema, bool own_schema, peloton::LayoutType layout_type) - : table_oid(table_oid), schema(schema), own_schema_(own_schema), layout_type(layout_type) {} + : table_oid(table_oid), schema(schema), own_schema_(own_schema) { + + // The default Layout should always be ROW or COLUMN + PELOTON_ASSERT((layout_type == LayoutType::ROW) || (layout_type == LayoutType::COLUMN)); + default_layout_ = std::shared_ptr(new const Layout(schema->GetColumnCount(), layout_type)); +} AbstractTable::~AbstractTable() { // clean up schema if (own_schema_) delete schema; } -column_map_type AbstractTable::GetTileGroupLayout() const { - column_map_type column_map; - - auto col_count = schema->GetColumnCount(); - - // pure row layout map - if (layout_type == LayoutType::ROW) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } - } - // pure column layout map - else if (layout_type == LayoutType::COLUMN) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(col_itr, 0); - } - } - // hybrid layout map - else if (layout_type == LayoutType::HYBRID) { - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } - } else { - throw Exception("Unknown tilegroup layout option : " + - LayoutTypeToString(layout_type)); - } - - return column_map; -} - TileGroup *AbstractTable::GetTileGroupWithLayout( - oid_t database_id, oid_t tile_group_id, const column_map_type &partitioning, + oid_t database_id, oid_t tile_group_id, + std::shared_ptr layout, const size_t num_tuples) { - std::vector schemas; - // Figure out the columns in each tile in new layout - std::map, oid_t> tile_column_map; - for (auto entry : partitioning) { - tile_column_map[entry.second] = entry.first; - } - - // Build the schema tile at a time - std::map> tile_schemas; - for (auto entry : tile_column_map) { - tile_schemas[entry.first.first].push_back(schema->GetColumn(entry.second)); - } - for (auto entry : tile_schemas) { - catalog::Schema tile_schema(entry.second); - schemas.push_back(tile_schema); - } + // Populate the schema for each tile + std::vector schemas = layout->GetLayoutSchemas(schema); TileGroup *tile_group = TileGroupFactory::GetTileGroup(database_id, GetOid(), tile_group_id, this, - schemas, partitioning, num_tuples); + schemas, layout, num_tuples); return tile_group; } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 2ad08b3a22f..59d0b1aece3 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -67,7 +67,6 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, table_name(table_name), tuples_per_tilegroup_(tuples_per_tilegroup), adapt_table_(adapt_table), - default_layout_(schema->GetColumnCount()), trigger_list_(new trigger::TriggerList()) { // Init default partition if (layout_type == LayoutType::ROW) { @@ -882,10 +881,10 @@ void DataTable::ResetDirty() { dirty_ = false; } //===--------------------------------------------------------------------===// TileGroup *DataTable::GetTileGroupWithLayout( - const column_map_type &partitioning) { + std::shared_ptr layout) { oid_t tile_group_id = catalog::Manager::GetInstance().GetNextTileGroupId(); return (AbstractTable::GetTileGroupWithLayout( - database_oid, tile_group_id, partitioning, tuples_per_tilegroup_)); + database_oid, tile_group_id, layout, tuples_per_tilegroup_)); } oid_t DataTable::AddDefaultIndirectionArray( @@ -910,14 +909,10 @@ oid_t DataTable::AddDefaultTileGroup() { } oid_t DataTable::AddDefaultTileGroup(const size_t &active_tile_group_id) { - column_map_type column_map; oid_t tile_group_id = INVALID_OID; - // Figure out the partitioning for given tilegroup layout - column_map = GetTileGroupLayout(); - // Create a tile group with that partitioning - std::shared_ptr tile_group(GetTileGroupWithLayout(column_map)); + std::shared_ptr tile_group(GetTileGroupWithLayout(default_layout_)); PELOTON_ASSERT(tile_group.get()); tile_group_id = tile_group->GetTileGroupId(); @@ -948,16 +943,17 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { std::vector schemas; schemas.push_back(*schema); + std::shared_ptr layout = nullptr; - column_map_type column_map; - // default column map - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); + if (default_layout_->IsRowStore()) { + layout = default_layout_; + } else { + layout = std::shared_ptr( + new const Layout(schema->GetColumnCount())); } std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( - database_oid, table_oid, tile_group_id, this, schemas, column_map, + database_oid, table_oid, tile_group_id, this, schemas, layout, tuples_per_tilegroup_)); auto tile_groups_exists = tile_groups_.Contains(tile_group_id); @@ -1301,7 +1297,7 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get orig tile group from catalog auto &catalog_manager = catalog::Manager::GetInstance(); auto tile_group = catalog_manager.GetTileGroup(tile_group_id); - auto diff = tile_group->GetLayout().GetLayoutDifference(default_layout_); + auto diff = tile_group->GetLayout().GetLayoutDifference(*default_layout_); // Check threshold for transformation if (diff < theta) { @@ -1319,7 +1315,7 @@ storage::TileGroup *DataTable::TransformTileGroup( TileGroupFactory::GetTileGroup( tile_group->GetDatabaseId(), tile_group->GetTableId(), tile_group->GetTileGroupId(), tile_group->GetAbstractTable(), - new_schema, default_partition_, + new_schema, default_layout_, tile_group->GetAllocatedTupleCount())); // Set the transformed tile group column-at-a-time @@ -1394,13 +1390,14 @@ std::map DataTable::GetColumnMapStats() { return column_map_stats; } -void DataTable::SetDefaultLayout(const column_map_type &layout) { - default_layout_ = Layout(layout); - default_partition_ = layout; +void DataTable::SetDefaultLayout(const column_map_type &column_map) { + default_layout_ = std::shared_ptr( + new const Layout(column_map)); + default_partition_ = column_map; } -Layout DataTable::GetDefaultLayout() const { - return default_layout_; +const Layout& DataTable::GetDefaultLayout() const { + return *default_layout_; } void DataTable::AddTrigger(trigger::Trigger new_trigger) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 483dcd5ecad..8fbc4848bfd 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -13,6 +13,8 @@ #include #include +#include "catalog/column.h" +#include "catalog/schema.h" #include "storage/layout.h" #include "util/stringbox_util.h" @@ -149,6 +151,29 @@ oid_t Layout::GetTileColumnId(oid_t column_id) const { return tile_column_id; } +std::vector Layout::GetLayoutSchemas( + catalog::Schema* const schema) const { + + std::vector schemas; + + // Build the schema tile at a time + std::map> tile_schemas; + + // This only works if the order of columns in the tile are the same + // as the schema. This snipet was initially in abstract_table.cpp. + for (auto column_info : column_layout_) { + tile_schemas[column_info.second.first].push_back( + schema->GetColumn(column_info.first)); + } + + for (auto entry : tile_schemas) { + catalog::Schema tile_schema(entry.second); + schemas.push_back(tile_schema); + } + + return schemas; +} + std::string Layout::GetColumnMapInfo() const { std::stringstream ss; std::map> tile_column_map; diff --git a/src/storage/temp_table.cpp b/src/storage/temp_table.cpp index b0739b912ce..0ecb4ec7dd5 100644 --- a/src/storage/temp_table.cpp +++ b/src/storage/temp_table.cpp @@ -104,8 +104,6 @@ std::shared_ptr TempTable::GetTileGroupById( } oid_t TempTable::AddDefaultTileGroup() { - column_map_type column_map; - // Well, a TempTable doesn't really care about TileGroupIds // And nobody else in the system should be referencing our boys directly, // so we're just going to use a simple counter for these ids @@ -114,14 +112,10 @@ oid_t TempTable::AddDefaultTileGroup() { oid_t tile_group_id = TEMPTABLE_TILEGROUP_ID + static_cast(tile_groups_.size()); - // Figure out the partitioning for given tilegroup layout - column_map = - AbstractTable::GetTileGroupLayout(); - // Create a tile group with that partitioning std::shared_ptr tile_group( AbstractTable::GetTileGroupWithLayout( - INVALID_OID, tile_group_id, column_map, TEMPTABLE_DEFAULT_SIZE)); + INVALID_OID, tile_group_id, default_layout_, TEMPTABLE_DEFAULT_SIZE)); PELOTON_ASSERT(tile_group.get()); tile_groups_.push_back(tile_group); diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 80351c6a09b..89d53239cc0 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -32,7 +32,7 @@ namespace storage { TileGroup::TileGroup(BackendType backend_type, TileGroupHeader *tile_group_header, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count) + std::shared_ptr layout, int tuple_count) : database_id(INVALID_OID), table_id(INVALID_OID), tile_group_id(INVALID_OID), @@ -40,7 +40,7 @@ TileGroup::TileGroup(BackendType backend_type, tile_group_header(tile_group_header), table(table), num_tuple_slots(tuple_count), - tile_group_layout_(column_map){ + tile_group_layout_(layout){ tile_count = schemas.size(); for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); @@ -319,7 +319,7 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, tile_column_id); return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } @@ -328,7 +328,7 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_.LocateTileAndColumn(column_id, + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } diff --git a/src/storage/tile_group_factory.cpp b/src/storage/tile_group_factory.cpp index f050d58e6df..c1fd897c09b 100644 --- a/src/storage/tile_group_factory.cpp +++ b/src/storage/tile_group_factory.cpp @@ -27,14 +27,14 @@ namespace storage { TileGroup *TileGroupFactory::GetTileGroup( oid_t database_id, oid_t table_id, oid_t tile_group_id, AbstractTable *table, const std::vector &schemas, - const column_map_type &column_map, int tuple_count) { + std::shared_ptr layout, int tuple_count) { // Allocate the data on appropriate backend BackendType backend_type = BackendType::MM; // logging::LoggingUtil::GetBackendType(peloton_logging_mode); TileGroupHeader *tile_header = new TileGroupHeader(backend_type, tuple_count); TileGroup *tile_group = new TileGroup(backend_type, tile_header, table, - schemas, column_map, tuple_count); + schemas, layout, tuple_count); tile_header->SetTileGroup(tile_group); From 1e00029e3d30991a1f3f0d78ce0fa5ea5db4e84e Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 09:03:05 -0400 Subject: [PATCH 15/42] Moved GetColumnMapStats -> GetColumnLayoutStats --- src/include/storage/data_table.h | 2 -- src/include/storage/layout.h | 2 ++ src/storage/data_table.cpp | 16 ---------------- src/storage/layout.cpp | 17 +++++++++++++++++ 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 1f986bc5e4e..73970825701 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -278,8 +278,6 @@ class DataTable : public AbstractTable { bool HasForeignKeys() const { return (foreign_keys_.empty() == false); } - std::map GetColumnMapStats(); - // try to insert into all indexes. // the last argument is the index entry in primary index holding the new // tuple. diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index cd619830bb4..af7e624c762 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -60,6 +60,8 @@ class Layout : public Printable { std::vector GetLayoutSchemas(catalog::Schema* const schema) const; + std::map GetColumnLayoutStats() const; + std::string GetColumnMapInfo() const; // Get a string representation for debugging diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 59d0b1aece3..f9b4065b700 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1374,22 +1374,6 @@ void DataTable::ClearIndexSamples() { } } -std::map DataTable::GetColumnMapStats() { - std::map column_map_stats; - - // Cluster per-tile column count - for (auto entry : default_partition_) { - auto tile_id = entry.second.first; - auto column_map_itr = column_map_stats.find(tile_id); - if (column_map_itr == column_map_stats.end()) - column_map_stats[tile_id] = 1; - else - column_map_stats[tile_id]++; - } - - return column_map_stats; -} - void DataTable::SetDefaultLayout(const column_map_type &column_map) { default_layout_ = std::shared_ptr( new const Layout(column_map)); diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 8fbc4848bfd..f4ee71d0686 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -174,6 +174,23 @@ std::vector Layout::GetLayoutSchemas( return schemas; } +std::map Layout::GetColumnLayoutStats() const { + std::map column_map_stats; + + // Cluster per-tile column count + for (auto entry : column_layout_) { + auto tile_id = entry.second.first; + auto column_map_itr = column_map_stats.find(tile_id); + if (column_map_itr == column_map_stats.end()) + column_map_stats[tile_id] = 1; + else + column_map_stats[tile_id]++; + } + + return column_map_stats; +} + + std::string Layout::GetColumnMapInfo() const { std::stringstream ss; std::map> tile_column_map; From 2df5b2c64f3fb5c3b917efc5204162bbc2557f87 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 09:06:51 -0400 Subject: [PATCH 16/42] Minor change in TableScanTranslator --- test/codegen/table_scan_translator_test.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 66404dd6e58..19c495827dc 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -41,7 +41,7 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { const int tuples_per_tilegroup= 100; const int tile_group_count = 5; const int tuple_count = tuples_per_tilegroup * tile_group_count; - const oid_t col_count = 128; + const oid_t col_count = 100; const bool is_inlined = true; @@ -66,8 +66,6 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { // Create table. ///////////////////////////////////////////////////////// - // bool own_schema = true; - // bool adapt_table = true; bool is_catalog = false; auto *catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); @@ -114,7 +112,7 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { txn_manager.CommitTransaction(txn); ///////////////////////////////////////////////////////// - // Do a seq scan with predicate on top of the table + // Do a seq scan on the table with the given layout ///////////////////////////////////////////////////////// // Column ids to be scanned. From 04843e779858be0e1b3bac0e85d8ffb1dc2dab9f Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 09:47:56 -0400 Subject: [PATCH 17/42] Modify TransformTileGroup --- src/include/storage/data_table.h | 3 --- src/storage/data_table.cpp | 17 +++++++++-------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 73970825701..eaf114575bd 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -420,9 +420,6 @@ class DataTable : public AbstractTable { // adapt table bool adapt_table_ = true; - // default partition map for table - column_map_type default_partition_; - // samples for layout tuning std::vector layout_samples_; diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index f9b4065b700..58381d3bd43 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1201,7 +1201,7 @@ catalog::ForeignKey *DataTable::GetForeignKeySrc(const size_t offset) const { // Get the schema for the new transformed tile group std::vector TransformTileGroupSchema( - storage::TileGroup *tile_group, const column_map_type &column_map) { + storage::TileGroup *tile_group, const Layout& layout) { std::vector new_schema; oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; @@ -1209,13 +1209,15 @@ std::vector TransformTileGroupSchema( // First, get info from the original tile group's schema std::map> schemas; - for (auto column_map_entry : column_map) { - new_tile_offset = column_map_entry.second.first; - new_tile_column_offset = column_map_entry.second.second; - oid_t column_offset = column_map_entry.first; - tile_group_layout.LocateTileAndColumn(column_offset, orig_tile_offset, + oid_t column_count = layout.GetColumnCount(); + for (oid_t col_id = 0; col_id < column_count; col_id++) { + // Get TileGroup layout's tile and offset for col_id. + tile_group_layout.LocateTileAndColumn(col_id, orig_tile_offset, orig_tile_column_offset); + // Get new layout's tile and offset for col_id. + layout.LocateTileAndColumn(col_id, new_tile_offset, + new_tile_column_offset); // Get the column info from original tile auto tile = tile_group->GetTile(orig_tile_offset); @@ -1308,7 +1310,7 @@ storage::TileGroup *DataTable::TransformTileGroup( // Get the schema for the new transformed tile group auto new_schema = - TransformTileGroupSchema(tile_group.get(), default_partition_); + TransformTileGroupSchema(tile_group.get(), *default_layout_); // Allocate space for the transformed tile group std::shared_ptr new_tile_group( @@ -1377,7 +1379,6 @@ void DataTable::ClearIndexSamples() { void DataTable::SetDefaultLayout(const column_map_type &column_map) { default_layout_ = std::shared_ptr( new const Layout(column_map)); - default_partition_ = column_map; } const Layout& DataTable::GetDefaultLayout() const { From 2f53ab5badb764819c3e2f145149aaff33bd6120 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 10:21:21 -0400 Subject: [PATCH 18/42] Change layout.cpp to better handle HYBRID layouts --- src/storage/layout.cpp | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index f4ee71d0686..98de2716414 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -29,8 +29,7 @@ Layout::Layout(const oid_t num_columns, LayoutType layout_type) // Constructor for the Layout class with column_map Layout::Layout(const column_map_type& column_map) - : layout_id_(ROW_STORE_OID), - num_columns_(column_map.size()), + : num_columns_(column_map.size()), column_layout_(column_map) { // Figure out the layout type if not present bool row_layout = true, column_layout = true; @@ -52,7 +51,9 @@ Layout::Layout(const column_map_type& column_map) layout_type_ = LayoutType::COLUMN; layout_id_ = COLUMN_STORE_OID; } else { - // TODO Pooja - Handle this by making an entry into the catalog + // layout_id_ is set to INVALID_OID, indicating that this + // layout is not stored in the catalog and thus not persistent. + // To be used only in TempTable or Tests. layout_type_ = LayoutType::HYBRID; layout_id_ = INVALID_OID; } @@ -254,8 +255,17 @@ bool operator==(const Layout& lhs, const Layout& rhs) { return false; } - // Check for the equality of the column_layout_ - return (lhs.column_layout_ == rhs.column_layout_); + // Check for the equality of the column_layout_ for + // LayoutType::HYBRID + if (lhs.layout_type_ == LayoutType::HYBRID) + { + return (lhs.column_layout_ == rhs.column_layout_); + } + // In case of LayoutType::ROW or LayoutType::COLUMN, + // there is no need to check for column_layout_ equality + + return true; + } bool operator!=(const Layout& lhs, const Layout& rhs) { From ed3db38d0c2afc481b9904669c3b3e1f84918e9d Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 10:39:35 -0400 Subject: [PATCH 19/42] use std::make_shared --- src/storage/abstract_table.cpp | 2 +- src/storage/data_table.cpp | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index ada7c892149..a1467832f1c 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -30,7 +30,7 @@ AbstractTable::AbstractTable(oid_t table_oid, catalog::Schema *schema, // The default Layout should always be ROW or COLUMN PELOTON_ASSERT((layout_type == LayoutType::ROW) || (layout_type == LayoutType::COLUMN)); - default_layout_ = std::shared_ptr(new const Layout(schema->GetColumnCount(), layout_type)); + default_layout_ = std::make_shared(schema->GetColumnCount(), layout_type); } AbstractTable::~AbstractTable() { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 58381d3bd43..b4686f66177 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -948,8 +948,7 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { if (default_layout_->IsRowStore()) { layout = default_layout_; } else { - layout = std::shared_ptr( - new const Layout(schema->GetColumnCount())); + layout = std::make_shared((schema->GetColumnCount()); } std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( @@ -1377,8 +1376,7 @@ void DataTable::ClearIndexSamples() { } void DataTable::SetDefaultLayout(const column_map_type &column_map) { - default_layout_ = std::shared_ptr( - new const Layout(column_map)); + default_layout_ = std::make_shared(column_map); } const Layout& DataTable::GetDefaultLayout() const { From 2daeb37c442d539a92467e82cebcc3a64e53f665 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 10:43:50 -0400 Subject: [PATCH 20/42] Modified tests to change calls to GetTileGroup() --- test/catalog/manager_test.cpp | 7 ++++--- test/executor/seq_scan_test.cpp | 6 ++++++ test/executor/testing_executor_util.cpp | 5 ++++- test/executor/update_test.cpp | 10 ++++++++-- test/storage/tile_group_test.cpp | 26 +++++++++++++++---------- 5 files changed, 38 insertions(+), 16 deletions(-) diff --git a/test/catalog/manager_test.cpp b/test/catalog/manager_test.cpp index a0112f41f46..f0556f01734 100644 --- a/test/catalog/manager_test.cpp +++ b/test/catalog/manager_test.cpp @@ -16,6 +16,7 @@ #include "common/macros.h" #include "catalog/manager.h" #include "catalog/schema.h" +#include "storage/layout.h" #include "storage/tile_group.h" #include "storage/tile_group_factory.h" @@ -48,12 +49,12 @@ void AddTileGroup(UNUSED_ATTRIBUTE uint64_t thread_id) { std::unique_ptr schema1(new catalog::Schema(columns)); schemas.push_back(*schema1); - std::map> column_map; - column_map[0] = std::make_pair(0, 0); + std::shared_ptr layout = + std::make_shared(columns.size()); for (oid_t txn_itr = 0; txn_itr < 100; txn_itr++) { std::unique_ptr tile_group(storage::TileGroupFactory::GetTileGroup( - INVALID_OID, INVALID_OID, INVALID_OID, nullptr, schemas, column_map, 3)); + INVALID_OID, INVALID_OID, INVALID_OID, nullptr, schemas, layout, 3)); } } diff --git a/test/executor/seq_scan_test.cpp b/test/executor/seq_scan_test.cpp index 0e9f11d6a87..dc6c0fc6c4a 100644 --- a/test/executor/seq_scan_test.cpp +++ b/test/executor/seq_scan_test.cpp @@ -83,12 +83,18 @@ storage::DataTable *CreateTable() { column_map1[1] = std::make_pair(0, 1); column_map1[2] = std::make_pair(1, 0); column_map1[3] = std::make_pair(1, 1); + std::shared_ptr layout1 = + std::make_shared(column_map1); + std::map> column_map2; column_map2[0] = std::make_pair(0, 0); column_map2[1] = std::make_pair(1, 0); column_map2[2] = std::make_pair(1, 1); column_map2[3] = std::make_pair(1, 2); + std::shared_ptr layout2 = + std::make_shared(column_map2); + // Create tile groups. table->AddTileGroup(std::shared_ptr( diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 0021f90d6fd..6aff93a2bf2 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -170,11 +170,14 @@ std::shared_ptr TestingExecutorUtil::CreateTileGroup( column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::make_shared(column_map); + std::shared_ptr tile_group_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, tuple_count)); + layout, tuple_count)); catalog::Manager::GetInstance().AddTileGroup(tile_group_ptr->GetTileGroupId(), tile_group_ptr); diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index f8f3c12d0a2..053446c4063 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -90,24 +90,30 @@ storage::DataTable *CreateTable() { column_map1[2] = std::make_pair(1, 0); column_map1[3] = std::make_pair(1, 1); + std::shared_ptr layout1 = + std::make_shared(column_map1); + std::map> column_map2; column_map2[0] = std::make_pair(0, 0); column_map2[1] = std::make_pair(1, 0); column_map2[2] = std::make_pair(1, 1); column_map2[3] = std::make_pair(1, 2); + std::shared_ptr layout2 = + std::make_shared(column_map2); + // Create tile groups. table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas1, column_map1, tuple_count))); + schemas1, layout1, tuple_count))); table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas2, column_map2, tuple_count))); + schemas2, layout2, tuple_count))); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(0), tuple_count); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(1), tuple_count); diff --git a/test/storage/tile_group_test.cpp b/test/storage/tile_group_test.cpp index d42fe4e3f8e..36c7a998f77 100644 --- a/test/storage/tile_group_test.cpp +++ b/test/storage/tile_group_test.cpp @@ -84,11 +84,14 @@ TEST_F(TileGroupTests, BasicTest) { column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::make_shared(column_map); + std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, 4)); + layout, 4)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); @@ -212,11 +215,14 @@ TEST_F(TileGroupTests, StressTest) { column_map[2] = std::make_pair(1, 0); column_map[3] = std::make_pair(1, 1); + std::shared_ptr layout = + std::make_shared(column_map); + std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, 10000)); + layout, 10000)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); @@ -272,11 +278,14 @@ TEST_F(TileGroupTests, StressTest) { // column_map[2] = std::make_pair(1, 0); // column_map[3] = std::make_pair(1, 1); // +// std::shared_ptr layout = +// std::make_shared(column_map); +// // std::shared_ptr tile_group = // storage::TileGroupFactory::GetTileGroup( // INVALID_OID, INVALID_OID, // TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, -// column_map, 3); +// layout, 3); // catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), // tile_group); // @@ -371,19 +380,16 @@ TEST_F(TileGroupTests, TileCopyTest) { column_names.push_back(tile_column_names); const int tuple_count = 4; - column_map_type column_map; - // default column map - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - column_map[col_itr] = std::make_pair(0, col_itr); - } + + std::shared_ptr layout = + std::make_shared(schema->GetColumnCount()); std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), nullptr, schemas, - column_map, tuple_count)); + layout, tuple_count)); catalog::Manager::GetInstance().AddTileGroup(tile_group->GetTileGroupId(), tile_group); From 8162337b2b0d1886176f7f7ebd782ea126b83976 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 17 Apr 2018 14:40:15 -0400 Subject: [PATCH 21/42] make check passes. Need to add test with hybrid layout. --- src/gc/gc_manager.cpp | 2 +- src/include/storage/tile_group.h | 6 ++--- src/storage/abstract_table.cpp | 3 ++- src/storage/data_table.cpp | 5 +++-- src/storage/layout.cpp | 29 +++++++++++++++++-------- src/storage/tile_group.cpp | 14 ++++++------ test/executor/seq_scan_test.cpp | 4 ++-- test/executor/testing_executor_util.cpp | 3 ++- test/executor/update_test.cpp | 6 +++-- 9 files changed, 44 insertions(+), 28 deletions(-) diff --git a/src/gc/gc_manager.cpp b/src/gc/gc_manager.cpp index 070ab6374b9..07bdbc34334 100644 --- a/src/gc/gc_manager.cpp +++ b/src/gc/gc_manager.cpp @@ -24,7 +24,7 @@ namespace gc { // Check a tuple and reclaim all varlen field void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id) { - oid_t tile_count = tile_group->tile_count; + oid_t tile_count = tile_group->tile_count_; oid_t tile_col_count; type::TypeId type_id; char *tuple_location; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 526bc41718c..2f9ea2cd815 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -130,7 +130,7 @@ class TileGroup : public Printable { // Get the tile at given offset in the tile group inline Tile *GetTile(const oid_t tile_offset) const { - PELOTON_ASSERT(tile_offset < tile_count); + PELOTON_ASSERT(tile_offset < tile_count_); Tile *tile = tiles[tile_offset].get(); return tile; } @@ -152,7 +152,7 @@ class TileGroup : public Printable { void SetTileGroupId(oid_t tile_group_id_) { tile_group_id = tile_group_id_; } - size_t GetTileCount() const { return tile_count; } + size_t GetTileCount() const { return tile_count_; } type::Value GetValue(oid_t tuple_id, oid_t column_id); @@ -189,7 +189,7 @@ class TileGroup : public Printable { oid_t num_tuple_slots; // number of tiles - oid_t tile_count; + oid_t tile_count_; std::mutex tile_group_mutex; diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index a1467832f1c..69557d847a1 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -30,7 +30,8 @@ AbstractTable::AbstractTable(oid_t table_oid, catalog::Schema *schema, // The default Layout should always be ROW or COLUMN PELOTON_ASSERT((layout_type == LayoutType::ROW) || (layout_type == LayoutType::COLUMN)); - default_layout_ = std::make_shared(schema->GetColumnCount(), layout_type); + default_layout_ = std::shared_ptr( + new Layout(schema->GetColumnCount(), layout_type)); } AbstractTable::~AbstractTable() { diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index b4686f66177..2549e92545a 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -948,7 +948,8 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { if (default_layout_->IsRowStore()) { layout = default_layout_; } else { - layout = std::make_shared((schema->GetColumnCount()); + layout = std::shared_ptr( + new const Layout(schema->GetColumnCount())); } std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( @@ -1376,7 +1377,7 @@ void DataTable::ClearIndexSamples() { } void DataTable::SetDefaultLayout(const column_map_type &column_map) { - default_layout_ = std::make_shared(column_map); + default_layout_ = std::shared_ptr(new const Layout(column_map)); } const Layout& DataTable::GetDefaultLayout() const { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 98de2716414..667a5162905 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -160,16 +160,27 @@ std::vector Layout::GetLayoutSchemas( // Build the schema tile at a time std::map> tile_schemas; - // This only works if the order of columns in the tile are the same - // as the schema. This snipet was initially in abstract_table.cpp. - for (auto column_info : column_layout_) { - tile_schemas[column_info.second.first].push_back( - schema->GetColumn(column_info.first)); - } + // Handle schemas population based on the layout type + if (layout_type_ == LayoutType::ROW) { + schemas.push_back(*schema); + } else if (layout_type_ == LayoutType::COLUMN) { + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + std::vector tile_schema({schema->GetColumn(col_id)}); + schemas.push_back(tile_schema); + } + } else { + // For LayoutType::HYBRID, use the old technique. + // This only works if the order of columns in the tile are the same + // as the schema. This snipet was initially in abstract_table.cpp. + for (auto column_info : column_layout_) { + tile_schemas[column_info.second.first].push_back( + schema->GetColumn(column_info.first)); + } - for (auto entry : tile_schemas) { - catalog::Schema tile_schema(entry.second); - schemas.push_back(tile_schema); + for (auto entry : tile_schemas) { + catalog::Schema tile_schema(entry.second); + schemas.push_back(tile_schema); + } } return schemas; diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 89d53239cc0..2b23587a06f 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -41,8 +41,8 @@ TileGroup::TileGroup(BackendType backend_type, table(table), num_tuple_slots(tuple_count), tile_group_layout_(layout){ - tile_count = schemas.size(); - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + tile_count_ = schemas.size(); + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); oid_t tile_id = manager.GetNextTileId(); @@ -107,7 +107,7 @@ void TileGroup::CopyTuple(const Tuple *tuple, const oid_t &tuple_slot_id) { oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); const catalog::Schema *schema = tile->GetSchema(); @@ -188,7 +188,7 @@ oid_t TileGroup::InsertTupleFromRecovery(cid_t commit_id, oid_t tuple_slot_id, oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); const catalog::Schema *schema = tile->GetSchema(); @@ -287,7 +287,7 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, oid_t tile_column_count; oid_t column_itr = 0; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); @@ -336,7 +336,7 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, std::shared_ptr TileGroup::GetTileReference( const oid_t tile_offset) const { - PELOTON_ASSERT(tile_offset < tile_count); + PELOTON_ASSERT(tile_offset < tile_count_); return tiles[tile_offset]; } @@ -360,7 +360,7 @@ const std::string TileGroup::GetInfo() const { os << "Table[" << table_id << "] " << std::endl; os << (*tile_group_header) << std::endl; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { Tile *tile = GetTile(tile_itr); if (tile != nullptr) { os << std::endl << (*tile); diff --git a/test/executor/seq_scan_test.cpp b/test/executor/seq_scan_test.cpp index dc6c0fc6c4a..d9e9d0dbf27 100644 --- a/test/executor/seq_scan_test.cpp +++ b/test/executor/seq_scan_test.cpp @@ -101,13 +101,13 @@ storage::DataTable *CreateTable() { storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas1, column_map1, tuple_count))); + schemas1, layout1, tuple_count))); table->AddTileGroup(std::shared_ptr( storage::TileGroupFactory::GetTileGroup( INVALID_OID, INVALID_OID, TestingHarness::GetInstance().GetNextTileGroupId(), table.get(), - schemas2, column_map2, tuple_count))); + schemas2, layout2, tuple_count))); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(0), tuple_count); TestingExecutorUtil::PopulateTiles(table->GetTileGroup(1), tuple_count); diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 6aff93a2bf2..65ee34ed272 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -29,6 +29,7 @@ #include "index/index_factory.h" #include "storage/data_table.h" #include "storage/database.h" +#include "storage/layout.h" #include "storage/table_factory.h" #include "storage/tile.h" #include "storage/tile_group.h" @@ -171,7 +172,7 @@ std::shared_ptr TestingExecutorUtil::CreateTileGroup( column_map[3] = std::make_pair(1, 1); std::shared_ptr layout = - std::make_shared(column_map); + std::shared_ptr(new const storage::Layout(column_map)); std::shared_ptr tile_group_ptr( storage::TileGroupFactory::GetTileGroup( diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index 053446c4063..7e0fd9d997d 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -91,7 +91,8 @@ storage::DataTable *CreateTable() { column_map1[3] = std::make_pair(1, 1); std::shared_ptr layout1 = - std::make_shared(column_map1); + std::shared_ptr( + new const storage::Layout(column_map1)); std::map> column_map2; column_map2[0] = std::make_pair(0, 0); @@ -100,7 +101,8 @@ storage::DataTable *CreateTable() { column_map2[3] = std::make_pair(1, 2); std::shared_ptr layout2 = - std::make_shared(column_map2); + std::shared_ptr( + new const storage::Layout(column_map1)); // Create tile groups. table->AddTileGroup(std::shared_ptr( From 541034804579bfe3bafd5c2a991f2f7fd838a6fd Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 18 Apr 2018 21:48:29 -0400 Subject: [PATCH 22/42] Modify print functions --- src/storage/layout.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 667a5162905..b4b27ca7d62 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -214,6 +214,12 @@ std::string Layout::GetColumnMapInfo() const { for (oid_t col_id = 0; col_id < num_columns_; col_id++) { tile_column_map[tile_id].push_back(col_id); } + } else if (layout_type_ == COLUMN_STORE_OID) { + // Column store always contains 1 column per tile. + oid_t tile_col_id = 0; + for (oid_t col_id = 0; col_id < num_columns_; col_id++) { + tile_column_map[col_id].push_back(tile_col_id); + } } else { for (auto column_info : column_layout_) { oid_t col_id = column_info.first; @@ -243,7 +249,7 @@ const std::string Layout::GetInfo() const { os << peloton::GETINFO_DOUBLE_STAR << " Layout[#" << layout_id_ << "] " << peloton::GETINFO_DOUBLE_STAR << std::endl; os << "Number of columns[" << num_columns_ << "] " << std::endl; - + os << "LayoutType[" << LayoutTypeToString(layout_type_) << std::endl; os << "ColumnMap[ " << GetColumnMapInfo() << "] " << std::endl; return peloton::StringUtil::Prefix(peloton::StringBoxUtil::Box(os.str()), From 8f8bf0e2f04b681deeb557252c61a69157a366be Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 18 Apr 2018 22:46:03 -0400 Subject: [PATCH 23/42] Add TODOs for catalog --- src/storage/data_table.cpp | 7 ++++++- src/storage/temp_table.cpp | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 2549e92545a..e66006c4ac0 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -945,6 +945,9 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { schemas.push_back(*schema); std::shared_ptr layout = nullptr; + // The TileGroup for recovery is always added in ROW layout, + // This was a part of the previous design. If you are planning + // to change this, make sure the layout is added to the catalog if (default_layout_->IsRowStore()) { layout = default_layout_; } else { @@ -1377,7 +1380,9 @@ void DataTable::ClearIndexSamples() { } void DataTable::SetDefaultLayout(const column_map_type &column_map) { - default_layout_ = std::shared_ptr(new const Layout(column_map)); + // TODO Pooja: Generate layout_oid_ and add it to the catalog + default_layout_ = std::shared_ptr( + new const Layout(column_map)); } const Layout& DataTable::GetDefaultLayout() const { diff --git a/src/storage/temp_table.cpp b/src/storage/temp_table.cpp index 0ecb4ec7dd5..d9db400e3b1 100644 --- a/src/storage/temp_table.cpp +++ b/src/storage/temp_table.cpp @@ -113,6 +113,7 @@ oid_t TempTable::AddDefaultTileGroup() { TEMPTABLE_TILEGROUP_ID + static_cast(tile_groups_.size()); // Create a tile group with that partitioning + // Its a TempTable, so we don't need to make the Layout persistent std::shared_ptr tile_group( AbstractTable::GetTileGroupWithLayout( INVALID_OID, tile_group_id, default_layout_, TEMPTABLE_DEFAULT_SIZE)); From faedaa1a53de014f10b9f5f597366d694b358a2c Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Thu, 19 Apr 2018 14:20:33 -0400 Subject: [PATCH 24/42] Added LayoutCatalog. Yet to modify TableCatalog --- src/catalog/layout_catalog.cpp | 183 +++++++++++++++++++++++++ src/include/catalog/catalog_defaults.h | 6 + src/include/catalog/layout_catalog.h | 94 +++++++++++++ src/include/catalog/table_catalog.h | 22 +++ src/include/storage/data_table.h | 22 ++- src/include/storage/layout.h | 5 + src/storage/data_table.cpp | 16 ++- src/storage/layout.cpp | 49 ++++++- 8 files changed, 391 insertions(+), 6 deletions(-) create mode 100644 src/catalog/layout_catalog.cpp create mode 100644 src/include/catalog/layout_catalog.h diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp new file mode 100644 index 00000000000..301e55715aa --- /dev/null +++ b/src/catalog/layout_catalog.cpp @@ -0,0 +1,183 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// column_catalog.h +// +// Identification: src/include/catalog/layout_catalog.cpp +// +// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +#include "catalog/layout_catalog.h" + +#include "catalog/column_catalog.h" +#include "catalog/table_catalog.h" +#include "concurrency/transaction_context.h" +#include "storage/data_table.h" +#include "storage/layout.h" + +namespace peloton { +namespace catalog { + +LayoutCatalog *LayoutCatalog::GetInstance(storage::Database *pg_catalog, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + static LayoutCatalog layout_catalog{pg_catalog, pool, txn}; + return &layout_catalog; +} + +LayoutCatalog::LayoutCatalog(storage::Database *pg_catalog, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) + : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, + InitializeSchema().release(), pg_catalog) { + // Add indexes for pg_attribute + AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, + LAYOUT_CATALOG_PKEY_OID, LAYOUT_CATALOG_NAME "_pkey", + IndexConstraintType::PRIMARY_KEY); + AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, + LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); + + // Insert columns into pg_attribute + ColumnCatalog *pg_attribute = + ColumnCatalog::GetInstance(pg_catalog, pool, txn); + + oid_t column_id = 0; + for (auto column : catalog_table_->GetSchema()->GetColumns()) { + pg_attribute->InsertColumn(LAYOUT_CATALOG_OID, column.GetName(), column_id, + column.GetOffset(), column.GetType(), + column.IsInlined(), column.GetConstraints(), + pool, txn); + column_id++; + } +} + +LayoutCatalog::~LayoutCatalog() {} + +std::unique_ptr ColumnCatalog::InitializeSchema() { + const std::string primary_key_constraint_name = "primary_key"; + const std::string not_null_constraint_name = "not_null"; + + auto table_id_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "table_oid", true); + table_id_column.AddConstraint(catalog::Constraint( + ConstraintType::PRIMARY, primary_key_constraint_name)); + table_id_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto layout_oid_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "layout_oid", true); + layout_oid_column.AddConstraint(catalog::Constraint( + ConstraintType::PRIMARY, primary_key_constraint_name)); + layout_oid_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto num_columns_column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "num_columns", true); + num_columns_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + auto column_map_column = catalog::Column( + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "column_map", false); + column_map_column.AddConstraint( + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + + std::unique_ptr column_catalog_schema(new catalog::Schema( + {table_id_column, layout_oid_column, num_columns_column, column_map_column})); + + return column_catalog_schema; +} + +bool LayoutCatalog::InsertLayout(oid_t table_oid, + std::shared_ptr layout, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + // Create the tuple first + std::unique_ptr tuple( + new storage::Tuple(catalog_table_->GetSchema(), true)); + + auto val0 = type::ValueFactory::GetIntegerValue(table_oid); + auto val1 = type::ValueFactory::GetIntegerValue(layout->GetLayoutId()); + auto val2 = type::ValueFactory::GetIntegerValue(layout->GetColumnCount()); + auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(), + nullptr); + + tuple->SetValue(ColumnId::TABLE_OID, val0); + tuple->SetValue(ColumnId::LAYOUT_OID, val1); + tuple->SetValue(ColumnId::NUM_COLUMNS, val2); + tuple->SetValue(ColumnId::COLUMN_MAP, val3); + + // Insert the tuple + return InsertTuple(std::move(tuple), txn); +} + +bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, + concurrency::TransactionContext *txn) { + oid_t index_offset = + IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid + + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(layout_id).Copy()); + + // delete column from cache + auto table_object = + TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + table_object->EvictLayout(layout_id); + + return DeleteWithIndexScan(index_offset, values, txn); +} + +const std::unordered_map> +LayoutCatalog::GetLayouts(oid_t table_oid, oid_t layout_id, + concurrency::TransactionContext *txn) { + // Try to find the layouts in the cache + auto table_object = + TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); + auto layout_objects = table_object->GetLayouts(true); + if (layout_objects.size() != 0) { + return layout_objects; + } + + // Cache miss, get from pg_catalog + std::vector column_ids(all_column_ids); + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + auto result_tiles = + GetResultWithIndexScan(column_ids, index_offset, values, txn); + + for (auto &tile : (*result_tiles)) { + for (auto tuple_id : *tile) { + oid_t layout_oid = tile->GetValue(tuple_id, ColumnId::LAYOUT_OID) + .GetAs(); + oid_t num_coulmns = tile->GetValue(tuple_id, ColumnId::NUM_COLUMNS) + .GetAs(); + + std::string column_map_str = tile->GetValue( + tuple_id, ColumnId::COLUMN_MAP).GetAs(); + auto column_map = storage::Layout::DeserializeColumnMap(num_coulmns, + column_map_str); + auto layout_object = + std::make_shared(column_map, layout_oid); + table_object->InsertLayout(layout_object); + } + } + + return table_object->GetLayouts(); +} + + + + + +} // namespace catalog +} // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index 2cfcacbda70..c0e3b681098 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -32,6 +32,7 @@ namespace catalog { #define TABLE_CATALOG_NAME "pg_table" #define INDEX_CATALOG_NAME "pg_index" #define COLUMN_CATALOG_NAME "pg_attribute" +#define LAYOUT_CATALOG_NAME "pg_layout" // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 @@ -62,6 +63,7 @@ namespace catalog { #define TABLE_CATALOG_OID (2 | TABLE_OID_MASK) #define INDEX_CATALOG_OID (3 | TABLE_OID_MASK) #define COLUMN_CATALOG_OID (4 | TABLE_OID_MASK) +#define LAYOUT_CATALOG_OID (5 | TABLE_OID_MASK) // Reserved pg_column index oid #define COLUMN_CATALOG_PKEY_OID (0 | INDEX_OID_MASK) @@ -86,6 +88,10 @@ namespace catalog { #define TABLE_CATALOG_SKEY0_OID (11 | INDEX_OID_MASK) #define TABLE_CATALOG_SKEY1_OID (12 | INDEX_OID_MASK) +// Reserve pg_layout index oid +#define LAYOUT_CATALOG_PKEY_OID (13 | INDEX_OID_MASK) +#define LAYOUT_CATALOG_SKEY0_OID (14 | INDEX_OID_MASK) + // Use upper 8 bits indicating catalog type #define CATALOG_TYPE_OFFSET 24 diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h new file mode 100644 index 00000000000..f8f24677e9e --- /dev/null +++ b/src/include/catalog/layout_catalog.h @@ -0,0 +1,94 @@ +//===----------------------------------------------------------------------===// +// +// Peloton +// +// layout_catalog.h +// +// Identification: src/include/catalog/layout_catalog.h +// +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// pg_layout +// +// Schema: (column offset: column_name) +// 0: table_oid (pkey) +// 1: layout_oid (pkey) +// 2: num_columns (number of columns in the layout) +// 3: column_map (map column_oid to ) +// +// Indexes: (index offset: indexed columns) +// 0: table_oid & layout_oid (unique & primary key) +// 1: table_oid (non-unique) +// +//===----------------------------------------------------------------------===// + +#pragma once + +#include "catalog/abstract_catalog.h" + +namespace peloton { + +namespace storage { +class Layout; +} // namespace storage + +namespace catalog { + +class LayoutCatalog : public AbstractCatalog { + friend class TableCatalogObject; + friend class Catalog; + +public: + // Global Singleton, only the first call requires passing parameters. + static LayoutCatalog *GetInstance(storage::Database *pg_catalog = nullptr, + type::AbstractPool *pool = nullptr, + concurrency::TransactionContext *txn = nullptr); + + ~LayoutCatalog(); + + //===--------------------------------------------------------------------===// + // write Related API + //===--------------------------------------------------------------------===// + bool InsertLayout(oid_t table_oid, + std::shared_ptr layout, + type::AbstractPool *pool, + concurrency::TransactionContext *txn); + + bool DeleteLayout(oid_t table_oid, oid_t layout_id, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// + // Read Related API + //===--------------------------------------------------------------------===// + const std::unordered_map> + GetLayouts(oid_t table_oid, oid_t layout_id, + concurrency::TransactionContext *txn); + +private: + + LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, + concurrency::TransactionContext *txn); + + std::unique_ptr InitializeSchema(); + + enum ColumnId { + TABLE_OID = 0, + LAYOUT_OID = 1, + NUM_COLUMNS = 2, + COLUMN_MAP = 3, + // Add new columns here in creation order + }; + std::vector all_column_ids = {0, 1, 2, 3}; + + enum IndexId { + PRIMARY_KEY = 0, + SKEY_TABLE_OID = 1, + // Add new indexes here in creation order + }; +}; + + +} // namespace catalog +} // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index 0dfc3f51fa9..fa8b8bb6606 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -34,6 +34,7 @@ #include "catalog/abstract_catalog.h" #include "executor/logical_tile.h" +#include "storage/layout.h" namespace peloton { namespace catalog { @@ -45,6 +46,7 @@ class TableCatalogObject { friend class TableCatalog; friend class IndexCatalog; friend class ColumnCatalog; + friend class LayoutCatalog; public: TableCatalogObject(executor::LogicalTile *tile, @@ -73,6 +75,15 @@ class TableCatalogObject { std::shared_ptr GetColumnObject( const std::string &column_name, bool cached_only = false); + // Evict all layouts from the cache + void EvictAllLayoutObjects(); + + // Get layouts + std::unordered_map> + GetLayouts(bool cached_only = false); + std::shared_ptr GetLayout( + oid_t layout_id, bool cached_entry = false); + inline oid_t GetTableOid() { return table_oid; } inline const std::string &GetTableName() { return table_name; } inline const std::string &GetSchemaName() { return schema_name; } @@ -97,6 +108,11 @@ class TableCatalogObject { bool EvictColumnObject(oid_t column_id); bool EvictColumnObject(const std::string &column_name); + // Insert layout into table object + bool InsertLayout(std::shared_ptr layout); + // Evict layout_id from the table object + bool EvictLayout(oid_t layout_id); + // cache for *all* index catalog objects in this table std::unordered_map> index_objects; std::unordered_map> @@ -110,6 +126,11 @@ class TableCatalogObject { column_names; bool valid_column_objects; + // cache for *all* layout objects in the table + std::unordered_map> + layout_objects_; + bool valid_layout_objects_; + // Pointer to its corresponding transaction concurrency::TransactionContext *txn; }; @@ -119,6 +140,7 @@ class TableCatalog : public AbstractCatalog { friend class DatabaseCatalogObject; friend class ColumnCatalog; friend class IndexCatalog; + friend class LayoutCatalog; friend class Catalog; public: diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index eaf114575bd..268a7c0c160 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -352,11 +352,27 @@ class DataTable : public AbstractTable { bool CheckForeignKeyConstraints(const AbstractTuple *tuple, concurrency::TransactionContext *transaction); + //===--------------------------------------------------------------------===// + // LAYOUT HELPERS + //===--------------------------------------------------------------------===// + + // Set the current_layout_oid_ to the given value if the current value + // is less than new_layout_oid. Return true on success. + // To be used for recovery. + bool SetCurrentLayoutOid(oid_t new_layout_oid); + + // Performs an atomic increment on the current_layout_oid_ + // and returns the incremented value. + oid_t GetNextLayoutOid() { return ++current_layout_oid_; } + private: + + //===--------------------------------------------------------------------===// + // STATIC MEMBERS + //===--------------------------------------------------------------------===// static size_t default_active_tilegroup_count_; static size_t default_active_indirection_array_count_; - private: //===--------------------------------------------------------------------===// // MEMBERS //===--------------------------------------------------------------------===// @@ -413,6 +429,10 @@ class DataTable : public AbstractTable { // dirty flag. for detecting whether the tile group has been used. bool dirty_ = false; + // Last used layout_oid. Used while creating new layouts + // Initialized to COLUMN_STORE_OID since its the highest predfeined value. + std::atomic current_layout_oid_ = ATOMIC_VAR_INIT(COLUMN_STORE_OID); + //===--------------------------------------------------------------------===// // TUNING MEMBERS //===--------------------------------------------------------------------===// diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index af7e624c762..ec455ca5d54 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -62,6 +62,11 @@ class Layout : public Printable { std::map GetColumnLayoutStats() const; + std::string SerializeColumnMap() const; + + static column_map_type DeserializeColumnMap(oid_t num_columns, + std::string column_map_str); + std::string GetColumnMapInfo() const; // Get a string representation for debugging diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index e66006c4ac0..3e5837a6afb 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1436,5 +1436,17 @@ bool DataTable::operator==(const DataTable &rhs) const { return true; } -} // namespace storage -} // namespace peloton +bool DataTable::SetCurrentLayoutOid(oid_t new_layout_oid) { + oid_t old_oid = current_layout_oid_; + while (old_oid <= new_layout_oid) { + if (current_layout_oid_.compare_exchange_strong( + old_oid, new_layout_oid)) { + return true; + } + old_oid = current_layout_oid_; + } + return false; +} + +} // End storage namespace +} // End peloton namespace diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index b4b27ca7d62..706b38af403 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -202,19 +202,62 @@ std::map Layout::GetColumnLayoutStats() const { return column_map_stats; } +std::string Layout::SerializeColumnMap() const { + std::stringstream ss; + + for (auto column_info : column_layout_) { + ss << column_info.first << ":"; + ss << column_info.second.first << ":"; + ss << column_info.second.second << ","; + } + + return ss.str(); +} + +column_map_type Layout::DeserializeColumnMap(oid_t num_columns, + std::string column_map_str) { + column_map_type column_map; + std::stringstream ss(column_map_str); + + for (oid_t col_id = 0; col_id < num_columns; col_id++) { + oid_t str_col_id, tile_id, tile_col_id; + // Read col_id from column_map_str + ss >> str_col_id; + PELOTON_ASSERT(str_col_id == col_id); + + PELOTON_ASSERT(ss.peek() == ':'); + ss.ignore(); + // Read tile_id from column_map_str + ss >> tile_id; + + PELOTON_ASSERT(ss.peek() == ':'); + ss.ignore(); + // Read tile_col_id from column_map_str + ss >> tile_col_id; + + // Insert the column info into column_map + column_map[col_id] = std::make_pair(tile_id, tile_col_id); + + if (ss.peek() == ',') { + ss.ignore(); + } + } + + return column_map; +} std::string Layout::GetColumnMapInfo() const { std::stringstream ss; std::map> tile_column_map; - if (layout_id_ == ROW_STORE_OID) { + if (layout_type_ == LayoutType::ROW) { // Row store always contains only 1 tile. The tile_id is always 0. oid_t tile_id = 0; tile_column_map[tile_id] = {}; for (oid_t col_id = 0; col_id < num_columns_; col_id++) { tile_column_map[tile_id].push_back(col_id); } - } else if (layout_type_ == COLUMN_STORE_OID) { + } else if (layout_type_ == LayoutType::COLUMN) { // Column store always contains 1 column per tile. oid_t tile_col_id = 0; for (oid_t col_id = 0; col_id < num_columns_; col_id++) { @@ -233,7 +276,7 @@ std::string Layout::GetColumnMapInfo() const { // Construct a string from tile_col_map for (auto tile_info : tile_column_map) { - ss << tile_info.first << ": "; + ss << tile_info.first << " : "; for (auto col_id : tile_info.second) { ss << col_id << " "; } From fe1e6b43c7c91d245a4fa6895f30d6ad4ed0c827 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 23 Apr 2018 17:52:50 -0400 Subject: [PATCH 25/42] Added catalog functions --- src/catalog/layout_catalog.cpp | 18 +++--- src/catalog/table_catalog.cpp | 94 +++++++++++++++++++++++++++- src/include/catalog/layout_catalog.h | 3 +- src/include/catalog/table_catalog.h | 8 ++- 4 files changed, 106 insertions(+), 17 deletions(-) diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 301e55715aa..dbf1e7b429a 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -56,7 +56,7 @@ LayoutCatalog::LayoutCatalog(storage::Database *pg_catalog, LayoutCatalog::~LayoutCatalog() {} -std::unique_ptr ColumnCatalog::InitializeSchema() { +std::unique_ptr LayoutCatalog::InitializeSchema() { const std::string primary_key_constraint_name = "primary_key"; const std::string not_null_constraint_name = "not_null"; @@ -108,10 +108,10 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(), nullptr); - tuple->SetValue(ColumnId::TABLE_OID, val0); - tuple->SetValue(ColumnId::LAYOUT_OID, val1); - tuple->SetValue(ColumnId::NUM_COLUMNS, val2); - tuple->SetValue(ColumnId::COLUMN_MAP, val3); + tuple->SetValue(ColumnId::TABLE_OID, val0, pool); + tuple->SetValue(ColumnId::LAYOUT_OID, val1, pool); + tuple->SetValue(ColumnId::NUM_COLUMNS, val2, pool); + tuple->SetValue(ColumnId::COLUMN_MAP, val3, pool); // Insert the tuple return InsertTuple(std::move(tuple), txn); @@ -135,8 +135,8 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, } const std::unordered_map> -LayoutCatalog::GetLayouts(oid_t table_oid, oid_t layout_id, - concurrency::TransactionContext *txn) { +LayoutCatalog::GetLayouts(oid_t table_oid, + concurrency::TransactionContext *txn) { // Try to find the layouts in the cache auto table_object = TableCatalog::GetInstance()->GetTableObject(table_oid, txn); @@ -175,9 +175,5 @@ LayoutCatalog::GetLayouts(oid_t table_oid, oid_t layout_id, return table_object->GetLayouts(); } - - - - } // namespace catalog } // namespace peloton \ No newline at end of file diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 34ef723e366..0830ba94599 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -18,6 +18,7 @@ #include "catalog/column_catalog.h" #include "catalog/database_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/system_catalogs.h" #include "concurrency/transaction_context.h" #include "storage/data_table.h" @@ -319,8 +320,8 @@ std::shared_ptr TableCatalogObject::GetColumnObject( TableCatalog::TableCatalog( storage::Database *database, UNUSED_ATTRIBUTE type::AbstractPool *pool, UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(TABLE_CATALOG_OID, TABLE_CATALOG_NAME, - InitializeSchema().release(), database) { + : AbstractCatalog(TABLE_CATALOG_OID, TABLE_CATALOG_NAME, + InitializeSchema().release(), database) { // Add indexes for pg_namespace AddIndex({0}, TABLE_CATALOG_PKEY_OID, TABLE_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); @@ -330,6 +331,95 @@ TableCatalog::TableCatalog( IndexConstraintType::DEFAULT); } +/* @brief Insert layout object into the cache. + * @param layout Layout object to be inserted + * @return false if layout already exists in cache + */ +bool TableCatalogObject::InsertLayout(std::shared_ptr layout) { + + // Invalid object + if (layout == nullptr) { + return false; + } + + oid_t layout_id = layout->GetLayoutId(); + // layout is already present in the cache. + if (layout_objects_.find(layout_id) != layout_objects_.end()) { + LOG_DEBUG("Layout %u already exists in cache!", layout_id); + return false; + } + + valid_layout_objects_ = true; + layout_objects_.insert(std::make_pair(layout_id, layout)); + return true; +} + +/* + * @brief evict all layout objects from cache + */ +void TableCatalogObject::EvictAllLayouts() { + layout_objects_.clear(); + valid_layout_objects_ = false; +} + +/* @brief Get all layout objects of this table. + * Add it to the cache if necessary. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Map from layout_oid to cached layout object. + */ +std::unordered_map> +TableCatalogObject::GetLayouts(bool cached_only) { + if (!valid_layout_objects_ && !cached_only) { + // get column catalog objects from pg_column + LayoutCatalog::GetInstance()->GetLayouts(table_oid, txn); + valid_column_objects = true; + } + return layout_objects_; +} + +/* @brief get the layout object of the given layout_id. + * @param layout_id The id of the layout to be fetched. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Layout object of corresponding to the layout_id if present. + */ +std::shared_ptr +TableCatalogObject::GetLayout(oid_t layout_id, bool cached_entry) { + // fetch layout objects in case we have not + GetLayouts(cached_entry); + auto it = layout_objects_.find(layout_id); + if (it != layout_objects_.end()) { + return it->second; + } + return nullptr; +} + +/* @brief Evict layout from the cache. + * @param layout_id Id of the layout to be deleted. + * @return true if layout_id is found and evicted; false if not found. + */ +bool TableCatalogObject::EvictLayout(oid_t layout_id) { + if (!valid_layout_objects_) return false; + + // find layout from the cache + auto it = layout_objects_.find(layout_id); + if (it == layout_objects_.end()) { + return false; // layout_id not found in cache + } + + auto layout = it->second; + PELOTON_ASSERT(layout); + layout_objects_.erase(it); + return true; +} + +TableCatalog *TableCatalog::GetInstance(storage::Database *pg_catalog, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + static TableCatalog table_catalog{pg_catalog, pool, txn}; + return &table_catalog; +} + + TableCatalog::~TableCatalog() {} /*@brief private function for initialize schema of pg_table diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index f8f24677e9e..340b2934297 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -63,8 +63,7 @@ class LayoutCatalog : public AbstractCatalog { // Read Related API //===--------------------------------------------------------------------===// const std::unordered_map> - GetLayouts(oid_t table_oid, oid_t layout_id, - concurrency::TransactionContext *txn); + GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); private: diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index fa8b8bb6606..ebe47e1537b 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -34,9 +34,13 @@ #include "catalog/abstract_catalog.h" #include "executor/logical_tile.h" -#include "storage/layout.h" namespace peloton { + +namespace storage { +class Layout; +} + namespace catalog { class IndexCatalogObject; @@ -76,7 +80,7 @@ class TableCatalogObject { const std::string &column_name, bool cached_only = false); // Evict all layouts from the cache - void EvictAllLayoutObjects(); + void EvictAllLayouts(); // Get layouts std::unordered_map> From 6b36654a48c04fa39d5504ae4895563f152ca074 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 23 Apr 2018 20:26:27 -0400 Subject: [PATCH 26/42] Added DeleteLayouts to delete all layouts of a table_oid --- src/catalog/layout_catalog.cpp | 14 ++++++++++++++ src/include/catalog/layout_catalog.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index dbf1e7b429a..11d05c8deec 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -134,6 +134,20 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, return DeleteWithIndexScan(index_offset, values, txn); } +bool LayoutCatalog::DeleteLayouts(oid_t table_oid, + concurrency::TransactionContext *txn) { + oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid + std::vector values; + values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); + + // delete layouts from cache + auto table_object = + TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + table_object->EvictAllLayouts(); + + return DeleteWithIndexScan(index_offset, values, txn); +} + const std::unordered_map> LayoutCatalog::GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn) { diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 340b2934297..d93e976fd9e 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -59,6 +59,8 @@ class LayoutCatalog : public AbstractCatalog { bool DeleteLayout(oid_t table_oid, oid_t layout_id, concurrency::TransactionContext *txn); + + bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// From 1965813d50aee240d5f17700862f68abb10c2187 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 23 Apr 2018 21:03:00 -0400 Subject: [PATCH 27/42] LayoutTunerTest fixed --- src/catalog/catalog.cpp | 3 +- src/include/storage/data_table.h | 4 +- src/include/tuning/layout_tuner.h | 15 +++-- src/storage/data_table.cpp | 32 +++++----- src/tuning/layout_tuner.cpp | 59 ++++++------------- test/executor/testing_executor_util.cpp | 23 ++++++++ test/include/executor/testing_executor_util.h | 4 ++ test/tuning/layout_tuner_test.cpp | 19 +++--- 8 files changed, 82 insertions(+), 77 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index f056de0b0d3..521cee5105e 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -18,6 +18,7 @@ #include "catalog/index_catalog.h" #include "catalog/index_metrics_catalog.h" #include "catalog/language_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/proc_catalog.h" #include "catalog/query_history_catalog.h" #include "catalog/query_metrics_catalog.h" @@ -701,7 +702,6 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, auto table_object = database_object->GetTableObject(table_oid); auto index_objects = table_object->GetIndexObjects(); LOG_TRACE("dropping #%d indexes", (int)index_objects.size()); - // delete trigger and records in pg_trigger auto pg_trigger = catalog_map_[database_object->GetDatabaseOid()]->GetTriggerCatalog(); @@ -724,7 +724,6 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, auto pg_table = catalog_map_[database_object->GetDatabaseOid()]->GetTableCatalog(); pg_table->DeleteTable(table_oid, txn); - database->GetTableWithOid(table_oid); txn->RecordDrop(database_oid, table_oid, INVALID_OID); diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 268a7c0c160..207fa0394c0 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -248,7 +248,9 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); - void SetDefaultLayout(const column_map_type &column_map); + bool SetDefaultLayout(const column_map_type &column_map, + type::AbstractPool *pool, + concurrency::TransactionContext *txn); const Layout& GetDefaultLayout() const; diff --git a/src/include/tuning/layout_tuner.h b/src/include/tuning/layout_tuner.h index bf17e9cd933..c3b59046be3 100644 --- a/src/include/tuning/layout_tuner.h +++ b/src/include/tuning/layout_tuner.h @@ -27,6 +27,10 @@ namespace storage { class DataTable; } +namespace type { +class AbstractPool; +} + namespace tuning { //===--------------------------------------------------------------------===// @@ -81,14 +85,6 @@ class LayoutTuner { */ void ClearTables(); - /** - * @brief Gets the column map information. - * - * @param[in] column_map The column map - * - * @return The column map information. - */ - std::string GetColumnMapInfo(const column_map_type &column_map); protected: /** @@ -142,6 +138,9 @@ class LayoutTuner { /** Desired layout tile count */ oid_t tile_count = 2; + /** Pool to add new layouts in the pg_layout table */ + std::unique_ptr pool_; + }; } // namespace indextuner diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 3e5837a6afb..12cd77955d8 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -16,6 +16,8 @@ #include "catalog/catalog.h" #include "catalog/foreign_key.h" #include "catalog/system_catalogs.h" +#include "catalog/layout_catalog.h" +#include "catalog/table_catalog.h" #include "common/container_tuple.h" #include "common/exception.h" #include "common/logger.h" @@ -29,7 +31,6 @@ #include "storage/abstract_table.h" #include "storage/data_table.h" #include "storage/database.h" -#include "storage/layout.h" #include "storage/storage_manager.h" #include "storage/tile.h" #include "storage/tile_group.h" @@ -68,19 +69,6 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, tuples_per_tilegroup_(tuples_per_tilegroup), adapt_table_(adapt_table), trigger_list_(new trigger::TriggerList()) { - // Init default partition - if (layout_type == LayoutType::ROW) { - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - default_partition_[col_itr] = std::make_pair(0, col_itr); - } - } else if (layout_type == LayoutType::COLUMN) { - auto col_count = schema->GetColumnCount(); - for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - default_partition_[col_itr] = std::make_pair(col_itr, 0); - } - } - if (is_catalog == true) { active_tilegroup_count_ = 1; active_indirection_array_count_ = 1; @@ -1379,10 +1367,18 @@ void DataTable::ClearIndexSamples() { } } -void DataTable::SetDefaultLayout(const column_map_type &column_map) { - // TODO Pooja: Generate layout_oid_ and add it to the catalog - default_layout_ = std::shared_ptr( - new const Layout(column_map)); +bool DataTable::SetDefaultLayout(const column_map_type &column_map, + type::AbstractPool *pool, + concurrency::TransactionContext *txn) { + oid_t layout_id = GetNextLayoutOid(); + auto new_layout = std::shared_ptr( + new const Layout(column_map, layout_id)); + auto layout_catalog = catalog::LayoutCatalog::GetInstance(); + bool result = layout_catalog->InsertLayout(table_oid, new_layout, pool, txn); + if (result) { + default_layout_ = new_layout; + } + return result; } const Layout& DataTable::GetDefaultLayout() const { diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index b189ecb85cd..cc344e69da7 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -19,7 +19,9 @@ #include "catalog/schema.h" #include "common/logger.h" #include "common/timer.h" +#include "concurrency/transaction_manager_factory.h" #include "storage/data_table.h" +#include "type/ephemeral_pool.h" namespace peloton { namespace tuning { @@ -30,7 +32,7 @@ LayoutTuner& LayoutTuner::GetInstance() { } LayoutTuner::LayoutTuner() { - // Nothing to do here ! + pool_.reset(new type::EphemeralPool()); } LayoutTuner::~LayoutTuner() {} @@ -45,42 +47,6 @@ void LayoutTuner::Start() { LOG_INFO("Started layout tuner"); } -/** - * @brief Print information from column map, used to inspect layout generated - * from clusterer. - * - * @param column_map The column map to be printed. - */ -std::string LayoutTuner::GetColumnMapInfo(const column_map_type& column_map) { - std::stringstream ss; - std::map> tile_column_map; - - // Construct a tile_id => [col_ids] map - for (auto itr = column_map.begin(); itr != column_map.end(); itr++) { - oid_t col_id = itr->first; - oid_t tile_id = itr->second.first; - - if (tile_column_map.find(tile_id) == tile_column_map.end()) { - tile_column_map[tile_id] = {}; - } - - tile_column_map[tile_id].push_back(col_id); - } - - // Construct a string from tile_col_map - for (auto itr = tile_column_map.begin(); itr != tile_column_map.end(); - itr++) { - oid_t tile_id = itr->first; - ss << tile_id << ": "; - for (oid_t col_id : itr->second) { - ss << col_id << " "; - } - ss << " :: "; - } - - return ss.str(); -} - Sample GetClustererSample(const Sample& sample, oid_t column_count) { // Copy over the sample @@ -142,12 +108,23 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { table->ClearLayoutSamples(); // Desired number of tiles - auto layout = clusterer.GetPartitioning(tile_count); - - LOG_TRACE("%s", GetColumnMapInfo(layout).c_str()); + auto column_map = clusterer.GetPartitioning(tile_count); // Update table layout - table->SetDefaultLayout(layout); + // Since we need the layout to be updated in the catalog, + // Start a transaction before updating the default layout. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto *txn = txn_manager.BeginTransaction(); + bool result = table->SetDefaultLayout(column_map, pool_.get(), txn); + if (!result) { + txn_manager.AbortTransaction(txn); + LOG_DEBUG("Layout Update to failed."); + return; + } + txn_manager.CommitTransaction(txn); + + UNUSED_ATTRIBUTE auto layout = table->GetDefaultLayout(); + LOG_TRACE("Updated Layout: %s", layout.GetInfo().c_str()); } void LayoutTuner::Tune() { diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 65ee34ed272..5d63253b9ca 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -419,6 +419,29 @@ storage::DataTable *TestingExecutorUtil::CreateTable( return table; } +storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( + int tuples_per_tilegroup_count, std::string &db_name) { + auto table_schema = std::unique_ptr(new catalog::Schema( + {GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), GetColumnInfo(3)})); + std::string table_name("test_table"); + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + // Insert table in catalog + catalog->CreateTable(db_name, table_name, std::move(table_schema), + txn, is_catalog, tuples_per_tilegroup_count); + txn_manager.EndTransaction(txn); + + txn = txn_manager.BeginTransaction(); + auto test_db = catalog->GetDatabaseWithName(db_name, txn); + txn_manager.EndTransaction(txn); + + auto table = test_db->GetTableWithName(table_name); + return table; +} + /** * @brief Convenience method to create table for test. * diff --git a/test/include/executor/testing_executor_util.h b/test/include/executor/testing_executor_util.h index 1c1ff85aea9..abb8e20ab9a 100644 --- a/test/include/executor/testing_executor_util.h +++ b/test/include/executor/testing_executor_util.h @@ -75,6 +75,10 @@ class TestingExecutorUtil { int tuples_per_tilegroup_count = TESTS_TUPLES_PER_TILEGROUP, bool indexes = true, oid_t table_oid = INVALID_OID); + /** @brief Creates a basic table and adds its entry to the catalog */ + static storage::DataTable *CreateTableUpdateCatalog( + int tuples_per_tilegroup_count, std::string &db_name); + /** @brief Creates a basic table with allocated and populated tuples */ static storage::DataTable *CreateAndPopulateTable(); diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index b550aa52957..8efc83d9d19 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -6,7 +6,7 @@ // // Identification: test/tuning/layout_tuner_test.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Database Group +// Copyright (c) 2015-16, Carnegie Mellon University Databa e Group // //===----------------------------------------------------------------------===// @@ -38,14 +38,17 @@ TEST_F(LayoutTunerTests, BasicTest) { const int tuple_count = TESTS_TUPLES_PER_TILEGROUP; + std::string db_name = "test_db"; + TestingExecutorUtil::InitializeDatabase(db_name); + + auto data_table = TestingExecutorUtil::CreateTableUpdateCatalog(tuple_count, db_name); + + // Create a table and populate it auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - std::unique_ptr data_table( - TestingExecutorUtil::CreateTable(tuple_count, false)); - TestingExecutorUtil::PopulateTable(data_table.get(), tuple_count, false, - false, - true, txn); + TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, + false, true, txn); txn_manager.CommitTransaction(txn); // Check column count @@ -56,7 +59,7 @@ TEST_F(LayoutTunerTests, BasicTest) { tuning::LayoutTuner &layout_tuner = tuning::LayoutTuner::GetInstance(); // Attach table to index tuner - layout_tuner.AddTable(data_table.get()); + layout_tuner.AddTable(data_table); // Check old default tile group layout auto old_default_layout = data_table->GetDefaultLayout(); @@ -121,6 +124,8 @@ TEST_F(LayoutTunerTests, BasicTest) { EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(1),0); EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2),0); EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3),1); + + TestingExecutorUtil::DeleteDatabase(db_name); } } // namespace test From 9489f5a0ed33ff12e4cfec4e42f0c0e5444c420a Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 24 Apr 2018 00:39:05 -0400 Subject: [PATCH 28/42] Fix build failures --- src/catalog/schema.cpp | 2 +- src/include/catalog/schema.h | 2 +- src/storage/data_table.cpp | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index df71b1315a9..4e58d9faf1e 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -218,7 +218,7 @@ Schema *Schema::AppendSchema(const Schema *first, std::vector &first_set, } // Append given schemas. -Schema *Schema::AppendSchemaList(std::vector &schema_list) { +Schema *Schema::AppendSchemaList(std::vector &schema_list) { // All we do here is convert vector to vector. // This is a convenience function. std::vector schema_ptr_list; diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index 97de588d62c..705e7f1e2c4 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -69,7 +69,7 @@ class Schema : public Printable { const Schema *second, std::vector &second_set); // Append given schemas. - static Schema *AppendSchemaList(std::vector &schema_list); + static Schema *AppendSchemaList(std::vector &schema_list); // Append given schemas. static Schema *AppendSchemaPtrList(const std::vector &schema_list); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 12cd77955d8..d7d8dda3a6d 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1241,8 +1241,8 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, // Check that both tile groups have the same schema // Currently done by checking that the number of columns are equal // TODO Pooja: Handle schena equality for multiple schema versions. - auto new_column_count = new_layout.GetColumnCount(); - auto orig_column_count = orig_layout.GetColumnCount(); + UNUSED_ATTRIBUTE auto new_column_count = new_layout.GetColumnCount(); + UNUSED_ATTRIBUTE auto orig_column_count = orig_layout.GetColumnCount(); PELOTON_ASSERT(new_column_count == orig_column_count); oid_t orig_tile_offset, orig_tile_column_offset; From 7408c2001bf65dc5edc8c102ea0d0cacdc2f8e63 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 25 Apr 2018 23:50:07 -0400 Subject: [PATCH 29/42] Access LayoutCatalog via the global Catalog object --- src/catalog/catalog.cpp | 104 +++++++++++++++++++++++++++ src/catalog/layout_catalog.cpp | 2 +- src/catalog/table_catalog.cpp | 2 +- src/include/catalog/catalog.h | 17 +++++ src/include/catalog/layout_catalog.h | 4 +- src/include/storage/data_table.h | 13 ++-- src/include/storage/layout.h | 4 +- src/include/tuning/layout_tuner.h | 7 -- src/storage/data_table.cpp | 14 ---- src/storage/layout.cpp | 22 +++--- src/tuning/layout_tuner.cpp | 15 ++-- 11 files changed, 156 insertions(+), 48 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 521cee5105e..fbf85a06eb5 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -557,6 +557,76 @@ ResultType Catalog::CreateIndex( return ResultType::SUCCESS; } +/* + * + // Create a new layout + ResultType CreateLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn); + // Create a new layout and set it as the default for the table + ResultType CreateDefaultLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn); + */ + +/* + * @brief create a new layout for a table + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ +std::shared_ptr +Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn) { + + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + + oid_t layout_oid = table->GetNextLayoutOid(); + // Ensure that the new layout + PELOTON_ASSERT(layout_oid < INVALID_OID); + auto new_layout = std::shared_ptr( + new const storage::Layout(column_map, layout_oid)); + auto layout_catalog = catalog::LayoutCatalog::GetInstance(); + bool result = layout_catalog->InsertLayout(table_oid, new_layout, + pool_.get(), txn); + if (!result) { + LOG_DEBUG("Failed to create a new layout for table %u", table_oid); + return nullptr; + } + return new_layout; +} + +/* + * @brief create a new layout for a table and make it the deafult if + * if the creating is successsful. + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ +std::shared_ptr +Catalog::CreateDefaultLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn) { + auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn); + // If the layout creation was successful, set it as the default + if (new_layout != nullptr) { + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + table->SetDefaultLayout(new_layout); + } + return new_layout; +} + //===----------------------------------------------------------------------===// // DROP FUNCTIONS //===----------------------------------------------------------------------===// @@ -764,6 +834,40 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, return ResultType::SUCCESS; } + +/*@brief Drop layout + * tile_groups + * @param database_oid the database to which the table belongs + * @param table_oid the table to which the layout belongs + * @param layout_oid the layout to be dropped + * @param txn TransactionContext + * @return TransactionContext ResultType(SUCCESS or FAILURE) + */ +ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, + oid_t layout_oid, + concurrency::TransactionContext *txn) { + // Check if the default_layout of the table is the same. + // If true reset it to a row store. + auto storage_manager = storage::StorageManager::GetInstance(); + auto database = storage_manager->GetDatabaseWithOid(database_oid); + auto table = database->GetTableWithOid(table_oid); + auto default_layout = table->GetDefaultLayout(); + + if (default_layout.GetOid() == layout_oid) { + table->ResetDefaultLayout(); + } + + auto layout_catalog = LayoutCatalog::GetInstance(); + if (!layout_catalog->DeleteLayout(table_oid, layout_oid, txn)) { + auto layout = table->GetDefaultLayout(); + LOG_DEBUG("Layout delete failed. Default layout id: %u", + layout.GetOid()); + return ResultType::FAILURE; + } + + return ResultType::SUCCESS; +} + //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 11d05c8deec..77ba987795e 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -103,7 +103,7 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, new storage::Tuple(catalog_table_->GetSchema(), true)); auto val0 = type::ValueFactory::GetIntegerValue(table_oid); - auto val1 = type::ValueFactory::GetIntegerValue(layout->GetLayoutId()); + auto val1 = type::ValueFactory::GetIntegerValue(layout->GetOid()); auto val2 = type::ValueFactory::GetIntegerValue(layout->GetColumnCount()); auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(), nullptr); diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 0830ba94599..387d1c7aa5b 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -342,7 +342,7 @@ bool TableCatalogObject::InsertLayout(std::shared_ptr lay return false; } - oid_t layout_id = layout->GetLayoutId(); + oid_t layout_id = layout->GetOid(); // layout is already present in the cache. if (layout_objects_.find(layout_id) != layout_objects_.end()) { LOG_DEBUG("Layout %u already exists in cache!", layout_id); diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 90905e7066e..ad744aa4251 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -38,6 +38,7 @@ class Index; namespace storage { class Database; class DataTable; +class Layout; class TableFactory; class Tuple; } // namespace storage @@ -119,6 +120,18 @@ class Catalog { concurrency::TransactionContext *txn, bool is_catalog = false); + // Create a new layout + std::shared_ptr + CreateLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn); + + // Create a new layout and set it as the default for the table + std::shared_ptr + CreateDefaultLayout(oid_t database_oid, oid_t table_oid, + const column_map_type &column_map, + concurrency::TransactionContext *txn); + //===--------------------------------------------------------------------===// // DROP FUNCTIONS //===--------------------------------------------------------------------===// @@ -146,6 +159,10 @@ class Catalog { // Drop an index, using its index_oid ResultType DropIndex(oid_t database_oid, oid_t index_oid, concurrency::TransactionContext *txn); + + // Delete a layout using its database_oid, table_oid and layout_oid + ResultType DropLayout(oid_t database_oid, oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// // GET WITH NAME - CHECK FROM CATALOG TABLES, USING TRANSACTION //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index d93e976fd9e..ac724d78d88 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -61,13 +61,15 @@ class LayoutCatalog : public AbstractCatalog { concurrency::TransactionContext *txn); bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + +private: + //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// const std::unordered_map> GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); -private: LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, concurrency::TransactionContext *txn); diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 207fa0394c0..525265388bc 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -40,6 +40,7 @@ class Sample; namespace catalog { class ForeignKey; +class Catalog; } // namespace catalog namespace index { @@ -77,6 +78,7 @@ class DataTable : public AbstractTable { friend class TileGroup; friend class TileGroupFactory; friend class TableFactory; + friend class catalog::Catalog; friend class logging::LogManager; DataTable() = delete; @@ -190,7 +192,6 @@ class DataTable : public AbstractTable { const std::vector> &GetIndexColumns() const { return indexes_columns_; } - //===--------------------------------------------------------------------===// // FOREIGN KEYS //===--------------------------------------------------------------------===// @@ -248,10 +249,14 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); - bool SetDefaultLayout(const column_map_type &column_map, - type::AbstractPool *pool, - concurrency::TransactionContext *txn); + inline void SetDefaultLayout(std::shared_ptr new_layout) { + default_layout_ = new_layout; + } + inline void ResetDefaultLayout() { + default_layout_ = std::shared_ptr( + new const Layout(schema->GetColumnCount())); + } const Layout& GetDefaultLayout() const; //===--------------------------------------------------------------------===// diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index ec455ca5d54..e37a5fb6948 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -43,7 +43,7 @@ class Layout : public Printable { // Check whether the Layout is ColumnStore. inline bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } - oid_t GetLayoutId() const { return layout_id_; } + oid_t GetOid() const { return layout_oid_; } // Sets the tile id and column id w.r.t that tile corresponding to // the specified tile group column id. @@ -78,7 +78,7 @@ class Layout : public Printable { private: // Layout Id of the tile - oid_t layout_id_; + oid_t layout_oid_; // Number of columns in the layout oid_t num_columns_; diff --git a/src/include/tuning/layout_tuner.h b/src/include/tuning/layout_tuner.h index c3b59046be3..5bcba99b6d7 100644 --- a/src/include/tuning/layout_tuner.h +++ b/src/include/tuning/layout_tuner.h @@ -27,10 +27,6 @@ namespace storage { class DataTable; } -namespace type { -class AbstractPool; -} - namespace tuning { //===--------------------------------------------------------------------===// @@ -138,9 +134,6 @@ class LayoutTuner { /** Desired layout tile count */ oid_t tile_count = 2; - /** Pool to add new layouts in the pg_layout table */ - std::unique_ptr pool_; - }; } // namespace indextuner diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index d7d8dda3a6d..cffa7900429 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1367,20 +1367,6 @@ void DataTable::ClearIndexSamples() { } } -bool DataTable::SetDefaultLayout(const column_map_type &column_map, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { - oid_t layout_id = GetNextLayoutOid(); - auto new_layout = std::shared_ptr( - new const Layout(column_map, layout_id)); - auto layout_catalog = catalog::LayoutCatalog::GetInstance(); - bool result = layout_catalog->InsertLayout(table_oid, new_layout, pool, txn); - if (result) { - default_layout_ = new_layout; - } - return result; -} - const Layout& DataTable::GetDefaultLayout() const { return *default_layout_; } diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 706b38af403..0a7915e9c89 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -46,16 +46,16 @@ Layout::Layout(const column_map_type& column_map) // If a table has one column, it would be LayoutType::Row if (row_layout) { layout_type_ = LayoutType::ROW; - layout_id_ = ROW_STORE_OID; + layout_oid_ = ROW_STORE_OID; } else if (column_layout) { layout_type_ = LayoutType::COLUMN; - layout_id_ = COLUMN_STORE_OID; + layout_oid_ = COLUMN_STORE_OID; } else { - // layout_id_ is set to INVALID_OID, indicating that this + // layout_oid_ is set to INVALID_OID, indicating that this // layout is not stored in the catalog and thus not persistent. // To be used only in TempTable or Tests. layout_type_ = LayoutType::HYBRID; - layout_id_ = INVALID_OID; + layout_oid_ = INVALID_OID; } if (layout_type_ != LayoutType::HYBRID) { @@ -64,15 +64,15 @@ Layout::Layout(const column_map_type& column_map) } } -// Constructor for Layout class with predefined layout_id +// Constructor for Layout class with predefined layout_oid Layout::Layout(const column_map_type &column_map, oid_t layout_id) - : layout_id_(layout_id), + : layout_oid_(layout_id), num_columns_(column_map.size()), column_layout_(column_map) { - if (layout_id_ == ROW_STORE_OID) { + if (layout_oid_ == ROW_STORE_OID) { layout_type_ = LayoutType::ROW; - } else if (layout_id == COLUMN_STORE_OID) { + } else if (layout_oid_ == COLUMN_STORE_OID) { layout_type_ = LayoutType::COLUMN; } else { layout_type_ = LayoutType::HYBRID; @@ -124,7 +124,7 @@ double Layout::GetLayoutDifference(const storage::Layout &other) const { // Check the schema before invoking this function. PELOTON_ASSERT(this->num_columns_ == other.num_columns_); - if ((this->layout_id_ != other.layout_id_)) { + if ((this->layout_oid_ != other.layout_oid_)) { for (oid_t col_itr = 0; col_itr < num_columns_; col_itr++) { @@ -289,7 +289,7 @@ std::string Layout::GetColumnMapInfo() const { const std::string Layout::GetInfo() const { std::ostringstream os; - os << peloton::GETINFO_DOUBLE_STAR << " Layout[#" << layout_id_ << "] " + os << peloton::GETINFO_DOUBLE_STAR << " Layout[#" << layout_oid_ << "] " << peloton::GETINFO_DOUBLE_STAR << std::endl; os << "Number of columns[" << num_columns_ << "] " << std::endl; os << "LayoutType[" << LayoutTypeToString(layout_type_) << std::endl; @@ -307,7 +307,7 @@ bool operator==(const Layout& lhs, const Layout& rhs) { } // Check the equality of layout_oid_ - if (lhs.GetLayoutId() != rhs.GetLayoutId()) { + if (lhs.GetOid() != rhs.GetOid()) { return false; } // Check the equality of column_count_ diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index cc344e69da7..9f3fc2b070f 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -16,12 +16,12 @@ #include #include +#include "catalog/catalog.h" #include "catalog/schema.h" #include "common/logger.h" #include "common/timer.h" #include "concurrency/transaction_manager_factory.h" #include "storage/data_table.h" -#include "type/ephemeral_pool.h" namespace peloton { namespace tuning { @@ -31,9 +31,7 @@ LayoutTuner& LayoutTuner::GetInstance() { return layout_tuner; } -LayoutTuner::LayoutTuner() { - pool_.reset(new type::EphemeralPool()); -} +LayoutTuner::LayoutTuner() {} LayoutTuner::~LayoutTuner() {} @@ -110,20 +108,23 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { // Desired number of tiles auto column_map = clusterer.GetPartitioning(tile_count); + auto table_oid = table->GetOid(); + auto database_oid = table->GetDatabaseOid(); // Update table layout // Since we need the layout to be updated in the catalog, // Start a transaction before updating the default layout. auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); - bool result = table->SetDefaultLayout(column_map, pool_.get(), txn); - if (!result) { + auto catalog = catalog::Catalog::GetInstance(); + if (catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn) + == nullptr) { txn_manager.AbortTransaction(txn); LOG_DEBUG("Layout Update to failed."); return; } txn_manager.CommitTransaction(txn); - UNUSED_ATTRIBUTE auto layout = table->GetDefaultLayout(); + UNUSED_ATTRIBUTE auto layout = table-> GetDefaultLayout(); LOG_TRACE("Updated Layout: %s", layout.GetInfo().c_str()); } From 4aaee91540f853110dbb78e2b0098f49e52dfebe Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Thu, 26 Apr 2018 08:35:24 -0400 Subject: [PATCH 30/42] Added Multi layout scan test --- src/include/storage/data_table.h | 6 +- test/codegen/table_scan_translator_test.cpp | 182 +++++++++++++++++++- 2 files changed, 184 insertions(+), 4 deletions(-) diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 525265388bc..f635a9180b0 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -250,12 +250,14 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); inline void SetDefaultLayout(std::shared_ptr new_layout) { + PELOTON_ASSERT(new_layout->GetColumnCount() == schema->GetColumnCount()); default_layout_ = new_layout; } - inline void ResetDefaultLayout() { + inline void ResetDefaultLayout(LayoutType type = LayoutType::ROW) { + PELOTON_ASSERT((type == LayoutType::ROW) || (type == LayoutType::COLUMN)); default_layout_ = std::shared_ptr( - new const Layout(schema->GetColumnCount())); + new const Layout(schema->GetColumnCount(), type)); } const Layout& GetDefaultLayout() const; diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 19c495827dc..39ee4ab7dd5 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -53,13 +53,15 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { auto column = - catalog::Column(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), "FIELD" + std::to_string(col_itr), is_inlined); columns.push_back(column); } - std::unique_ptr table_schema = std::unique_ptr(new catalog::Schema(columns)); + std::unique_ptr table_schema = + std::unique_ptr(new catalog::Schema(columns)); std::string table_name("TEST_TABLE"); ///////////////////////////////////////////////////////// @@ -715,5 +717,181 @@ TEST_F(TableScanTranslatorTest, ScanColumnLayout) { ExecuteTileGroupTest(LayoutType::COLUMN); } +TEST_F(TableScanTranslatorTest, MultiLayoutScan) { + // + // Creates a table with LayoutType::ROW + // Sets the default_layout_ as LayoutType::COLUMN + // Inserts tuples + // Sets the default_layout_ as LayoutType::HYBRID + // Inserts some more tuples + // invokes the TableScanTranslator + // + + const int tuples_per_tilegroup= 100; + const int col_count = 6; + const bool is_inlined = true; + oid_t tuple_count = 100; + + + ///////////////////////////////////////////////////////// + // Define the schema. + ///////////////////////////////////////////////////////// + + std::vector columns; + + for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { + auto column = + catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); + + columns.push_back(column); + } + + std::unique_ptr table_schema = + std::unique_ptr(new catalog::Schema(columns)); + std::string table_name("MULTI_LAYOUT_TABLE"); + + ///////////////////////////////////////////////////////// + // Create table. + ///////////////////////////////////////////////////////// + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + const bool allocate = true; + auto txn = txn_manager.BeginTransaction(); + + // Insert table in catalog + catalog->CreateTable(test_db_name, table_name, std::move(table_schema), + txn, is_catalog, tuples_per_tilegroup, LayoutType::ROW); + txn_manager.EndTransaction(txn); + + auto table = GetDatabase().GetTableWithName(table_name); + + ///////////////////////////////////////////////////////// + // Reset default_layout_ to LayoutType::COLUMN + ///////////////////////////////////////////////////////// + table->ResetDefaultLayout(LayoutType::COLUMN); + + ///////////////////////////////////////////////////////// + // Load in 100 tuples + ///////////////////////////////////////////////////////// + txn = txn_manager.BeginTransaction(); + auto table_schema_ptr = table->GetSchema(); + auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); + + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Set default_layout_ to LayoutType::HYBRID + ///////////////////////////////////////////////////////// + // Populate column_map with HYBRID layout + column_map_type column_map; + column_map[0] = std::make_pair(0, 0); + column_map[1] = std::make_pair(1, 0); + column_map[2] = std::make_pair(1, 1); + column_map[3] = std::make_pair(2, 0); + column_map[4] = std::make_pair(2, 1); + column_map[5] = std::make_pair(2, 2); + + auto database_oid = table->GetDatabaseOid(); + auto table_oid = table->GetOid(); + + txn = txn_manager.BeginTransaction(); + auto layout = catalog->CreateDefaultLayout(database_oid, table_oid, + column_map, txn); + EXPECT_NE(nullptr, layout); + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Load in 200 tuples + ///////////////////////////////////////////////////////// + tuple_count = 200; + oid_t prev_tuple_count = 100; + txn = txn_manager.BeginTransaction(); + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id + prev_tuple_count; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); + + ///////////////////////////////////////////////////////// + // Do a seq scan on the table + ///////////////////////////////////////////////////////// + + // Reset Tuple Count + tuple_count = tuple_count + prev_tuple_count; + // Column ids to be scanned. + std::vector column_ids; + for(oid_t col_id = 0 ; col_id < col_count; col_id++) { + column_ids.push_back(col_id); + } + + + // Setup the scan plan node + planner::SeqScanPlan scan(table, nullptr, column_ids); + + // Do binding + planner::BindingContext context; + scan.PerformBinding(context); + + // Printing consumer + codegen::BufferingConsumer buffer{column_ids, context}; + + + // COMPILE and execute + CompileAndExecute(scan, buffer); + + // Check that we got all the results + const auto &results = buffer.GetOutputTuples(); + EXPECT_EQ(results.size(), tuple_count); + + for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { + auto &tuple = results[tuple_id]; + int tuple_id_value = tuple_id; + for (oid_t col_id = 0; col_id < col_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); + EXPECT_EQ(CmpBool::CmpTrue, tuple.GetValue(col_id).CompareEquals(value)); + } + } +} + } // namespace test } // namespace peloton From 429c9d076d688abd740966abf35f54d8a5635958 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 1 May 2018 10:48:32 -0400 Subject: [PATCH 31/42] Fixed tests after catalog refactor --- src/catalog/catalog.cpp | 18 +++++--- src/catalog/layout_catalog.cpp | 42 +++++++++---------- src/catalog/system_catalogs.cpp | 6 ++- src/catalog/table_catalog.cpp | 8 ---- src/include/catalog/catalog_defaults.h | 4 +- src/include/catalog/layout_catalog.h | 7 ++-- src/include/catalog/system_catalogs.h | 9 ++++ src/include/parser/sql_statement.h | 2 +- src/include/parser/table_ref.h | 2 +- src/optimizer/stats/tuple_samples_storage.cpp | 10 ++--- src/tuning/index_tuner.cpp | 2 +- test/binder/binder_test.cpp | 6 +-- test/catalog/catalog_test.cpp | 22 +++++----- test/catalog/constraints_test.cpp | 4 +- test/codegen/bloom_filter_test.cpp | 6 +-- test/codegen/table_scan_translator_test.cpp | 25 ++++++----- test/codegen/testing_codegen_util.cpp | 8 ++-- test/executor/create_index_test.cpp | 2 +- test/executor/create_test.cpp | 16 +++---- test/executor/delete_test.cpp | 4 +- test/executor/drop_test.cpp | 26 ++++++------ test/executor/insert_test.cpp | 4 +- test/executor/testing_executor_util.cpp | 13 +++--- test/executor/update_test.cpp | 4 +- .../catalog/testing_constraints_util.h | 4 +- test/optimizer/old_optimizer_test.cpp | 2 +- test/optimizer/selectivity_test.cpp | 4 +- test/optimizer/table_stats_collector_test.cpp | 4 +- test/optimizer/tuple_samples_storage_test.cpp | 2 +- test/planner/plan_util_test.cpp | 18 ++++---- test/planner/planner_test.cpp | 16 +++---- test/sql/drop_sql_test.cpp | 8 ++-- test/sql/optimizer_sql_test.cpp | 4 +- test/statistics/stats_test.cpp | 4 +- test/statistics/testing_stats_util.cpp | 2 +- test/trigger/trigger_test.cpp | 12 +++--- 36 files changed, 172 insertions(+), 158 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index fbf85a06eb5..5cb3bd4f0fb 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -162,7 +162,7 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, system_catalogs->GetSchemaCatalog()->InsertSchema( CATALOG_SCHEMA_OID, CATALOG_SCHEMA_NAME, pool_.get(), txn); system_catalogs->GetSchemaCatalog()->InsertSchema( - DEFUALT_SCHEMA_OID, DEFUALT_SCHEMA_NAME, pool_.get(), txn); + DEFUALT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn); // Insert catalog tables into pg_table // pg_database record is shared across different databases @@ -592,8 +592,11 @@ Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, PELOTON_ASSERT(layout_oid < INVALID_OID); auto new_layout = std::shared_ptr( new const storage::Layout(column_map, layout_oid)); - auto layout_catalog = catalog::LayoutCatalog::GetInstance(); - bool result = layout_catalog->InsertLayout(table_oid, new_layout, + + // Add the layout the pg_layout table + auto pg_layout = + catalog_map_[database_oid]->GetLayoutCatalog(); + bool result = pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn); if (!result) { LOG_DEBUG("Failed to create a new layout for table %u", table_oid); @@ -790,6 +793,11 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, catalog_map_[database_object->GetDatabaseOid()]->GetColumnCatalog(); pg_attribute->DeleteColumns(table_oid, txn); + // delete record in pg_layout + auto pg_layout = + catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); + pg_layout->DeleteLayouts(table_oid, txn); + // delete record in pg_table auto pg_table = catalog_map_[database_object->GetDatabaseOid()]->GetTableCatalog(); @@ -857,8 +865,8 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, table->ResetDefaultLayout(); } - auto layout_catalog = LayoutCatalog::GetInstance(); - if (!layout_catalog->DeleteLayout(table_oid, layout_oid, txn)) { + auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); + if (!pg_layout->DeleteLayout(table_oid, layout_oid, txn)) { auto layout = table->GetDefaultLayout(); LOG_DEBUG("Layout delete failed. Default layout id: %u", layout.GetOid()); diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 77ba987795e..52bf9dfee72 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -12,7 +12,8 @@ #include "catalog/layout_catalog.h" -#include "catalog/column_catalog.h" +#include "catalog/catalog.h" +#include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" #include "concurrency/transaction_context.h" #include "storage/data_table.h" @@ -28,9 +29,10 @@ LayoutCatalog *LayoutCatalog::GetInstance(storage::Database *pg_catalog, return &layout_catalog; } -LayoutCatalog::LayoutCatalog(storage::Database *pg_catalog, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) +LayoutCatalog::LayoutCatalog( + storage::Database *pg_catalog, + UNUSED_ATTRIBUTE type::AbstractPool *pool, + UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, InitializeSchema().release(), pg_catalog) { // Add indexes for pg_attribute @@ -39,19 +41,6 @@ LayoutCatalog::LayoutCatalog(storage::Database *pg_catalog, IndexConstraintType::PRIMARY_KEY); AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); - - // Insert columns into pg_attribute - ColumnCatalog *pg_attribute = - ColumnCatalog::GetInstance(pg_catalog, pool, txn); - - oid_t column_id = 0; - for (auto column : catalog_table_->GetSchema()->GetColumns()) { - pg_attribute->InsertColumn(LAYOUT_CATALOG_OID, column.GetName(), column_id, - column.GetOffset(), column.GetType(), - column.IsInlined(), column.GetConstraints(), - pool, txn); - column_id++; - } } LayoutCatalog::~LayoutCatalog() {} @@ -126,9 +115,12 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(layout_id).Copy()); + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + // delete column from cache - auto table_object = - TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + auto table_object = pg_table->GetTableObject(table_oid, txn); table_object->EvictLayout(layout_id); return DeleteWithIndexScan(index_offset, values, txn); @@ -141,8 +133,10 @@ bool LayoutCatalog::DeleteLayouts(oid_t table_oid, values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); // delete layouts from cache - auto table_object = - TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableObject(table_oid, txn); table_object->EvictAllLayouts(); return DeleteWithIndexScan(index_offset, values, txn); @@ -152,8 +146,10 @@ const std::unordered_map> LayoutCatalog::GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn) { // Try to find the layouts in the cache - auto table_object = - TableCatalog::GetInstance()->GetTableObject(table_oid, txn); + auto pg_table = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); + auto table_object = pg_table->GetTableObject(table_oid, txn); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); auto layout_objects = table_object->GetLayouts(true); if (layout_objects.size() != 0) { diff --git a/src/catalog/system_catalogs.cpp b/src/catalog/system_catalogs.cpp index b1371ddd379..3900c165f74 100644 --- a/src/catalog/system_catalogs.cpp +++ b/src/catalog/system_catalogs.cpp @@ -13,6 +13,7 @@ #include "catalog/system_catalogs.h" #include "catalog/column_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/table_catalog.h" #include "storage/data_table.h" #include "storage/database.h" @@ -38,6 +39,7 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, pg_namespace_ = new SchemaCatalog(database, pool, txn); pg_table_ = new TableCatalog(database, pool, txn); pg_index_ = new IndexCatalog(database, pool, txn); + pg_layout_ = new LayoutCatalog(database, pool, txn); // TODO: can we move this to BootstrapSystemCatalogs()? // insert column information into pg_attribute @@ -45,7 +47,8 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, {CATALOG_DATABASE_OID, DATABASE_CATALOG_OID}, {database_oid, TABLE_CATALOG_OID}, {database_oid, SCHEMA_CATALOG_OID}, - {database_oid, INDEX_CATALOG_OID}}; + {database_oid, INDEX_CATALOG_OID}, + {database_oid, LAYOUT_CATALOG_OID}}; for (int i = 0; i < (int)shared_tables.size(); i++) { oid_t column_id = 0; @@ -65,6 +68,7 @@ SystemCatalogs::SystemCatalogs(storage::Database *database, SystemCatalogs::~SystemCatalogs() { delete pg_index_; + delete pg_layout_; delete pg_table_; delete pg_attribute_; delete pg_namespace_; diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 387d1c7aa5b..3432f44c7e2 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -412,14 +412,6 @@ bool TableCatalogObject::EvictLayout(oid_t layout_id) { return true; } -TableCatalog *TableCatalog::GetInstance(storage::Database *pg_catalog, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { - static TableCatalog table_catalog{pg_catalog, pool, txn}; - return &table_catalog; -} - - TableCatalog::~TableCatalog() {} /*@brief private function for initialize schema of pg_table diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index c0e3b681098..f6343e85b8e 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -36,7 +36,7 @@ namespace catalog { // Local oids from START_OID = 0 to START_OID + OID_OFFSET are reserved #define OID_OFFSET 100 -#define CATALOG_TABLES_COUNT 8 +#define CATALOG_TABLES_COUNT 9 // Oid mask for each type #define DATABASE_OID_MASK (static_cast(catalog::CatalogType::DATABASE)) @@ -55,7 +55,7 @@ namespace catalog { #define CATALOG_SCHEMA_OID (0 | SCHEMA_OID_MASK) #define DEFUALT_SCHEMA_OID (1 | SCHEMA_OID_MASK) #define CATALOG_SCHEMA_NAME "pg_catalog" -#define DEFUALT_SCHEMA_NAME "public" +#define DEFAULT_SCHEMA_NAME "public" // Reserved pg_xxx table oid #define DATABASE_CATALOG_OID (0 | TABLE_OID_MASK) diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index ac724d78d88..70c8e226fdc 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -47,6 +47,9 @@ class LayoutCatalog : public AbstractCatalog { type::AbstractPool *pool = nullptr, concurrency::TransactionContext *txn = nullptr); + LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, + concurrency::TransactionContext *txn); + ~LayoutCatalog(); //===--------------------------------------------------------------------===// @@ -70,10 +73,6 @@ class LayoutCatalog : public AbstractCatalog { const std::unordered_map> GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); - - LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, - concurrency::TransactionContext *txn); - std::unique_ptr InitializeSchema(); enum ColumnId { diff --git a/src/include/catalog/system_catalogs.h b/src/include/catalog/system_catalogs.h index 7791c019097..9792d180f9d 100644 --- a/src/include/catalog/system_catalogs.h +++ b/src/include/catalog/system_catalogs.h @@ -34,6 +34,7 @@ class SchemaCatalog; class TableCatalog; class IndexCatalog; class ColumnCatalog; +class LayoutCatalog; class SystemCatalogs { public: @@ -79,6 +80,13 @@ class SystemCatalogs { return pg_index_; } + LayoutCatalog *GetLayoutCatalog() { + if (!pg_layout_) { + throw CatalogException("Layout catalog has not been initialized"); + } + return pg_layout_; + } + TriggerCatalog *GetTriggerCatalog() { if (!pg_trigger_) { throw CatalogException("Trigger catalog has not been initialized"); @@ -112,6 +120,7 @@ class SystemCatalogs { SchemaCatalog *pg_namespace_; TableCatalog *pg_table_; IndexCatalog *pg_index_; + LayoutCatalog *pg_layout_; TriggerCatalog *pg_trigger_; // ProcCatalog *pg_proc; diff --git a/src/include/parser/sql_statement.h b/src/include/parser/sql_statement.h index 012f4090cb9..8b6343f7ea1 100644 --- a/src/include/parser/sql_statement.h +++ b/src/include/parser/sql_statement.h @@ -73,7 +73,7 @@ class TableRefStatement : public SQLStatement { table_info_->database_name = default_database_name; // if schema name is not specified, then it's default value is "public" if (table_info_->schema_name.empty()) - table_info_->schema_name = DEFUALT_SCHEMA_NAME; + table_info_->schema_name = DEFAULT_SCHEMA_NAME; } virtual inline std::string GetTableName() const { diff --git a/src/include/parser/table_ref.h b/src/include/parser/table_ref.h index e3b4fab31d6..5788f382d52 100644 --- a/src/include/parser/table_ref.h +++ b/src/include/parser/table_ref.h @@ -76,7 +76,7 @@ struct TableRef { } if (table_info_->schema_name.empty()) { - table_info_->schema_name = DEFUALT_SCHEMA_NAME; + table_info_->schema_name = DEFAULT_SCHEMA_NAME; } } diff --git a/src/optimizer/stats/tuple_samples_storage.cpp b/src/optimizer/stats/tuple_samples_storage.cpp index 9ae85256758..edb41c2f820 100644 --- a/src/optimizer/stats/tuple_samples_storage.cpp +++ b/src/optimizer/stats/tuple_samples_storage.cpp @@ -66,11 +66,11 @@ void TupleSamplesStorage::AddSamplesTable( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); catalog->CreateTable(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), samples_table_name, + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, std::move(schema_ptr), txn, is_catalog); auto samples_table = catalog->GetTableWithName( - std::string(SAMPLES_DB_NAME), std::string(DEFUALT_SCHEMA_NAME), + std::string(SAMPLES_DB_NAME), std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); for (auto &tuple : sampled_tuples) { @@ -94,7 +94,7 @@ ResultType TupleSamplesStorage::DeleteSamplesTable( ResultType result = ResultType::FAILURE; try { result = catalog->DropTable(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); } catch (CatalogException &e) { // Samples table does not exist, no need to drop @@ -187,7 +187,7 @@ TupleSamplesStorage::GetTupleSamples(oid_t database_id, oid_t table_id) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); auto col_count = data_table->GetSchema()->GetColumnCount(); @@ -214,7 +214,7 @@ void TupleSamplesStorage::GetColumnSamples( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto data_table = catalog->GetTableWithName(std::string(SAMPLES_DB_NAME), - std::string(DEFUALT_SCHEMA_NAME), + std::string(DEFAULT_SCHEMA_NAME), samples_table_name, txn); std::vector column_ids({column_id}); diff --git a/src/tuning/index_tuner.cpp b/src/tuning/index_tuner.cpp index ca96907e605..cbd1bd57926 100644 --- a/src/tuning/index_tuner.cpp +++ b/src/tuning/index_tuner.cpp @@ -598,7 +598,7 @@ void IndexTuner::BootstrapTPCC(const std::string &path) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto table = catalog->GetTableWithName( - database_name, std::string(DEFUALT_SCHEMA_NAME), table_name, txn); + database_name, std::string(DEFAULT_SCHEMA_NAME), table_name, txn); txn_manager.CommitTransaction(txn); PELOTON_ASSERT(table != nullptr); for (auto &sample : samples) { diff --git a/test/binder/binder_test.cpp b/test/binder/binder_test.cpp index b82df1ec72a..b5b266c21cf 100644 --- a/test/binder/binder_test.cpp +++ b/test/binder/binder_test.cpp @@ -129,11 +129,11 @@ TEST_F(BinderCorrectnessTest, SelectStatementTest) { catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); oid_t tableA_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "a", txn) + DEFAULT_SCHEMA_NAME, "a", txn) ->GetOid(); oid_t tableB_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "b", txn) + DEFAULT_SCHEMA_NAME, "b", txn) ->GetOid(); txn_manager.CommitTransaction(txn); @@ -262,7 +262,7 @@ TEST_F(BinderCorrectnessTest, DeleteStatementTest) { catalog_ptr->GetDatabaseWithName(default_database_name, txn)->GetOid(); oid_t tableB_oid = catalog_ptr ->GetTableWithName(default_database_name, - DEFUALT_SCHEMA_NAME, "b", txn) + DEFAULT_SCHEMA_NAME, "b", txn) ->GetOid(); string deleteSQL = "DELETE FROM b WHERE 1 = b1 AND b2 = 'str'"; diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index 3e6b5e15872..b22e1f8b8a4 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -79,11 +79,11 @@ TEST_F(CatalogTests, CreatingTable) { new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFUALT_SCHEMA_NAME, "emp_table", std::move(table_schema), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFUALT_SCHEMA_NAME, + "emp_db", DEFAULT_SCHEMA_NAME, "emp_table", std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema_2), txn); - catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, "salary_table", std::move(table_schema_3), txn); // insert random tuple into DATABASE_METRICS_CATALOG and check @@ -112,7 +112,7 @@ TEST_F(CatalogTests, CreatingTable) { EXPECT_EQ('a', *param1.buf); // check colum object EXPECT_EQ("name", catalog::Catalog::GetInstance() - ->GetTableObject("emp_db", DEFUALT_SCHEMA_NAME, + ->GetTableObject("emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn) ->GetColumnObject(1) ->GetColumnName()); @@ -124,7 +124,7 @@ TEST_F(CatalogTests, TableObject) { auto txn = txn_manager.BeginTransaction(); auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); auto index_objects = table_object->GetIndexObjects(); auto column_objects = table_object->GetColumnObjects(); @@ -158,7 +158,7 @@ TEST_F(CatalogTests, TableObject) { bool update_result = pg_table->UpdateVersionId(1, department_table_oid, txn); // get version id after update, invalidate old cache table_object = catalog::Catalog::GetInstance()->GetTableObject( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); uint32_t version_oid = table_object->GetVersionId(); EXPECT_NE(department_table_oid, INVALID_OID); EXPECT_EQ(update_result, true); @@ -241,14 +241,14 @@ TEST_F(CatalogTests, DroppingTable) { auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); EXPECT_NE(nullptr, database_object); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); EXPECT_NE(nullptr, database_object); auto department_table_object = - database_object->GetTableObject("department_table", DEFUALT_SCHEMA_NAME); + database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); EXPECT_EQ( 10, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); @@ -259,7 +259,7 @@ TEST_F(CatalogTests, DroppingTable) { // Try to drop again txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn), + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), CatalogException); // EXPECT_EQ( @@ -270,7 +270,7 @@ TEST_F(CatalogTests, DroppingTable) { // Drop a table that does not exist txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( - "emp_db", DEFUALT_SCHEMA_NAME, "void_table", txn), + "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), CatalogException); EXPECT_EQ( 10, @@ -279,7 +279,7 @@ TEST_F(CatalogTests, DroppingTable) { // Drop the other table txn = txn_manager.BeginTransaction(); - catalog::Catalog::GetInstance()->DropTable("emp_db", DEFUALT_SCHEMA_NAME, + catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "emp_table", txn); EXPECT_EQ( 9, diff --git a/test/catalog/constraints_test.cpp b/test/catalog/constraints_test.cpp index d37f4f7081e..dfd8b015a81 100644 --- a/test/catalog/constraints_test.cpp +++ b/test/catalog/constraints_test.cpp @@ -237,10 +237,10 @@ TEST_F(ConstraintsTests, UNIQUETest) { new catalog::Schema({column1, column2})); std::string table_name("TEST_TABLE"); catalog::Catalog::GetInstance()->CreateTable(DEFAULT_DB_NAME, - DEFUALT_SCHEMA_NAME, table_name, + DEFAULT_SCHEMA_NAME, table_name, std::move(table_schema), txn); storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); // table->AddUNIQUEIndex(); diff --git a/test/codegen/bloom_filter_test.cpp b/test/codegen/bloom_filter_test.cpp index bf9962ca834..ec632255dc9 100644 --- a/test/codegen/bloom_filter_test.cpp +++ b/test/codegen/bloom_filter_test.cpp @@ -213,7 +213,7 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { int curr_size = 0; std::vector numbers; std::unordered_set number_set; - auto *table1 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto *table1 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table1_name, txn); while (curr_size < table1_target_size) { // Find a unique random number @@ -234,7 +234,7 @@ TEST_F(BloomFilterCodegenTest, PerformanceTest) { LOG_INFO("Finish populating test1"); // Load the inner table which contains twice tuples as the outer table - auto *table2 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto *table2 = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table2_name, txn); unsigned outer_table_cardinality = numbers.size() * outer_to_inner_ratio; for (unsigned i = 0; i < outer_table_cardinality; i++) { @@ -334,7 +334,7 @@ void BloomFilterCodegenTest::CreateTable(std::string table_name, int tuple_size, } auto *catalog = catalog::Catalog::GetInstance(); catalog->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, std::unique_ptr(new catalog::Schema(cols)), txn); } diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 39ee4ab7dd5..52b18d5d139 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -75,12 +75,15 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(test_db_name, table_name, std::move(table_schema), - txn, is_catalog, tuples_per_tilegroup, layout_type); + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup, layout_type); + // Get table reference + auto table = catalog->GetTableWithName(test_db_name, + DEFAULT_SCHEMA_NAME, + table_name, txn); txn_manager.EndTransaction(txn); - auto table = GetDatabase().GetTableWithName(table_name); - ///////////////////////////////////////////////////////// // Load in the data ///////////////////////////////////////////////////////// @@ -184,11 +187,11 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { std::unique_ptr schema{new catalog::Schema(cols)}; // Insert table in catalog - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, all_cols_table_name, + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, std::move(schema), txn); all_cols_table = catalog->GetTableWithName( - test_db_name, DEFUALT_SCHEMA_NAME, all_cols_table_name, txn); + test_db_name, DEFAULT_SCHEMA_NAME, all_cols_table_name, txn); auto *table_schema = all_cols_table->GetSchema(); // Insert one row where all columns are NULL @@ -763,12 +766,14 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(test_db_name, table_name, std::move(table_schema), - txn, is_catalog, tuples_per_tilegroup, LayoutType::ROW); + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup, LayoutType::ROW); + // Get table reference + auto table = catalog->GetTableWithName(test_db_name, + DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.EndTransaction(txn); - auto table = GetDatabase().GetTableWithName(table_name); - ///////////////////////////////////////////////////////// // Reset default_layout_ to LayoutType::COLUMN ///////////////////////////////////////////////////////// diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index fe8f6c26440..dc46599010b 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -107,23 +107,23 @@ void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, auto *catalog = catalog::Catalog::GetInstance(); for (int i = 0; i < 4; i++) { auto table_schema = CreateTestSchema(); - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, test_table_names[i], + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, tuples_per_tilegroup); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, - DEFUALT_SCHEMA_NAME, + DEFAULT_SCHEMA_NAME, test_table_names[i], txn) ->GetTableOid()); } for (int i = 4; i < 5; i++) { auto table_schema = CreateTestSchema(true); - catalog->CreateTable(test_db_name, DEFUALT_SCHEMA_NAME, test_table_names[i], + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, tuples_per_tilegroup); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, - DEFUALT_SCHEMA_NAME, + DEFAULT_SCHEMA_NAME, test_table_names[i], txn) ->GetTableOid()); } diff --git a/test/executor/create_index_test.cpp b/test/executor/create_index_test.cpp index 034eca553fa..47bbd509307 100644 --- a/test/executor/create_index_test.cpp +++ b/test/executor/create_index_test.cpp @@ -205,7 +205,7 @@ TEST_F(CreateIndexTests, CreatingIndex) { txn = txn_manager.BeginTransaction(); auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Expected 2 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); diff --git a/test/executor/create_test.cpp b/test/executor/create_test.cpp index 0faede0508d..214f32fa44d 100644 --- a/test/executor/create_test.cpp +++ b/test/executor/create_test.cpp @@ -99,7 +99,7 @@ TEST_F(CreateTests, CreatingTable) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); @@ -141,7 +141,7 @@ TEST_F(CreateTests, CreatingUDFs) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -243,7 +243,7 @@ TEST_F(CreateTests, CreatingTrigger) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -333,7 +333,7 @@ TEST_F(CreateTests, CreatingTrigger) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -373,7 +373,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -421,7 +421,7 @@ TEST_F(CreateTests, CreatingTriggerWithoutWhen) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -462,7 +462,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node("accounts", DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node("accounts", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -502,7 +502,7 @@ TEST_F(CreateTests, CreatingTriggerInCatalog) { // check whether the trigger catalog table contains this new trigger auto table_object = catalog::Catalog::GetInstance()->GetTableObject( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); auto trigger_list = catalog::Catalog::GetInstance() ->GetSystemCatalogs(table_object->GetDatabaseOid()) diff --git a/test/executor/delete_test.cpp b/test/executor/delete_test.cpp index f081b766720..648944c8fe9 100644 --- a/test/executor/delete_test.cpp +++ b/test/executor/delete_test.cpp @@ -116,7 +116,7 @@ TEST_F(DeleteTests, VariousOperations) { new catalog::Schema({id_column, name_column})); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); @@ -125,7 +125,7 @@ TEST_F(DeleteTests, VariousOperations) { LOG_INFO("Table created!"); storage::DataTable *table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 19e6c1293da..84a395936d3 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -96,12 +96,12 @@ TEST_F(DropTests, DroppingTable) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_2", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_2", std::move(table_schema2), txn); txn_manager.CommitTransaction(txn); @@ -113,7 +113,7 @@ TEST_F(DropTests, DroppingTable) { 10); // Now dropping the table using the executor - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() @@ -148,7 +148,7 @@ TEST_F(DropTests, DroppingTrigger) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -180,7 +180,7 @@ TEST_F(DropTests, DroppingTrigger) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -212,7 +212,7 @@ TEST_F(DropTests, DroppingTrigger) { // Now dropping the table using the executer txn = txn_manager.BeginTransaction(); - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); EXPECT_EQ(8, (int)catalog::Catalog::GetInstance() ->GetDatabaseObject(TEST_DB_NAME, txn) @@ -250,18 +250,18 @@ TEST_F(DropTests, DroppingIndexByName) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", txn); oid_t col_id = source_table->GetSchema()->GetColumnID(id_column.column_name); std::vector source_col_ids; source_col_ids.push_back(col_id); std::string index_name1 = "Testing_Drop_Index_By_Name"; - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", source_col_ids, index_name1, false, IndexType::BWTREE, txn); txn_manager.CommitTransaction(txn); @@ -276,25 +276,25 @@ TEST_F(DropTests, DroppingIndexByName) { ->GetSystemCatalogs(database_object->GetDatabaseOid()) ->GetIndexCatalog(); auto index_object = - pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn); + pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn); EXPECT_NE(nullptr, index_object); // Check the effect of drop // Most major check in this test case // Now dropping the index using the DropIndex functionality catalog->DropIndex(database_object->GetDatabaseOid(), index_object->GetIndexOid(), txn); - EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn), + EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); txn_manager.CommitTransaction(txn); // Drop the table just created txn = txn_manager.BeginTransaction(); // Check the effect of drop index - EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFUALT_SCHEMA_NAME, txn), + EXPECT_EQ(pg_index->GetIndexObject(index_name1, DEFAULT_SCHEMA_NAME, txn), nullptr); // Now dropping the table - catalog->DropTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table_01", + catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table_01", txn); txn_manager.CommitTransaction(txn); diff --git a/test/executor/insert_test.cpp b/test/executor/insert_test.cpp index 4499eb6f53e..489b871121d 100644 --- a/test/executor/insert_test.cpp +++ b/test/executor/insert_test.cpp @@ -53,11 +53,11 @@ TEST_F(InsertTests, InsertRecord) { txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "TEST_TABLE", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", std::move(table_schema), txn); auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "TEST_TABLE", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "TEST_TABLE", txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index 5d63253b9ca..d592f58f6a7 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -430,15 +430,16 @@ storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); // Insert table in catalog - catalog->CreateTable(db_name, table_name, std::move(table_schema), - txn, is_catalog, tuples_per_tilegroup_count); - txn_manager.EndTransaction(txn); + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup_count); + txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - auto test_db = catalog->GetDatabaseWithName(db_name, txn); - txn_manager.EndTransaction(txn); + auto table = catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); + txn_manager.CommitTransaction(txn); - auto table = test_db->GetTableWithName(table_name); return table; } diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index 7e0fd9d997d..6ade606efb4 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -184,7 +184,7 @@ TEST_F(UpdateTests, UpdatingOld) { new catalog::Schema({id_column, manager_id_column, name_column})); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); @@ -194,7 +194,7 @@ TEST_F(UpdateTests, UpdatingOld) { LOG_INFO("Table created!"); storage::DataTable *table = catalog->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); // Inserting a tuple end-to-end diff --git a/test/include/catalog/testing_constraints_util.h b/test/include/catalog/testing_constraints_util.h index 30a9b658223..1c96b995fa9 100644 --- a/test/include/catalog/testing_constraints_util.h +++ b/test/include/catalog/testing_constraints_util.h @@ -132,13 +132,13 @@ class TestingConstraintsUtil { // Create table. txn = txn_manager.BeginTransaction(); auto result = - catalog->CreateTable(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, + catalog->CreateTable(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, std::move(table_schema), txn, false); txn_manager.CommitTransaction(txn); EXPECT_EQ(ResultType::SUCCESS, result); txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_NE(nullptr, table); diff --git a/test/optimizer/old_optimizer_test.cpp b/test/optimizer/old_optimizer_test.cpp index 1069ab87909..92949cc8521 100644 --- a/test/optimizer/old_optimizer_test.cpp +++ b/test/optimizer/old_optimizer_test.cpp @@ -171,7 +171,7 @@ TEST_F(OldOptimizerTests, UpdateDelWithIndexScanTest) { txn = txn_manager.BeginTransaction(); auto target_table_ = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Expected 1 , Primary key index + created index EXPECT_EQ(target_table_->GetIndexCount(), 2); txn_manager.CommitTransaction(txn); diff --git a/test/optimizer/selectivity_test.cpp b/test/optimizer/selectivity_test.cpp index 0f5e4a9ab8c..cbb8df08d2c 100644 --- a/test/optimizer/selectivity_test.cpp +++ b/test/optimizer/selectivity_test.cpp @@ -72,7 +72,7 @@ TEST_F(SelectivityTests, RangeSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, TEST_TABLE_NAME, txn); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); @@ -181,7 +181,7 @@ TEST_F(SelectivityTests, EqualSelectivityTest) { txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); auto database = catalog->GetDatabaseWithName(DEFAULT_DB_NAME, txn); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, TEST_TABLE_NAME, txn); txn_manager.CommitTransaction(txn); oid_t db_id = database->GetOid(); diff --git a/test/optimizer/table_stats_collector_test.cpp b/test/optimizer/table_stats_collector_test.cpp index 7cafebe3e9e..e98f1a54bf9 100644 --- a/test/optimizer/table_stats_collector_test.cpp +++ b/test/optimizer/table_stats_collector_test.cpp @@ -60,7 +60,7 @@ TEST_F(TableStatsCollectorTests, SingleColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; @@ -111,7 +111,7 @@ TEST_F(TableStatsCollectorTests, MultiColumnTableTest) { } txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); txn_manager.CommitTransaction(txn); TableStatsCollector stats{table}; diff --git a/test/optimizer/tuple_samples_storage_test.cpp b/test/optimizer/tuple_samples_storage_test.cpp index 2103d63549c..72ed769f270 100644 --- a/test/optimizer/tuple_samples_storage_test.cpp +++ b/test/optimizer/tuple_samples_storage_test.cpp @@ -85,7 +85,7 @@ TEST_F(TupleSamplesStorageTests, AddSamplesTableTest) { data_table->GetDatabaseOid(), data_table->GetOid()); txn = txn_manager.BeginTransaction(); storage::DataTable *samples_table = catalog->GetTableWithName( - SAMPLES_DB_NAME, DEFUALT_SCHEMA_NAME, samples_table_name, txn); + SAMPLES_DB_NAME, DEFAULT_SCHEMA_NAME, samples_table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_TRUE(samples_table != nullptr); diff --git a/test/planner/plan_util_test.cpp b/test/planner/plan_util_test.cpp index 8455e98efac..77df6f54e88 100644 --- a/test/planner/plan_util_test.cpp +++ b/test/planner/plan_util_test.cpp @@ -54,10 +54,10 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", std::move(table_schema), txn); auto source_table = catalog->GetTableWithName( - TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", txn); + TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", txn); EXPECT_NE(source_table, nullptr); txn_manager.CommitTransaction(txn); @@ -67,7 +67,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { source_col_ids.push_back(col_id); // create index on 'id' - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", source_col_ids, "test_id_idx", false, IndexType::BWTREE, txn); @@ -75,7 +75,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { col_id = source_table->GetSchema()->GetColumnID(fname_column.column_name); source_col_ids.push_back(col_id); - catalog->CreateIndex(TEST_DB_NAME, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateIndex(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "test_table", source_col_ids, "test_fname_idx", false, IndexType::BWTREE, txn); txn_manager.CommitTransaction(txn); @@ -91,7 +91,7 @@ TEST_F(PlanUtilTests, GetAffectedIndexesTest) { // 1) id // 2) id and first_name auto table_object = - db_object->GetTableObject("test_table", DEFUALT_SCHEMA_NAME); + db_object->GetTableObject("test_table", DEFAULT_SCHEMA_NAME); EXPECT_NE(table_object, nullptr); oid_t id_idx_oid = table_object->GetIndexObject("test_id_idx")->GetIndexOid(); oid_t fname_idx_oid = @@ -198,14 +198,14 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { new catalog::Schema({id_column, fname_column, lname_column})); txn = txn_manager.BeginTransaction(); - catalog->CreateTable(TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table", + catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); auto source_table = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table", txn); + TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table", txn); txn_manager.CommitTransaction(txn); oid_t table_id = source_table->GetOid(); @@ -228,14 +228,14 @@ TEST_F(PlanUtilTests, GetIndexableColumnsTest) { std::unique_ptr job_table_schema( new catalog::Schema({age_column, job_column, pid_column})); - catalog->CreateTable(TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table_job", + catalog->CreateTable(TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", std::move(job_table_schema), txn); txn_manager.CommitTransaction(txn); // Obtain ids for the table and columns txn = txn_manager.BeginTransaction(); auto source_table_job = catalog->GetTableWithName( - TEST_DB_COLUMNS, DEFUALT_SCHEMA_NAME, "test_table_job", txn); + TEST_DB_COLUMNS, DEFAULT_SCHEMA_NAME, "test_table_job", txn); oid_t table_job_id = source_table_job->GetOid(); oid_t age_col_oid = source_table_job->GetSchema()->GetColumnID(age_column.column_name); diff --git a/test/planner/planner_test.cpp b/test/planner/planner_test.cpp index 7b263bccd79..5fd1709eaca 100644 --- a/test/planner/planner_test.cpp +++ b/test/planner/planner_test.cpp @@ -85,7 +85,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -100,7 +100,7 @@ TEST_F(PlannerTest, DeletePlanTestParameter) { ExpressionType::COMPARE_EQUAL, tuple_expr, parameter_expr); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); // Create delete plan std::unique_ptr delete_plan( @@ -149,7 +149,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -159,7 +159,7 @@ TEST_F(PlannerTest, UpdatePlanTestParameter) { auto table_name = std::string("department_table"); auto database_name = DEFAULT_DB_NAME; auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - database_name, DEFUALT_SCHEMA_NAME, table_name, txn); + database_name, DEFAULT_SCHEMA_NAME, table_name, txn); auto schema = target_table->GetSchema(); TargetList tlist; @@ -246,7 +246,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); auto ret = catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); if (ret != ResultType::SUCCESS) LOG_TRACE("create table failed"); txn_manager.CommitTransaction(txn); @@ -276,7 +276,7 @@ TEST_F(PlannerTest, InsertPlanTestParameter) { std::unique_ptr(parameter_expr_2)); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, @@ -320,7 +320,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog::Catalog::GetInstance()->CreateTable( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); txn_manager.CommitTransaction(txn); @@ -354,7 +354,7 @@ TEST_F(PlannerTest, InsertPlanTestParameterColumns) { std::unique_ptr(parameter_expr_2)); auto target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "department_table", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); std::unique_ptr insert_plan( new planner::InsertPlan(target_table, &insert_statement->columns, diff --git a/test/sql/drop_sql_test.cpp b/test/sql/drop_sql_test.cpp index 64f6af5f001..90ebbe69bf0 100644 --- a/test/sql/drop_sql_test.cpp +++ b/test/sql/drop_sql_test.cpp @@ -41,7 +41,7 @@ TEST_F(DropSQLTests, DropTableTest) { txn = txn_manager.BeginTransaction(); try { table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); } catch (CatalogException &e) { table = nullptr; } @@ -77,7 +77,7 @@ TEST_F(DropSQLTests, DropTableTest) { txn = txn_manager.BeginTransaction(); try { table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test", txn); } catch (CatalogException &e) { txn_manager.CommitTransaction(txn); table = nullptr; @@ -114,7 +114,7 @@ TEST_F(DropSQLTests, DropIndexTest) { std::shared_ptr index; txn = txn_manager.BeginTransaction(); try { - index = pg_index->GetIndexObject("idx", DEFUALT_SCHEMA_NAME, txn); + index = pg_index->GetIndexObject("idx", DEFAULT_SCHEMA_NAME, txn); } catch (CatalogException &e) { index = nullptr; @@ -128,7 +128,7 @@ TEST_F(DropSQLTests, DropIndexTest) { // Check if index is not in catalog txn = txn_manager.BeginTransaction(); - index = pg_index->GetIndexObject("idx", DEFUALT_SCHEMA_NAME, txn); + index = pg_index->GetIndexObject("idx", DEFAULT_SCHEMA_NAME, txn); EXPECT_EQ(index, nullptr); // Free the database just created diff --git a/test/sql/optimizer_sql_test.cpp b/test/sql/optimizer_sql_test.cpp index a81734c529d..3855c015e20 100644 --- a/test/sql/optimizer_sql_test.cpp +++ b/test/sql/optimizer_sql_test.cpp @@ -332,7 +332,7 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { auto txn = txn_manager.BeginTransaction(); // using transaction to get table from catalog auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test2", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn); EXPECT_NE(nullptr, table); auto cols = table->GetSchema()->GetColumns(); EXPECT_EQ(3, cols.size()); @@ -354,7 +354,7 @@ TEST_F(OptimizerSQLTests, DDLSqlTest) { txn = txn_manager.BeginTransaction(); EXPECT_THROW(catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "test2", txn), + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "test2", txn), peloton::Exception); txn_manager.CommitTransaction(txn); } diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp index a53775e67a6..77d4a65361d 100644 --- a/test/statistics/stats_test.cpp +++ b/test/statistics/stats_test.cpp @@ -137,7 +137,7 @@ TEST_F(StatsTests, MultiThreadStatsTest) { new catalog::Schema({id_column, name_column})); catalog->CreateDatabase("emp_db", txn); catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", std::move(table_schema), txn); // Create multiple stat worker threads @@ -145,7 +145,7 @@ TEST_F(StatsTests, MultiThreadStatsTest) { storage::Database *database = catalog->GetDatabaseWithName("emp_db", txn); storage::DataTable *table = catalog->GetTableWithName( - "emp_db", DEFUALT_SCHEMA_NAME, "department_table", txn); + "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn); txn_manager.CommitTransaction(txn); LaunchParallelTest(num_threads, TransactionTest, database, table); // Wait for aggregation to finish diff --git a/test/statistics/testing_stats_util.cpp b/test/statistics/testing_stats_util.cpp index 7b0f87852fe..5c087e4aba4 100644 --- a/test/statistics/testing_stats_util.cpp +++ b/test/statistics/testing_stats_util.cpp @@ -125,7 +125,7 @@ void TestingStatsUtil::CreateTable(bool has_primary_key) { auto txn = txn_manager.BeginTransaction(); std::unique_ptr context( new executor::ExecutorContext(txn)); - planner::CreatePlan node("department_table", DEFUALT_SCHEMA_NAME, "emp_db", + planner::CreatePlan node("department_table", DEFAULT_SCHEMA_NAME, "emp_db", std::move(table_schema), CreateType::TABLE); executor::CreateExecutor create_executor(&node, context.get()); create_executor.Init(); diff --git a/test/trigger/trigger_test.cpp b/test/trigger/trigger_test.cpp index ea22d4deca5..c9c5b238ca3 100644 --- a/test/trigger/trigger_test.cpp +++ b/test/trigger/trigger_test.cpp @@ -56,7 +56,7 @@ class TriggerTests : public PelotonTest { new executor::ExecutorContext(txn)); // Create plans - planner::CreatePlan node(table_name, DEFUALT_SCHEMA_NAME, DEFAULT_DB_NAME, + planner::CreatePlan node(table_name, DEFAULT_SCHEMA_NAME, DEFAULT_DB_NAME, std::move(table_schema), CreateType::TABLE); // Create executer @@ -73,7 +73,7 @@ class TriggerTests : public PelotonTest { auto txn = txn_manager.BeginTransaction(); auto table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, std::string(table_name), txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, std::string(table_name), txn); std::unique_ptr context( new executor::ExecutorContext(txn)); @@ -148,7 +148,7 @@ class TriggerTests : public PelotonTest { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(trigger_number, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -276,7 +276,7 @@ TEST_F(TriggerTests, BeforeAndAfterRowInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -363,7 +363,7 @@ TEST_F(TriggerTests, AfterStatmentInsertTriggers) { // Check the effect of creation storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, "accounts", txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, "accounts", txn); txn_manager.CommitTransaction(txn); EXPECT_EQ(1, target_table->GetTriggerNumber()); trigger::Trigger *new_trigger = target_table->GetTriggerByIndex(0); @@ -467,7 +467,7 @@ TEST_F(TriggerTests, OtherTypesTriggers) { auto txn = txn_manager.BeginTransaction(); storage::DataTable *target_table = catalog::Catalog::GetInstance()->GetTableWithName( - DEFAULT_DB_NAME, DEFUALT_SCHEMA_NAME, table_name, txn); + DEFAULT_DB_NAME, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); trigger::TriggerList *new_trigger_list = target_table->GetTriggerList(); From f9187ccd8fda16c0848aab47ba7b030be2f17308 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 1 May 2018 22:03:57 -0400 Subject: [PATCH 32/42] Failing LayoutCatalog test because of deserialization --- src/catalog/catalog.cpp | 23 ++-- src/catalog/layout_catalog.cpp | 32 ++++-- src/include/catalog/layout_catalog.h | 10 +- src/include/storage/layout.h | 2 +- src/storage/layout.cpp | 2 +- test/catalog/catalog_test.cpp | 101 ++++++++++++++++++ test/include/executor/testing_executor_util.h | 6 +- test/tuning/layout_tuner_test.cpp | 8 ++ 8 files changed, 152 insertions(+), 32 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index 5cb3bd4f0fb..b69fc281f83 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -181,6 +181,9 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, system_catalogs->GetTableCatalog()->InsertTable( COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, CATALOG_SCHEMA_NAME, database_oid, pool_.get(), txn); + system_catalogs->GetTableCatalog()->InsertTable( + LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME, + database_oid, pool_.get(), txn); } void Catalog::Bootstrap() { @@ -557,18 +560,6 @@ ResultType Catalog::CreateIndex( return ResultType::SUCCESS; } -/* - * - // Create a new layout - ResultType CreateLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn); - // Create a new layout and set it as the default for the table - ResultType CreateDefaultLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn); - */ - /* * @brief create a new layout for a table * @param database_oid database to which the table belongs to @@ -580,8 +571,8 @@ ResultType Catalog::CreateIndex( */ std::shared_ptr Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn) { + const column_map_type &column_map, + concurrency::TransactionContext *txn) { auto storage_manager = storage::StorageManager::GetInstance(); auto database = storage_manager->GetDatabaseWithOid(database_oid); @@ -617,8 +608,8 @@ Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, */ std::shared_ptr Catalog::CreateDefaultLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn) { + const column_map_type &column_map, + concurrency::TransactionContext *txn) { auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn); // If the layout creation was successful, set it as the default if (new_layout != nullptr) { diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 52bf9dfee72..74b33b4d3ef 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -14,7 +14,6 @@ #include "catalog/catalog.h" #include "catalog/system_catalogs.h" -#include "catalog/table_catalog.h" #include "concurrency/transaction_context.h" #include "storage/data_table.h" #include "storage/layout.h" @@ -97,10 +96,10 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, auto val3 = type::ValueFactory::GetVarcharValue(layout->SerializeColumnMap(), nullptr); - tuple->SetValue(ColumnId::TABLE_OID, val0, pool); - tuple->SetValue(ColumnId::LAYOUT_OID, val1, pool); - tuple->SetValue(ColumnId::NUM_COLUMNS, val2, pool); - tuple->SetValue(ColumnId::COLUMN_MAP, val3, pool); + tuple->SetValue(LayoutCatalog::ColumnId::TABLE_OID, val0, pool); + tuple->SetValue(LayoutCatalog::ColumnId::LAYOUT_OID, val1, pool); + tuple->SetValue(LayoutCatalog::ColumnId::NUM_COLUMNS, val2, pool); + tuple->SetValue(LayoutCatalog::ColumnId::COLUMN_MAP, val3, pool); // Insert the tuple return InsertTuple(std::move(tuple), txn); @@ -167,13 +166,16 @@ LayoutCatalog::GetLayouts(oid_t table_oid, for (auto &tile : (*result_tiles)) { for (auto tuple_id : *tile) { - oid_t layout_oid = tile->GetValue(tuple_id, ColumnId::LAYOUT_OID) + oid_t layout_oid = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) .GetAs(); - oid_t num_coulmns = tile->GetValue(tuple_id, ColumnId::NUM_COLUMNS) + oid_t num_coulmns = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) .GetAs(); - std::string column_map_str = tile->GetValue( - tuple_id, ColumnId::COLUMN_MAP).GetAs(); + std::string column_map_str = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) + .GetAs(); auto column_map = storage::Layout::DeserializeColumnMap(num_coulmns, column_map_str); auto layout_object = @@ -185,5 +187,17 @@ LayoutCatalog::GetLayouts(oid_t table_oid, return table_object->GetLayouts(); } +std::shared_ptr +LayoutCatalog::GetLayoutWithOid(oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn) { + auto table_layouts = GetLayouts(table_oid, txn); + for (const auto &layout_entry : table_layouts) { + if (layout_entry.second->GetOid() == layout_oid) { + return layout_entry.second; + } + } + return nullptr; +} + } // namespace catalog } // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 70c8e226fdc..70059dc6a59 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -38,8 +38,6 @@ class Layout; namespace catalog { class LayoutCatalog : public AbstractCatalog { - friend class TableCatalogObject; - friend class Catalog; public: // Global Singleton, only the first call requires passing parameters. @@ -65,14 +63,18 @@ class LayoutCatalog : public AbstractCatalog { bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); -private: - //===--------------------------------------------------------------------===// // Read Related API //===--------------------------------------------------------------------===// const std::unordered_map> GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); + std::shared_ptr + GetLayoutWithOid(oid_t table_oid, oid_t layout_oid, + concurrency::TransactionContext *txn); + +private: + std::unique_ptr InitializeSchema(); enum ColumnId { diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index e37a5fb6948..44f44f0b708 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -60,7 +60,7 @@ class Layout : public Printable { std::vector GetLayoutSchemas(catalog::Schema* const schema) const; - std::map GetColumnLayoutStats() const; + std::map GetLayoutStats() const; std::string SerializeColumnMap() const; diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 0a7915e9c89..d343085fb79 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -186,7 +186,7 @@ std::vector Layout::GetLayoutSchemas( return schemas; } -std::map Layout::GetColumnLayoutStats() const { +std::map Layout::GetLayoutStats() const { std::map column_map_stats; // Cluster per-tile column count diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index b22e1f8b8a4..0fff0f5a786 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -15,6 +15,7 @@ #include "catalog/database_catalog.h" #include "catalog/database_metrics_catalog.h" #include "catalog/index_catalog.h" +#include "catalog/layout_catalog.h" #include "catalog/query_metrics_catalog.h" #include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" @@ -303,5 +304,105 @@ TEST_F(CatalogTests, DroppingCatalog) { EXPECT_NE(nullptr, catalog); } +TEST_F(CatalogTests, LayoutCatalogTest) { + // This test creates a table, changes its layout. + // Create another additional layout. + // Ensure that default is not changed. + // Drops layout and verifies that the default_layout is reset. + // It also queries pg_layout to ensure that the entry is removed. + + auto db_name = "temp_db"; + auto table_name = "temp_table"; + auto catalog = catalog::Catalog::GetInstance(); + // Create database. + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + auto txn = txn_manager.BeginTransaction(); + catalog->CreateDatabase(db_name, txn); + + // Create table. + auto val0 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "val0", true); + auto val1 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "val1", true); + auto val2 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "val2", true); + auto val3 = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "val3", true); + std::unique_ptr table_schema( + new catalog::Schema({val0, val1, val2, val3})); + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, + table_name, std::move(table_schema), txn); + txn_manager.CommitTransaction(txn); + + txn = txn_manager.BeginTransaction(); + auto table = catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); + auto table_oid = table->GetOid(); + auto database_oid = table->GetDatabaseOid(); + txn_manager.CommitTransaction(txn); + + // Change default layout. + + std::map> default_map; + default_map[0] = std::make_pair(0, 0); + default_map[1] = std::make_pair(0, 1); + default_map[2] = std::make_pair(1, 0); + default_map[3] = std::make_pair(1, 1); + + txn = txn_manager.BeginTransaction(); + auto default_layout = catalog->CreateDefaultLayout(database_oid, table_oid, + default_map, txn); + txn_manager.CommitTransaction(txn); + EXPECT_NE(nullptr, default_layout); + + // Create additional layout. + std::map> non_default_map; + non_default_map[0] = std::make_pair(0, 0); + non_default_map[1] = std::make_pair(0, 1); + non_default_map[2] = std::make_pair(1, 0); + non_default_map[3] = std::make_pair(1, 1); + + txn = txn_manager.BeginTransaction(); + auto other_layout = catalog->CreateLayout(database_oid, table_oid, + non_default_map, txn); + txn_manager.CommitTransaction(txn); + + // Check that the default layout is still the same. + EXPECT_NE(*other_layout.get(), table->GetDefaultLayout()); + + // Drop the default layout. + auto default_layout_oid = default_layout->GetOid(); + txn = txn_manager.BeginTransaction(); + EXPECT_EQ(ResultType::SUCCESS, catalog->DropLayout( + database_oid, table_oid, default_layout_oid, txn)); + txn_manager.CommitTransaction(txn); + + // Check that default layout is reset and set to row_store. + EXPECT_NE(*default_layout.get(), table->GetDefaultLayout()); + EXPECT_TRUE(table->GetDefaultLayout().IsRowStore()); + + // Query pg_layout to ensure that the entry is dropped + txn = txn_manager.BeginTransaction(); + auto pg_layout = catalog-> + GetSystemCatalogs(database_oid)->GetLayoutCatalog(); + EXPECT_EQ(nullptr, + pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn)); + + // The additional layout must be present in pg_layout + auto other_layout_oid = other_layout->GetOid(); + EXPECT_NE(nullptr, + pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn)); + txn_manager.CommitTransaction(txn); + + // Drop database + txn = txn_manager.BeginTransaction(); + catalog->DropDatabaseWithName(db_name, txn); + txn_manager.CommitTransaction(txn); +} + } // namespace test } // namespace peloton diff --git a/test/include/executor/testing_executor_util.h b/test/include/executor/testing_executor_util.h index abb8e20ab9a..10459c6c8fc 100644 --- a/test/include/executor/testing_executor_util.h +++ b/test/include/executor/testing_executor_util.h @@ -75,7 +75,11 @@ class TestingExecutorUtil { int tuples_per_tilegroup_count = TESTS_TUPLES_PER_TILEGROUP, bool indexes = true, oid_t table_oid = INVALID_OID); - /** @brief Creates a basic table and adds its entry to the catalog */ + /** + * @brief Creates a basic table and adds its entry to the catalog + * + * @return A pointer to the DataTable created. + */ static storage::DataTable *CreateTableUpdateCatalog( int tuples_per_tilegroup_count, std::string &db_name); diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index 8efc83d9d19..42ed404e112 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -125,6 +125,14 @@ TEST_F(LayoutTunerTests, BasicTest) { EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2),0); EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3),1); + // Check the per tile stats of the new layout + // The layout must contain 2 tiles with the following stats + // 0 -> 3 + // 1 -> 1 + auto layout_stats = new_default_layout.GetLayoutStats(); + EXPECT_EQ(layout_stats[0], 3); + EXPECT_EQ(layout_stats[1], 1); + TestingExecutorUtil::DeleteDatabase(db_name); } From 5e62206e8ffe5b6b11eec0a7f48b35a0a2b67871 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 2 May 2018 01:01:34 -0400 Subject: [PATCH 33/42] Fix all tests --- src/catalog/layout_catalog.cpp | 47 ++++++++++++++++++++-------------- src/include/storage/layout.h | 2 +- src/storage/layout.cpp | 2 +- test/catalog/catalog_test.cpp | 17 +++++------- test/executor/drop_test.cpp | 8 +++--- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 74b33b4d3ef..5720473d550 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -2,7 +2,7 @@ // // Peloton // -// column_catalog.h +// layout_catalog.h // // Identification: src/include/catalog/layout_catalog.cpp // @@ -15,6 +15,7 @@ #include "catalog/catalog.h" #include "catalog/system_catalogs.h" #include "concurrency/transaction_context.h" +#include "executor/logical_tile.h" #include "storage/data_table.h" #include "storage/layout.h" @@ -164,24 +165,32 @@ LayoutCatalog::GetLayouts(oid_t table_oid, auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); - for (auto &tile : (*result_tiles)) { - for (auto tuple_id : *tile) { - oid_t layout_oid = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) - .GetAs(); - oid_t num_coulmns = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) - .GetAs(); - - std::string column_map_str = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) - .GetAs(); - auto column_map = storage::Layout::DeserializeColumnMap(num_coulmns, - column_map_str); - auto layout_object = - std::make_shared(column_map, layout_oid); - table_object->InsertLayout(layout_object); - } + // result_tiles should contain only 1 LogicalTile per table + auto result_size = (*result_tiles).size(); + PELOTON_ASSERT(result_size <= 1); + if (result_size == 0) { + LOG_DEBUG("No entry for table %u in pg_layout", table_oid); + return table_object->GetLayouts(); + } + + // Get the LogicalTile + auto tile = (*result_tiles)[0].get(); + auto tuple_count = tile->GetTupleCount(); + for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { + oid_t layout_oid = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) + .GetAs(); + oid_t num_columns = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) + .GetAs(); + std::string column_map_str = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) + .ToString(); + auto column_map = storage::Layout::DeserializeColumnMap(num_columns, + column_map_str); + auto layout_object = + std::make_shared(column_map, layout_oid); + table_object->InsertLayout(layout_object); } return table_object->GetLayouts(); diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 44f44f0b708..bfa520b8031 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -2,7 +2,7 @@ // // Peloton // -// tuple.h +// layout.h // // Identification: src/include/storage/layout.h // diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index d343085fb79..243df76725f 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -2,7 +2,7 @@ // // Peloton // -// tuple.cpp +// layout.cpp // // Identification: src/storage/layout.cpp // diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index 0fff0f5a786..cda93ac7d68 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -235,9 +235,8 @@ TEST_F(CatalogTests, DroppingTable) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside - EXPECT_EQ( - 11, + // NOTE: everytime we create a database, there will be 9 catalog tables inside + EXPECT_EQ(CATALOG_TABLES_COUNT+3, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); @@ -250,8 +249,7 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_NE(nullptr, database_object); auto department_table_object = database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); - EXPECT_EQ( - 10, + EXPECT_EQ(CATALOG_TABLES_COUNT+2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -263,8 +261,7 @@ TEST_F(CatalogTests, DroppingTable) { "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), CatalogException); // - EXPECT_EQ( - 10, + EXPECT_EQ(CATALOG_TABLES_COUNT+2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -273,8 +270,7 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), CatalogException); - EXPECT_EQ( - 10, + EXPECT_EQ(CATALOG_TABLES_COUNT+2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -282,8 +278,7 @@ TEST_F(CatalogTests, DroppingTable) { txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "emp_table", txn); - EXPECT_EQ( - 9, + EXPECT_EQ(CATALOG_TABLES_COUNT+1, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); } diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 84a395936d3..f780b506b81 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -109,16 +109,14 @@ TEST_F(DropTests, DroppingTable) { // NOTE: everytime we create a database, there will be 8 catalog tables inside EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() - .size(), - 10); + .size(), (CATALOG_TABLES_COUNT + 2)); // Now dropping the table using the executor catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() - .size(), - 9); + .size(), (CATALOG_TABLES_COUNT + 1)); // free the database just created catalog->DropDatabaseWithName(TEST_DB_NAME, txn); @@ -214,7 +212,7 @@ TEST_F(DropTests, DroppingTrigger) { txn = txn_manager.BeginTransaction(); catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); - EXPECT_EQ(8, (int)catalog::Catalog::GetInstance() + EXPECT_EQ(CATALOG_TABLES_COUNT, (int)catalog::Catalog::GetInstance() ->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size()); From 6ce9fc8f3cea58fa63c34ab56764fe87b62e686e Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 2 May 2018 16:13:02 -0400 Subject: [PATCH 34/42] Added documentation --- src/catalog/catalog.cpp | 3 +- src/catalog/layout_catalog.cpp | 94 +++++++++++++++++----------- src/catalog/table_catalog.cpp | 39 ++++++------ src/include/catalog/layout_catalog.h | 2 +- src/include/storage/layout.h | 64 ++++++++++++++++--- src/include/storage/tile_group.h | 1 + src/storage/layout.cpp | 2 +- src/storage/tile_group.cpp | 1 + src/tuning/layout_tuner.cpp | 2 +- test/catalog/catalog_test.cpp | 4 +- 10 files changed, 140 insertions(+), 72 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index b69fc281f83..b182ef54fc9 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -353,7 +353,8 @@ ResultType Catalog::CreateTable(const std::string &database_name, bool adapt_table = false; auto table = storage::TableFactory::GetDataTable( database_object->GetDatabaseOid(), table_oid, schema.release(), - table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog, layout_type); + table_name, tuples_per_tilegroup, own_schema, adapt_table, is_catalog, + layout_type); database->AddTable(table, is_catalog); // put data table object into rw_object_set txn->RecordCreate(database_object->GetDatabaseOid(), table_oid, INVALID_OID); diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index 5720473d550..cb9628210c1 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -22,13 +22,9 @@ namespace peloton { namespace catalog { -LayoutCatalog *LayoutCatalog::GetInstance(storage::Database *pg_catalog, - type::AbstractPool *pool, - concurrency::TransactionContext *txn) { - static LayoutCatalog layout_catalog{pg_catalog, pool, txn}; - return &layout_catalog; -} - +/** @brief Constructor invoked by the SystemsCatalog constructor. + * @param pg_catalog The database to which this pg_layout belongs. + */ LayoutCatalog::LayoutCatalog( storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, @@ -42,9 +38,12 @@ LayoutCatalog::LayoutCatalog( AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); } - +/** @brief Destructor. Do nothing. Layouts will be dropped by DropTable. */ LayoutCatalog::~LayoutCatalog() {} +/** @brief Initilailizes the schema for the pg_layout table. + * @return unique_ptr of the schema for pg_layout. + */ std::unique_ptr LayoutCatalog::InitializeSchema() { const std::string primary_key_constraint_name = "primary_key"; const std::string not_null_constraint_name = "not_null"; @@ -83,6 +82,13 @@ std::unique_ptr LayoutCatalog::InitializeSchema() { return column_catalog_schema; } +/** @brief Insert a layout into the pg_layout table. + * @param table_oid oid of the table to which the new layout belongs. + * @param layout layout to be added to the pg_layout table. + * @param pool to allocate memory for the column_map column. + * @param txn TransactionContext for adding the layout. + * @return true on success. + */ bool LayoutCatalog::InsertLayout(oid_t table_oid, std::shared_ptr layout, type::AbstractPool *pool, @@ -106,14 +112,20 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, return InsertTuple(std::move(tuple), txn); } -bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, +/** @brief Delete a layout from the pg_layout table. + * @param table_oid oid of the table to which the old layout belongs. + * @param layout_oid oid of the layout to be deleted. + * @param txn TransactionContext for deleting the layout. + * @return true on success. + */ +bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); - values.push_back(type::ValueFactory::GetIntegerValue(layout_id).Copy()); + values.push_back(type::ValueFactory::GetIntegerValue(layout_oid).Copy()); auto pg_table = Catalog::GetInstance() ->GetSystemCatalogs(database_oid) @@ -121,11 +133,16 @@ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_id, // delete column from cache auto table_object = pg_table->GetTableObject(table_oid, txn); - table_object->EvictLayout(layout_id); + table_object->EvictLayout(layout_oid); return DeleteWithIndexScan(index_offset, values, txn); } +/** @brief Delete all layouts correponding to a table from the pg_layout. + * @param table_oid oid of the table to delete all layouts. + * @param txn TransactionContext for deleting the layouts. + * @return true on success. + */ bool LayoutCatalog::DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn) { oid_t index_offset = IndexId::SKEY_TABLE_OID; // Index of table_oid @@ -142,6 +159,11 @@ bool LayoutCatalog::DeleteLayouts(oid_t table_oid, return DeleteWithIndexScan(index_offset, values, txn); } +/** @brief Get all layouts correponding to a table from the pg_layout. + * @param table_oid oid of the table to fetch all layouts. + * @param txn TransactionContext for getting the layouts. + * @return unordered_map containing a layout_oid -> layout mapping. + */ const std::unordered_map> LayoutCatalog::GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn) { @@ -165,37 +187,35 @@ LayoutCatalog::GetLayouts(oid_t table_oid, auto result_tiles = GetResultWithIndexScan(column_ids, index_offset, values, txn); - // result_tiles should contain only 1 LogicalTile per table - auto result_size = (*result_tiles).size(); - PELOTON_ASSERT(result_size <= 1); - if (result_size == 0) { - LOG_DEBUG("No entry for table %u in pg_layout", table_oid); - return table_object->GetLayouts(); - } - - // Get the LogicalTile - auto tile = (*result_tiles)[0].get(); - auto tuple_count = tile->GetTupleCount(); - for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { - oid_t layout_oid = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) - .GetAs(); - oid_t num_columns = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) - .GetAs(); - std::string column_map_str = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) - .ToString(); - auto column_map = storage::Layout::DeserializeColumnMap(num_columns, - column_map_str); - auto layout_object = - std::make_shared(column_map, layout_oid); - table_object->InsertLayout(layout_object); + for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles + for (auto tuple_id : *tile) { + oid_t layout_oid = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) + .GetAs(); + oid_t num_columns = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) + .GetAs(); + std::string column_map_str = + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) + .ToString(); + auto column_map = storage::Layout::DeserializeColumnMap(num_columns, + column_map_str); + auto layout_object = + std::make_shared(column_map, layout_oid); + table_object->InsertLayout(layout_object); + } } return table_object->GetLayouts(); } +/** @brief Get the layout by layout_oid from the pg_layout. + * @param table_oid oid of the table to fetch the layout. + * @param layout_oid oid of the layout being queried. + * @param txn TransactionContext for getting the layout. + * @return shared_ptr corresponding to the layout_oid if found. + * nullptr otherwise. + */ std::shared_ptr LayoutCatalog::GetLayoutWithOid(oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 3432f44c7e2..bdbc4e819ba 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -331,9 +331,9 @@ TableCatalog::TableCatalog( IndexConstraintType::DEFAULT); } -/* @brief Insert layout object into the cache. - * @param layout Layout object to be inserted - * @return false if layout already exists in cache +/** @brief Insert layout object into the cache. + * @param layout Layout object to be inserted + * @return false if layout already exists in cache */ bool TableCatalogObject::InsertLayout(std::shared_ptr layout) { @@ -354,33 +354,34 @@ bool TableCatalogObject::InsertLayout(std::shared_ptr lay return true; } -/* - * @brief evict all layout objects from cache - */ +/** @brief evict all layout objects from cache. */ void TableCatalogObject::EvictAllLayouts() { layout_objects_.clear(); valid_layout_objects_ = false; } -/* @brief Get all layout objects of this table. - * Add it to the cache if necessary. - * @param cached_only If set to true, don't fetch the layout objects. - * @return Map from layout_oid to cached layout object. +/** @brief Get all layout objects of this table. + * Add it to the cache if necessary. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Map from layout_oid to cached layout object. */ std::unordered_map> TableCatalogObject::GetLayouts(bool cached_only) { if (!valid_layout_objects_ && !cached_only) { - // get column catalog objects from pg_column - LayoutCatalog::GetInstance()->GetLayouts(table_oid, txn); + // get layout catalog objects from pg_layout + auto pg_layout = Catalog::GetInstance() + ->GetSystemCatalogs(database_oid) + ->GetLayoutCatalog(); + pg_layout->GetLayouts(table_oid, txn); valid_column_objects = true; } return layout_objects_; } -/* @brief get the layout object of the given layout_id. - * @param layout_id The id of the layout to be fetched. - * @param cached_only If set to true, don't fetch the layout objects. - * @return Layout object of corresponding to the layout_id if present. +/** @brief Get the layout object of the given layout_id. + * @param layout_id The id of the layout to be fetched. + * @param cached_only If set to true, don't fetch the layout objects. + * @return Layout object of corresponding to the layout_id if present. */ std::shared_ptr TableCatalogObject::GetLayout(oid_t layout_id, bool cached_entry) { @@ -393,9 +394,9 @@ TableCatalogObject::GetLayout(oid_t layout_id, bool cached_entry) { return nullptr; } -/* @brief Evict layout from the cache. - * @param layout_id Id of the layout to be deleted. - * @return true if layout_id is found and evicted; false if not found. +/** @brief Evict layout from the cache. + * @param layout_id Id of the layout to be deleted. + * @return true if layout_id is found and evicted; false if not found. */ bool TableCatalogObject::EvictLayout(oid_t layout_id) { if (!valid_layout_objects_) return false; diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 70059dc6a59..87c0721ffb2 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -58,7 +58,7 @@ class LayoutCatalog : public AbstractCatalog { type::AbstractPool *pool, concurrency::TransactionContext *txn); - bool DeleteLayout(oid_t table_oid, oid_t layout_id, + bool DeleteLayout(oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn); bool DeleteLayouts(oid_t table_oid, concurrency::TransactionContext *txn); diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index bfa520b8031..f6a9637bae6 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -17,8 +17,11 @@ #include "common/internal_types.h" #include "common/printable.h" +//@{ +/** @brief Pre-defined OIDs for ROW and COLUMN store. */ #define ROW_STORE_OID 0 #define COLUMN_STORE_OID 1 +//@} namespace peloton { @@ -28,64 +31,105 @@ class Schema; namespace storage { +/** + * @brief Class to store the physical layout of a TileGroup. + */ class Layout : public Printable { public: + /** + * @brief Constructor for predefined layout types. + * @param num_columns Number of columns in the layouts. + * @param layout_type Has to be LayoutType::ROW or LayoutType::COLUMN. + */ Layout (const oid_t num_columns, LayoutType layout_type = LayoutType::ROW); + /** + * @brief Constructor for arbitrary column_maps. + * If the column_map is of type LayoutType::HYBRID, + * the layout_oid_ is set to INVALID_OID. For all other + * layouts, the pre-defined OIDs are used. Should be used + * only for testing & TempTables, the layout isn't persistent. + * @param column_map Column map of the layout to be constructed + * + */ Layout(const column_map_type& column_map); - Layout(const column_map_type& column_map, oid_t layout_id); + /** + * @brief Constructor for arbitrary column_maps. + * @param column_map Column map of the layout to be constructed. + * @param layout_oid Per-table unique OID. Generted by DataTable. + */ + Layout(const column_map_type& column_map, oid_t layout_oid); - // Check whether the Layout is a RowStore. + /** @brief Check whether this layout is a row store. */ inline bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } - // Check whether the Layout is ColumnStore. + /** @brief Check whether this layout is a column store. */ inline bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } + /** @brief Return the layout_oid_ of this object. */ oid_t GetOid() const { return layout_oid_; } - // Sets the tile id and column id w.r.t that tile corresponding to - // the specified tile group column id. + /** @brief Sets the tile id and column id w.r.t. of the tile corresponding + * to the specified tile group column id. + */ void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, oid_t &tile_column_offset) const; + /** @brief Returns the layout difference w.r.t. the other Layout. + * @double The delta between the layouts. Used by LayoutTuner class. + */ double GetLayoutDifference(const storage::Layout &other) const; + /** @brief Returns the tile_id of the given column_id. */ oid_t GetTileIdFromColumnId(oid_t column_id) const; - oid_t GetTileColumnId(oid_t column_id) const; + /** @brief Returns the column offset of the column_id. */ + oid_t GetTileColumnOffset(oid_t column_id) const; + /** @brief Returns the number of columns in the layout. */ oid_t GetColumnCount() const { return num_columns_;} + /** @brief Constructs the schema for the given layout. Thid function + * is used only in TempTables and LogicalTiles. + */ std::vector GetLayoutSchemas(catalog::Schema* const schema) const; + /** @brief Returns the layout statistics used by the LayoutTuner. */ std::map GetLayoutStats() const; + /** @brief Serialzies the column_map to be added to pg_layout. */ std::string SerializeColumnMap() const; + /** @brief Deserializes the string that has been read from pg_layout. */ static column_map_type DeserializeColumnMap(oid_t num_columns, std::string column_map_str); + /** @brief Returns a string containing the column_map of this layout. */ std::string GetColumnMapInfo() const; - // Get a string representation for debugging + /** @brief Returns a string representation for debugging function. */ const std::string GetInfo() const; + //@{ + /** @brief Operators for checking equality of two layouts. */ friend bool operator==(const Layout& lhs, const Layout& rhs); friend bool operator!=(const Layout& lhs, const Layout& rhs); + //@} private: - // Layout Id of the tile + /** @brief Layout Oid of the layout object. */ oid_t layout_oid_; - // Number of columns in the layout + /** @brief Number of columns in the layout. */ oid_t num_columns_; - // Layout of the columns. + /** @brief column_map of the layout. */ column_map_type column_layout_; + /** @brief Layout type is always (ROW, COLUMN or HYBRID). */ LayoutType layout_type_; }; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 2f9ea2cd815..7179cd1c9fb 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -161,6 +161,7 @@ class TileGroup : public Printable { // Sync the contents void Sync(); + // Get the layout of the TileGroup. Used to locate columns. const storage::Layout& GetLayout() const { return *tile_group_layout_; } protected: diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 243df76725f..95d54407db5 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -146,7 +146,7 @@ oid_t Layout::GetTileIdFromColumnId(oid_t column_id) const { return tile_offset; } -oid_t Layout::GetTileColumnId(oid_t column_id) const { +oid_t Layout::GetTileColumnOffset(oid_t column_id) const { oid_t tile_column_id, tile_offset; LocateTileAndColumn(column_id, tile_offset, tile_column_id); return tile_column_id; diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 2b23587a06f..ada74b154a9 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -358,6 +358,7 @@ const std::string TileGroup::GetInfo() const { << peloton::GETINFO_DOUBLE_STAR << std::endl; os << "Database[" << database_id << "] // "; os << "Table[" << table_id << "] " << std::endl; + os << "Layout[" << *tile_group_layout_ << std::endl; os << (*tile_group_header) << std::endl; for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index 9f3fc2b070f..db3e02fcf62 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -124,7 +124,7 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { } txn_manager.CommitTransaction(txn); - UNUSED_ATTRIBUTE auto layout = table-> GetDefaultLayout(); + UNUSED_ATTRIBUTE auto layout = table->GetDefaultLayout(); LOG_TRACE("Updated Layout: %s", layout.GetInfo().c_str()); } diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index cda93ac7d68..31283e80dd8 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -389,8 +389,8 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // The additional layout must be present in pg_layout auto other_layout_oid = other_layout->GetOid(); - EXPECT_NE(nullptr, - pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn)); + EXPECT_EQ(*(other_layout.get()), *(pg_layout + ->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); txn_manager.CommitTransaction(txn); // Drop database From d7ce6cd057f9fc2d8fa3cb438c1815ce8e3c59e4 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 2 May 2018 16:31:50 -0400 Subject: [PATCH 35/42] Style Fix --- src/catalog/catalog.cpp | 34 +++---- src/catalog/layout_catalog.cpp | 91 +++++++++---------- src/catalog/schema.cpp | 6 +- src/catalog/table_catalog.cpp | 18 ++-- src/codegen/proxy/runtime_functions_proxy.cpp | 3 +- src/codegen/runtime_functions.cpp | 6 +- src/codegen/tile_group.cpp | 14 ++- src/codegen/updater.cpp | 2 +- src/executor/logical_tile_factory.cpp | 2 +- src/gc/gc_manager.cpp | 59 ++++++------ src/include/catalog/catalog.h | 14 ++- src/include/catalog/layout_catalog.h | 21 ++--- src/include/catalog/schema.h | 9 +- src/include/catalog/table_catalog.h | 8 +- .../codegen/proxy/runtime_functions_proxy.h | 2 +- src/include/gc/gc_manager.h | 3 +- src/include/storage/abstract_table.h | 6 +- src/include/storage/data_table.h | 7 +- src/include/storage/layout.h | 30 +++--- src/include/storage/tile_group.h | 6 +- src/storage/abstract_table.cpp | 15 ++- src/storage/data_table.cpp | 35 +++---- src/storage/layout.cpp | 82 +++++++---------- src/storage/tile_group.cpp | 13 ++- src/tuning/layout_tuner.cpp | 4 +- test/catalog/catalog_test.cpp | 71 ++++++++------- test/catalog/manager_test.cpp | 10 +- test/codegen/table_scan_translator_test.cpp | 61 ++++++------- test/executor/drop_test.cpp | 12 ++- test/executor/logical_tile_test.cpp | 1 - test/executor/seq_scan_test.cpp | 6 +- test/executor/testing_executor_util.cpp | 16 ++-- test/executor/update_test.cpp | 8 +- test/include/executor/testing_executor_util.h | 2 +- test/statistics/stats_test.cpp | 6 +- test/storage/tile_group_test.cpp | 16 ++-- test/tuning/layout_tuner_test.cpp | 19 ++-- 37 files changed, 343 insertions(+), 375 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index b182ef54fc9..bf80063d580 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -182,8 +182,8 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, COLUMN_CATALOG_OID, COLUMN_CATALOG_NAME, CATALOG_SCHEMA_NAME, database_oid, pool_.get(), txn); system_catalogs->GetTableCatalog()->InsertTable( - LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME, - database_oid, pool_.get(), txn); + LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, CATALOG_SCHEMA_NAME, + database_oid, pool_.get(), txn); } void Catalog::Bootstrap() { @@ -570,11 +570,9 @@ ResultType Catalog::CreateIndex( * @return shared_ptr shared_ptr to the newly created layout in case of * success. nullptr in case of failure. */ -std::shared_ptr -Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn) { - +std::shared_ptr Catalog::CreateLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn) { auto storage_manager = storage::StorageManager::GetInstance(); auto database = storage_manager->GetDatabaseWithOid(database_oid); auto table = database->GetTableWithOid(table_oid); @@ -583,13 +581,12 @@ Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, // Ensure that the new layout PELOTON_ASSERT(layout_oid < INVALID_OID); auto new_layout = std::shared_ptr( - new const storage::Layout(column_map, layout_oid)); + new const storage::Layout(column_map, layout_oid)); // Add the layout the pg_layout table - auto pg_layout = - catalog_map_[database_oid]->GetLayoutCatalog(); - bool result = pg_layout->InsertLayout(table_oid, new_layout, - pool_.get(), txn); + auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); + bool result = + pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn); if (!result) { LOG_DEBUG("Failed to create a new layout for table %u", table_oid); return nullptr; @@ -607,10 +604,9 @@ Catalog::CreateLayout(oid_t database_oid, oid_t table_oid, * @return shared_ptr shared_ptr to the newly created layout in case of * success. nullptr in case of failure. */ -std::shared_ptr -Catalog::CreateDefaultLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn) { +std::shared_ptr Catalog::CreateDefaultLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn) { auto new_layout = CreateLayout(database_oid, table_oid, column_map, txn); // If the layout creation was successful, set it as the default if (new_layout != nullptr) { @@ -787,7 +783,7 @@ ResultType Catalog::DropTable(oid_t database_oid, oid_t table_oid, // delete record in pg_layout auto pg_layout = - catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); + catalog_map_[database_object->GetDatabaseOid()]->GetLayoutCatalog(); pg_layout->DeleteLayouts(table_oid, txn); // delete record in pg_table @@ -834,7 +830,6 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, return ResultType::SUCCESS; } - /*@brief Drop layout * tile_groups * @param database_oid the database to which the table belongs @@ -860,8 +855,7 @@ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, auto pg_layout = catalog_map_[database_oid]->GetLayoutCatalog(); if (!pg_layout->DeleteLayout(table_oid, layout_oid, txn)) { auto layout = table->GetDefaultLayout(); - LOG_DEBUG("Layout delete failed. Default layout id: %u", - layout.GetOid()); + LOG_DEBUG("Layout delete failed. Default layout id: %u", layout.GetOid()); return ResultType::FAILURE; } diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index cb9628210c1..d4173706020 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -26,15 +26,13 @@ namespace catalog { * @param pg_catalog The database to which this pg_layout belongs. */ LayoutCatalog::LayoutCatalog( - storage::Database *pg_catalog, - UNUSED_ATTRIBUTE type::AbstractPool *pool, - UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, - InitializeSchema().release(), pg_catalog) { + storage::Database *pg_catalog, UNUSED_ATTRIBUTE type::AbstractPool *pool, + UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) + : AbstractCatalog(LAYOUT_CATALOG_OID, LAYOUT_CATALOG_NAME, + InitializeSchema().release(), pg_catalog) { // Add indexes for pg_attribute - AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, - LAYOUT_CATALOG_PKEY_OID, LAYOUT_CATALOG_NAME "_pkey", - IndexConstraintType::PRIMARY_KEY); + AddIndex({ColumnId::TABLE_OID, ColumnId::LAYOUT_OID}, LAYOUT_CATALOG_PKEY_OID, + LAYOUT_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); AddIndex({ColumnId::TABLE_OID}, LAYOUT_CATALOG_SKEY0_OID, LAYOUT_CATALOG_NAME "_skey0", IndexConstraintType::DEFAULT); } @@ -49,35 +47,36 @@ std::unique_ptr LayoutCatalog::InitializeSchema() { const std::string not_null_constraint_name = "not_null"; auto table_id_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "table_oid", true); + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "table_oid", true); table_id_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); + ConstraintType::PRIMARY, primary_key_constraint_name)); table_id_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto layout_oid_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "layout_oid", true); + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "layout_oid", true); layout_oid_column.AddConstraint(catalog::Constraint( - ConstraintType::PRIMARY, primary_key_constraint_name)); + ConstraintType::PRIMARY, primary_key_constraint_name)); layout_oid_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto num_columns_column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "num_columns", true); + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "num_columns", true); num_columns_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); auto column_map_column = catalog::Column( - type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), - "column_map", false); + type::TypeId::VARCHAR, type::Type::GetTypeSize(type::TypeId::VARCHAR), + "column_map", false); column_map_column.AddConstraint( - catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); + catalog::Constraint(ConstraintType::NOTNULL, not_null_constraint_name)); - std::unique_ptr column_catalog_schema(new catalog::Schema( - {table_id_column, layout_oid_column, num_columns_column, column_map_column})); + std::unique_ptr column_catalog_schema( + new catalog::Schema({table_id_column, layout_oid_column, + num_columns_column, column_map_column})); return column_catalog_schema; } @@ -95,7 +94,7 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, concurrency::TransactionContext *txn) { // Create the tuple first std::unique_ptr tuple( - new storage::Tuple(catalog_table_->GetSchema(), true)); + new storage::Tuple(catalog_table_->GetSchema(), true)); auto val0 = type::ValueFactory::GetIntegerValue(table_oid); auto val1 = type::ValueFactory::GetIntegerValue(layout->GetOid()); @@ -120,16 +119,15 @@ bool LayoutCatalog::InsertLayout(oid_t table_oid, */ bool LayoutCatalog::DeleteLayout(oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { - oid_t index_offset = - IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid + oid_t index_offset = IndexId::PRIMARY_KEY; // Index of table_oid & layout_oid std::vector values; values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); values.push_back(type::ValueFactory::GetIntegerValue(layout_oid).Copy()); auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetTableCatalog(); + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); // delete column from cache auto table_object = pg_table->GetTableObject(table_oid, txn); @@ -151,8 +149,8 @@ bool LayoutCatalog::DeleteLayouts(oid_t table_oid, // delete layouts from cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetTableCatalog(); + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); auto table_object = pg_table->GetTableObject(table_oid, txn); table_object->EvictAllLayouts(); @@ -169,8 +167,8 @@ LayoutCatalog::GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn) { // Try to find the layouts in the cache auto pg_table = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetTableCatalog(); + ->GetSystemCatalogs(database_oid) + ->GetTableCatalog(); auto table_object = pg_table->GetTableObject(table_oid, txn); PELOTON_ASSERT(table_object && table_object->GetTableOid() == table_oid); auto layout_objects = table_object->GetLayouts(true); @@ -185,23 +183,23 @@ LayoutCatalog::GetLayouts(oid_t table_oid, values.push_back(type::ValueFactory::GetIntegerValue(table_oid).Copy()); auto result_tiles = - GetResultWithIndexScan(column_ids, index_offset, values, txn); + GetResultWithIndexScan(column_ids, index_offset, values, txn); - for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles + for (auto &tile : (*result_tiles)) { // Iterate through the result_tiles for (auto tuple_id : *tile) { oid_t layout_oid = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) - .GetAs(); + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::LAYOUT_OID) + .GetAs(); oid_t num_columns = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) - .GetAs(); + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::NUM_COLUMNS) + .GetAs(); std::string column_map_str = - tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) - .ToString(); - auto column_map = storage::Layout::DeserializeColumnMap(num_columns, - column_map_str); + tile->GetValue(tuple_id, LayoutCatalog::ColumnId::COLUMN_MAP) + .ToString(); + auto column_map = + storage::Layout::DeserializeColumnMap(num_columns, column_map_str); auto layout_object = - std::make_shared(column_map, layout_oid); + std::make_shared(column_map, layout_oid); table_object->InsertLayout(layout_object); } } @@ -216,9 +214,8 @@ LayoutCatalog::GetLayouts(oid_t table_oid, * @return shared_ptr corresponding to the layout_oid if found. * nullptr otherwise. */ -std::shared_ptr -LayoutCatalog::GetLayoutWithOid(oid_t table_oid, oid_t layout_oid, - concurrency::TransactionContext *txn) { +std::shared_ptr LayoutCatalog::GetLayoutWithOid( + oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { auto table_layouts = GetLayouts(table_oid, txn); for (const auto &layout_entry : table_layouts) { if (layout_entry.second->GetOid() == layout_oid) { diff --git a/src/catalog/schema.cpp b/src/catalog/schema.cpp index 4e58d9faf1e..ab38f8c8501 100644 --- a/src/catalog/schema.cpp +++ b/src/catalog/schema.cpp @@ -211,7 +211,8 @@ Schema *Schema::AppendSchema(const Schema *first, const Schema *second) { // Append subset of columns in the two given schemas Schema *Schema::AppendSchema(const Schema *first, std::vector &first_set, - const Schema *second, std::vector &second_set) { + const Schema *second, + std::vector &second_set) { const std::vector schema_list({first, second}); const std::vector> subsets({first_set, second_set}); return AppendSchemaPtrList(schema_list, subsets); @@ -229,7 +230,8 @@ Schema *Schema::AppendSchemaList(std::vector &schema_list) { } // Append given schemas. -Schema *Schema::AppendSchemaPtrList(const std::vector &schema_list) { +Schema *Schema::AppendSchemaPtrList( + const std::vector &schema_list) { std::vector> subsets; for (unsigned int i = 0; i < schema_list.size(); i++) { diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index bdbc4e819ba..0a2229123ed 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -320,8 +320,8 @@ std::shared_ptr TableCatalogObject::GetColumnObject( TableCatalog::TableCatalog( storage::Database *database, UNUSED_ATTRIBUTE type::AbstractPool *pool, UNUSED_ATTRIBUTE concurrency::TransactionContext *txn) - : AbstractCatalog(TABLE_CATALOG_OID, TABLE_CATALOG_NAME, - InitializeSchema().release(), database) { + : AbstractCatalog(TABLE_CATALOG_OID, TABLE_CATALOG_NAME, + InitializeSchema().release(), database) { // Add indexes for pg_namespace AddIndex({0}, TABLE_CATALOG_PKEY_OID, TABLE_CATALOG_NAME "_pkey", IndexConstraintType::PRIMARY_KEY); @@ -335,14 +335,14 @@ TableCatalog::TableCatalog( * @param layout Layout object to be inserted * @return false if layout already exists in cache */ -bool TableCatalogObject::InsertLayout(std::shared_ptr layout) { - +bool TableCatalogObject::InsertLayout( + std::shared_ptr layout) { // Invalid object if (layout == nullptr) { return false; } - oid_t layout_id = layout->GetOid(); + oid_t layout_id = layout->GetOid(); // layout is already present in the cache. if (layout_objects_.find(layout_id) != layout_objects_.end()) { LOG_DEBUG("Layout %u already exists in cache!", layout_id); @@ -370,8 +370,8 @@ TableCatalogObject::GetLayouts(bool cached_only) { if (!valid_layout_objects_ && !cached_only) { // get layout catalog objects from pg_layout auto pg_layout = Catalog::GetInstance() - ->GetSystemCatalogs(database_oid) - ->GetLayoutCatalog(); + ->GetSystemCatalogs(database_oid) + ->GetLayoutCatalog(); pg_layout->GetLayouts(table_oid, txn); valid_column_objects = true; } @@ -383,8 +383,8 @@ TableCatalogObject::GetLayouts(bool cached_only) { * @param cached_only If set to true, don't fetch the layout objects. * @return Layout object of corresponding to the layout_id if present. */ -std::shared_ptr -TableCatalogObject::GetLayout(oid_t layout_id, bool cached_entry) { +std::shared_ptr TableCatalogObject::GetLayout( + oid_t layout_id, bool cached_entry) { // fetch layout objects in case we have not GetLayouts(cached_entry); auto it = layout_objects_.find(layout_id); diff --git a/src/codegen/proxy/runtime_functions_proxy.cpp b/src/codegen/proxy/runtime_functions_proxy.cpp index 5dd34f7bae2..db37ae16c25 100644 --- a/src/codegen/proxy/runtime_functions_proxy.cpp +++ b/src/codegen/proxy/runtime_functions_proxy.cpp @@ -24,8 +24,7 @@ DEFINE_TYPE(ColumnLayoutInfo, "peloton::ColumnLayoutInfo", DEFINE_TYPE(AbstractExpression, "peloton::expression::AbstractExpression", MEMBER(opaque)); -DEFINE_TYPE(Schema, "peloton::catalog::Schema", - MEMBER(opaque)); +DEFINE_TYPE(Schema, "peloton::catalog::Schema", MEMBER(opaque)); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, HashCrc64); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, GetTileGroup); diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index d26da38b88a..d7e291a0298 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -20,8 +20,8 @@ #include "expression/expression_util.h" #include "storage/data_table.h" #include "storage/layout.h" -#include "storage/tile_group.h" #include "storage/tile.h" +#include "storage/tile_group.h" #include "storage/zone_map_manager.h" #include "type/value_factory.h" @@ -102,10 +102,10 @@ void RuntimeFunctions::FillPredicateArray( // to skip over to find successive values of the column. //===----------------------------------------------------------------------===// void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, - const catalog::Schema *schema, + const catalog::Schema *schema, ColumnLayoutInfo *infos, uint32_t num_cols) { - const auto& layout = tile_group->GetLayout(); + const auto &layout = tile_group->GetLayout(); // For LayoutType::ROW, the tile group contains a single tile // and all the columns are in the same order as the table schema. if (layout.IsRowStore()) { diff --git a/src/codegen/tile_group.cpp b/src/codegen/tile_group.cpp index 8ec29f2e987..6c404b50a41 100644 --- a/src/codegen/tile_group.cpp +++ b/src/codegen/tile_group.cpp @@ -88,17 +88,15 @@ llvm::Value *TileGroup::GetTileGroupId(CodeGen &codegen, std::vector TileGroup::GetColumnLayouts( CodeGen &codegen, llvm::Value *tile_group_ptr, llvm::Value *column_layout_infos) const { - - llvm::Value *schema_ptr = codegen->CreateIntToPtr( - codegen.Const64((int64_t)&schema_), - SchemaProxy::GetType(codegen)->getPointerTo()); + llvm::Value *schema_ptr = + codegen->CreateIntToPtr(codegen.Const64((int64_t)&schema_), + SchemaProxy::GetType(codegen)->getPointerTo()); // Call RuntimeFunctions::GetTileGroupLayout() uint32_t num_cols = schema_.GetColumnCount(); - codegen.Call( - RuntimeFunctionsProxy::GetTileGroupLayout, - {tile_group_ptr, schema_ptr, column_layout_infos, - codegen.Const32(num_cols)}); + codegen.Call(RuntimeFunctionsProxy::GetTileGroupLayout, + {tile_group_ptr, schema_ptr, column_layout_infos, + codegen.Const32(num_cols)}); // Collect triplets of all columns std::vector layouts; diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index 5e595bf922f..cb32ea51fb3 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -45,7 +45,7 @@ char *Updater::GetDataPtr(uint32_t tile_group_id, uint32_t tuple_offset) { auto tile_group = table_->GetTileGroupById(tile_group_id); // Get the tile offset assuming that it is still a row store - // Hence the Tile offset is 0. + // Hence the Tile offset is 0. auto layout = tile_group->GetLayout(); PELOTON_ASSERT(layout.IsRowStore()); tile_ = tile_group->GetTileReference(0); diff --git a/src/executor/logical_tile_factory.cpp b/src/executor/logical_tile_factory.cpp index b53cf934cd1..793a22ee784 100644 --- a/src/executor/logical_tile_factory.cpp +++ b/src/executor/logical_tile_factory.cpp @@ -89,7 +89,7 @@ LogicalTile *LogicalTileFactory::WrapTileGroup( CreateIdentityPositionList(tile_group->GetActiveTupleCount())); // Construct schema. - unsigned int num_tiles = tile_group->NumTiles(); + unsigned int num_tiles = tile_group->NumTiles(); for (unsigned int i = 0; i < num_tiles; i++) { auto base_tile_ref = tile_group->GetTileReference(i); auto schema = base_tile_ref->GetSchema(); diff --git a/src/gc/gc_manager.cpp b/src/gc/gc_manager.cpp index 07bdbc34334..76fb59966ce 100644 --- a/src/gc/gc_manager.cpp +++ b/src/gc/gc_manager.cpp @@ -23,37 +23,40 @@ namespace peloton { namespace gc { // Check a tuple and reclaim all varlen field -void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id) { - oid_t tile_count = tile_group->tile_count_; - oid_t tile_col_count; - type::TypeId type_id; - char *tuple_location; - char *field_location; - char *varlen_ptr; +void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, + oid_t tuple_id) { + oid_t tile_count = tile_group->tile_count_; + oid_t tile_col_count; + type::TypeId type_id; + char *tuple_location; + char *field_location; + char *varlen_ptr; - for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { - storage::Tile *tile = tile_group->GetTile(tile_itr); - PELOTON_ASSERT(tile); - const catalog::Schema *schema = tile->GetSchema(); - tile_col_count = schema->GetColumnCount(); - for (oid_t tile_col_itr = 0; tile_col_itr < tile_col_count; ++tile_col_itr) { - type_id = schema->GetType(tile_col_itr); + for (oid_t tile_itr = 0; tile_itr < tile_count; tile_itr++) { + storage::Tile *tile = tile_group->GetTile(tile_itr); + PELOTON_ASSERT(tile); + const catalog::Schema *schema = tile->GetSchema(); + tile_col_count = schema->GetColumnCount(); + for (oid_t tile_col_itr = 0; tile_col_itr < tile_col_count; + ++tile_col_itr) { + type_id = schema->GetType(tile_col_itr); - if ((type_id != type::TypeId::VARCHAR && type_id != type::TypeId::VARBINARY) - || (schema->IsInlined(tile_col_itr) == true)) { - // Not of varlen type, or is inlined, skip - continue; - } - // Get the raw varlen pointer - tuple_location = tile->GetTupleLocation(tuple_id); - field_location = tuple_location + schema->GetOffset(tile_col_itr); - varlen_ptr = type::Value::GetDataFromStorage(type_id, field_location); - // Call the corresponding varlen pool free - if (varlen_ptr != nullptr) { - tile->pool->Free(varlen_ptr); - } - } + if ((type_id != type::TypeId::VARCHAR && + type_id != type::TypeId::VARBINARY) || + (schema->IsInlined(tile_col_itr) == true)) { + // Not of varlen type, or is inlined, skip + continue; } + // Get the raw varlen pointer + tuple_location = tile->GetTupleLocation(tuple_id); + field_location = tuple_location + schema->GetOffset(tile_col_itr); + varlen_ptr = type::Value::GetDataFromStorage(type_id, field_location); + // Call the corresponding varlen pool free + if (varlen_ptr != nullptr) { + tile->pool->Free(varlen_ptr); + } + } + } } } diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index ad744aa4251..6e3fd8e33bf 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -121,16 +121,14 @@ class Catalog { bool is_catalog = false); // Create a new layout - std::shared_ptr - CreateLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn); + std::shared_ptr CreateLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn); // Create a new layout and set it as the default for the table - std::shared_ptr - CreateDefaultLayout(oid_t database_oid, oid_t table_oid, - const column_map_type &column_map, - concurrency::TransactionContext *txn); + std::shared_ptr CreateDefaultLayout( + oid_t database_oid, oid_t table_oid, const column_map_type &column_map, + concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// // DROP FUNCTIONS diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 87c0721ffb2..1819f36f53c 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -33,17 +33,17 @@ namespace peloton { namespace storage { class Layout; -} // namespace storage +} // namespace storage namespace catalog { class LayoutCatalog : public AbstractCatalog { - -public: + public: // Global Singleton, only the first call requires passing parameters. - static LayoutCatalog *GetInstance(storage::Database *pg_catalog = nullptr, - type::AbstractPool *pool = nullptr, - concurrency::TransactionContext *txn = nullptr); + static LayoutCatalog *GetInstance( + storage::Database *pg_catalog = nullptr, + type::AbstractPool *pool = nullptr, + concurrency::TransactionContext *txn = nullptr); LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, concurrency::TransactionContext *txn); @@ -69,12 +69,10 @@ class LayoutCatalog : public AbstractCatalog { const std::unordered_map> GetLayouts(oid_t table_oid, concurrency::TransactionContext *txn); - std::shared_ptr - GetLayoutWithOid(oid_t table_oid, oid_t layout_oid, - concurrency::TransactionContext *txn); - -private: + std::shared_ptr GetLayoutWithOid( + oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn); + private: std::unique_ptr InitializeSchema(); enum ColumnId { @@ -93,6 +91,5 @@ class LayoutCatalog : public AbstractCatalog { }; }; - } // namespace catalog } // namespace peloton \ No newline at end of file diff --git a/src/include/catalog/schema.h b/src/include/catalog/schema.h index 705e7f1e2c4..43a62d6444f 100644 --- a/src/include/catalog/schema.h +++ b/src/include/catalog/schema.h @@ -65,14 +65,17 @@ class Schema : public Printable { static Schema *AppendSchema(const Schema *first, const Schema *second); // Append subset of columns in the two given schemas - static Schema *AppendSchema(const Schema *first, std::vector &first_set, - const Schema *second, std::vector &second_set); + static Schema *AppendSchema(const Schema *first, + std::vector &first_set, + const Schema *second, + std::vector &second_set); // Append given schemas. static Schema *AppendSchemaList(std::vector &schema_list); // Append given schemas. - static Schema *AppendSchemaPtrList(const std::vector &schema_list); + static Schema *AppendSchemaPtrList( + const std::vector &schema_list); // Append subsets of columns in the given schemas. static Schema *AppendSchemaPtrList( diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index ebe47e1537b..bdf900357c8 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -83,10 +83,10 @@ class TableCatalogObject { void EvictAllLayouts(); // Get layouts - std::unordered_map> - GetLayouts(bool cached_only = false); - std::shared_ptr GetLayout( - oid_t layout_id, bool cached_entry = false); + std::unordered_map> GetLayouts( + bool cached_only = false); + std::shared_ptr GetLayout(oid_t layout_id, + bool cached_entry = false); inline oid_t GetTableOid() { return table_oid; } inline const std::string &GetTableName() { return table_name; } diff --git a/src/include/codegen/proxy/runtime_functions_proxy.h b/src/include/codegen/proxy/runtime_functions_proxy.h index d264b160963..a61940cf3b3 100644 --- a/src/include/codegen/proxy/runtime_functions_proxy.h +++ b/src/include/codegen/proxy/runtime_functions_proxy.h @@ -12,8 +12,8 @@ #pragma once -#include #include +#include #include "catalog/schema.h" #include "codegen/proxy/proxy.h" #include "codegen/proxy/type_builder.h" diff --git a/src/include/gc/gc_manager.h b/src/include/gc/gc_manager.h index 213ab82fa49..1adce28a944 100644 --- a/src/include/gc/gc_manager.h +++ b/src/include/gc/gc_manager.h @@ -79,7 +79,8 @@ class GCManager { concurrency::TransactionContext *txn UNUSED_ATTRIBUTE) {} protected: - void CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id); + void CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, + oid_t tuple_id); protected: volatile bool is_running_; diff --git a/src/include/storage/abstract_table.h b/src/include/storage/abstract_table.h index b13a08d64df..6b49cf9e431 100644 --- a/src/include/storage/abstract_table.h +++ b/src/include/storage/abstract_table.h @@ -19,9 +19,9 @@ #include #include +#include "common/internal_types.h" #include "common/item_pointer.h" #include "common/printable.h" -#include "common/internal_types.h" #include "storage/layout.h" //===--------------------------------------------------------------------===// @@ -80,9 +80,7 @@ class AbstractTable : public Printable { default_layout_ = layout; } - std::shared_ptr GetDefaultLayout() { - return default_layout_; - } + std::shared_ptr GetDefaultLayout() { return default_layout_; } //===--------------------------------------------------------------------===// // TILE GROUP //===--------------------------------------------------------------------===// diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index f635a9180b0..48f36dc075d 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -18,9 +18,9 @@ #include #include +#include "common/container/lock_free_array.h" #include "common/item_pointer.h" #include "common/platform.h" -#include "common/container/lock_free_array.h" #include "storage/abstract_table.h" #include "storage/indirection_array.h" #include "storage/layout.h" @@ -257,9 +257,9 @@ class DataTable : public AbstractTable { inline void ResetDefaultLayout(LayoutType type = LayoutType::ROW) { PELOTON_ASSERT((type == LayoutType::ROW) || (type == LayoutType::COLUMN)); default_layout_ = std::shared_ptr( - new const Layout(schema->GetColumnCount(), type)); + new const Layout(schema->GetColumnCount(), type)); } - const Layout& GetDefaultLayout() const; + const Layout &GetDefaultLayout() const; //===--------------------------------------------------------------------===// // INDEX TUNER @@ -375,7 +375,6 @@ class DataTable : public AbstractTable { oid_t GetNextLayoutOid() { return ++current_layout_oid_; } private: - //===--------------------------------------------------------------------===// // STATIC MEMBERS //===--------------------------------------------------------------------===// diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index f6a9637bae6..1995685ea8e 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -36,13 +36,12 @@ namespace storage { */ class Layout : public Printable { public: - /** * @brief Constructor for predefined layout types. * @param num_columns Number of columns in the layouts. * @param layout_type Has to be LayoutType::ROW or LayoutType::COLUMN. */ - Layout (const oid_t num_columns, LayoutType layout_type = LayoutType::ROW); + Layout(const oid_t num_columns, LayoutType layout_type = LayoutType::ROW); /** * @brief Constructor for arbitrary column_maps. @@ -53,23 +52,25 @@ class Layout : public Printable { * @param column_map Column map of the layout to be constructed * */ - Layout(const column_map_type& column_map); + Layout(const column_map_type &column_map); /** * @brief Constructor for arbitrary column_maps. * @param column_map Column map of the layout to be constructed. * @param layout_oid Per-table unique OID. Generted by DataTable. */ - Layout(const column_map_type& column_map, oid_t layout_oid); + Layout(const column_map_type &column_map, oid_t layout_oid); /** @brief Check whether this layout is a row store. */ inline bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } /** @brief Check whether this layout is a column store. */ - inline bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } + inline bool IsColumnStore() const { + return (layout_type_ == LayoutType::COLUMN); + } /** @brief Return the layout_oid_ of this object. */ - oid_t GetOid() const { return layout_oid_; } + oid_t GetOid() const { return layout_oid_; } /** @brief Sets the tile id and column id w.r.t. of the tile corresponding * to the specified tile group column id. @@ -89,12 +90,13 @@ class Layout : public Printable { oid_t GetTileColumnOffset(oid_t column_id) const; /** @brief Returns the number of columns in the layout. */ - oid_t GetColumnCount() const { return num_columns_;} + oid_t GetColumnCount() const { return num_columns_; } /** @brief Constructs the schema for the given layout. Thid function * is used only in TempTables and LogicalTiles. */ - std::vector GetLayoutSchemas(catalog::Schema* const schema) const; + std::vector GetLayoutSchemas( + catalog::Schema *const schema) const; /** @brief Returns the layout statistics used by the LayoutTuner. */ std::map GetLayoutStats() const; @@ -114,25 +116,23 @@ class Layout : public Printable { //@{ /** @brief Operators for checking equality of two layouts. */ - friend bool operator==(const Layout& lhs, const Layout& rhs); - friend bool operator!=(const Layout& lhs, const Layout& rhs); + friend bool operator==(const Layout &lhs, const Layout &rhs); + friend bool operator!=(const Layout &lhs, const Layout &rhs); //@} private: - /** @brief Layout Oid of the layout object. */ oid_t layout_oid_; /** @brief Number of columns in the layout. */ - oid_t num_columns_; + oid_t num_columns_; /** @brief column_map of the layout. */ column_map_type column_layout_; /** @brief Layout type is always (ROW, COLUMN or HYBRID). */ LayoutType layout_type_; - }; -} //namespace storage -} //namespace peloton \ No newline at end of file +} // namespace storage +} // namespace peloton \ No newline at end of file diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index 7179cd1c9fb..dbf35e9f281 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -18,12 +18,12 @@ #include #include +#include "common/internal_types.h" #include "common/item_pointer.h" #include "common/printable.h" #include "planner/project_info.h" #include "storage/layout.h" #include "type/abstract_pool.h" -#include "common/internal_types.h" #include "type/value.h" namespace peloton { @@ -161,8 +161,8 @@ class TileGroup : public Printable { // Sync the contents void Sync(); - // Get the layout of the TileGroup. Used to locate columns. - const storage::Layout& GetLayout() const { return *tile_group_layout_; } + // Get the layout of the TileGroup. Used to locate columns. + const storage::Layout &GetLayout() const { return *tile_group_layout_; } protected: //===--------------------------------------------------------------------===// diff --git a/src/storage/abstract_table.cpp b/src/storage/abstract_table.cpp index 69557d847a1..893da6e5cd9 100644 --- a/src/storage/abstract_table.cpp +++ b/src/storage/abstract_table.cpp @@ -27,11 +27,11 @@ namespace storage { AbstractTable::AbstractTable(oid_t table_oid, catalog::Schema *schema, bool own_schema, peloton::LayoutType layout_type) : table_oid(table_oid), schema(schema), own_schema_(own_schema) { - // The default Layout should always be ROW or COLUMN - PELOTON_ASSERT((layout_type == LayoutType::ROW) || (layout_type == LayoutType::COLUMN)); + PELOTON_ASSERT((layout_type == LayoutType::ROW) || + (layout_type == LayoutType::COLUMN)); default_layout_ = std::shared_ptr( - new Layout(schema->GetColumnCount(), layout_type)); + new Layout(schema->GetColumnCount(), layout_type)); } AbstractTable::~AbstractTable() { @@ -41,15 +41,12 @@ AbstractTable::~AbstractTable() { TileGroup *AbstractTable::GetTileGroupWithLayout( oid_t database_id, oid_t tile_group_id, - std::shared_ptr layout, - const size_t num_tuples) { - + std::shared_ptr layout, const size_t num_tuples) { // Populate the schema for each tile std::vector schemas = layout->GetLayoutSchemas(schema); - TileGroup *tile_group = - TileGroupFactory::GetTileGroup(database_id, GetOid(), tile_group_id, this, - schemas, layout, num_tuples); + TileGroup *tile_group = TileGroupFactory::GetTileGroup( + database_id, GetOid(), tile_group_id, this, schemas, layout, num_tuples); return tile_group; } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index cffa7900429..e81ecc4a6ee 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -15,8 +15,8 @@ #include "catalog/catalog.h" #include "catalog/foreign_key.h" -#include "catalog/system_catalogs.h" #include "catalog/layout_catalog.h" +#include "catalog/system_catalogs.h" #include "catalog/table_catalog.h" #include "common/container_tuple.h" #include "common/exception.h" @@ -871,8 +871,8 @@ void DataTable::ResetDirty() { dirty_ = false; } TileGroup *DataTable::GetTileGroupWithLayout( std::shared_ptr layout) { oid_t tile_group_id = catalog::Manager::GetInstance().GetNextTileGroupId(); - return (AbstractTable::GetTileGroupWithLayout( - database_oid, tile_group_id, layout, tuples_per_tilegroup_)); + return (AbstractTable::GetTileGroupWithLayout(database_oid, tile_group_id, + layout, tuples_per_tilegroup_)); } oid_t DataTable::AddDefaultIndirectionArray( @@ -900,7 +900,8 @@ oid_t DataTable::AddDefaultTileGroup(const size_t &active_tile_group_id) { oid_t tile_group_id = INVALID_OID; // Create a tile group with that partitioning - std::shared_ptr tile_group(GetTileGroupWithLayout(default_layout_)); + std::shared_ptr tile_group( + GetTileGroupWithLayout(default_layout_)); PELOTON_ASSERT(tile_group.get()); tile_group_id = tile_group->GetTileGroupId(); @@ -940,7 +941,7 @@ void DataTable::AddTileGroupWithOidForRecovery(const oid_t &tile_group_id) { layout = default_layout_; } else { layout = std::shared_ptr( - new const Layout(schema->GetColumnCount())); + new const Layout(schema->GetColumnCount())); } std::shared_ptr tile_group(TileGroupFactory::GetTileGroup( @@ -1192,7 +1193,7 @@ catalog::ForeignKey *DataTable::GetForeignKeySrc(const size_t offset) const { // Get the schema for the new transformed tile group std::vector TransformTileGroupSchema( - storage::TileGroup *tile_group, const Layout& layout) { + storage::TileGroup *tile_group, const Layout &layout) { std::vector new_schema; oid_t orig_tile_offset, orig_tile_column_offset; oid_t new_tile_offset, new_tile_column_offset; @@ -1207,8 +1208,7 @@ std::vector TransformTileGroupSchema( tile_group_layout.LocateTileAndColumn(col_id, orig_tile_offset, orig_tile_column_offset); // Get new layout's tile and offset for col_id. - layout.LocateTileAndColumn(col_id, new_tile_offset, - new_tile_column_offset); + layout.LocateTileAndColumn(col_id, new_tile_offset, new_tile_column_offset); // Get the column info from original tile auto tile = tile_group->GetTile(orig_tile_offset); @@ -1234,7 +1234,6 @@ std::vector TransformTileGroupSchema( // Set the transformed tile group column-at-a-time void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, storage::TileGroup *new_tile_group) { - auto new_layout = new_tile_group->GetLayout(); auto orig_layout = orig_tile_group->GetLayout(); @@ -1254,10 +1253,10 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, for (oid_t column_itr = 0; column_itr < column_count; column_itr++) { // Locate the original base tile and tile column offset orig_layout.LocateTileAndColumn(column_itr, orig_tile_offset, - orig_tile_column_offset); + orig_tile_column_offset); new_layout.LocateTileAndColumn(column_itr, new_tile_offset, - new_tile_column_offset); + new_tile_column_offset); auto orig_tile = orig_tile_group->GetTile(orig_tile_offset); auto new_tile = new_tile_group->GetTile(new_tile_offset); @@ -1308,8 +1307,7 @@ storage::TileGroup *DataTable::TransformTileGroup( TileGroupFactory::GetTileGroup( tile_group->GetDatabaseId(), tile_group->GetTableId(), tile_group->GetTileGroupId(), tile_group->GetAbstractTable(), - new_schema, default_layout_, - tile_group->GetAllocatedTupleCount())); + new_schema, default_layout_, tile_group->GetAllocatedTupleCount())); // Set the transformed tile group column-at-a-time SetTransformedTileGroup(tile_group.get(), new_tile_group.get()); @@ -1367,9 +1365,7 @@ void DataTable::ClearIndexSamples() { } } -const Layout& DataTable::GetDefaultLayout() const { - return *default_layout_; -} +const Layout &DataTable::GetDefaultLayout() const { return *default_layout_; } void DataTable::AddTrigger(trigger::Trigger new_trigger) { trigger_list_->AddTrigger(new_trigger); @@ -1421,8 +1417,7 @@ bool DataTable::operator==(const DataTable &rhs) const { bool DataTable::SetCurrentLayoutOid(oid_t new_layout_oid) { oid_t old_oid = current_layout_oid_; while (old_oid <= new_layout_oid) { - if (current_layout_oid_.compare_exchange_strong( - old_oid, new_layout_oid)) { + if (current_layout_oid_.compare_exchange_strong(old_oid, new_layout_oid)) { return true; } old_oid = current_layout_oid_; @@ -1430,5 +1425,5 @@ bool DataTable::SetCurrentLayoutOid(oid_t new_layout_oid) { return false; } -} // End storage namespace -} // End peloton namespace +} // namespace storage +} // namespace peloton diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 95d54407db5..c8d96c4c382 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -10,8 +10,8 @@ // //===----------------------------------------------------------------------===// -#include #include +#include #include "catalog/column.h" #include "catalog/schema.h" @@ -24,13 +24,11 @@ namespace storage { // Constructor for the layout class with column_count // The default layout is always a ROW_STORE Layout::Layout(const oid_t num_columns, LayoutType layout_type) - : num_columns_(num_columns), - layout_type_(layout_type){} + : num_columns_(num_columns), layout_type_(layout_type) {} // Constructor for the Layout class with column_map -Layout::Layout(const column_map_type& column_map) - : num_columns_(column_map.size()), - column_layout_(column_map) { +Layout::Layout(const column_map_type &column_map) + : num_columns_(column_map.size()), column_layout_(column_map) { // Figure out the layout type if not present bool row_layout = true, column_layout = true; for (oid_t column_id = 0; column_id < num_columns_; column_id++) { @@ -48,12 +46,12 @@ Layout::Layout(const column_map_type& column_map) layout_type_ = LayoutType::ROW; layout_oid_ = ROW_STORE_OID; } else if (column_layout) { - layout_type_ = LayoutType::COLUMN; + layout_type_ = LayoutType::COLUMN; layout_oid_ = COLUMN_STORE_OID; } else { // layout_oid_ is set to INVALID_OID, indicating that this // layout is not stored in the catalog and thus not persistent. - // To be used only in TempTable or Tests. + // To be used only in TempTable or Tests. layout_type_ = LayoutType::HYBRID; layout_oid_ = INVALID_OID; } @@ -66,10 +64,9 @@ Layout::Layout(const column_map_type& column_map) // Constructor for Layout class with predefined layout_oid Layout::Layout(const column_map_type &column_map, oid_t layout_id) - : layout_oid_(layout_id), - num_columns_(column_map.size()), - column_layout_(column_map) { - + : layout_oid_(layout_id), + num_columns_(column_map.size()), + column_layout_(column_map) { if (layout_oid_ == ROW_STORE_OID) { layout_type_ = LayoutType::ROW; } else if (layout_oid_ == COLUMN_STORE_OID) { @@ -84,22 +81,20 @@ Layout::Layout(const column_map_type &column_map, oid_t layout_id) } } - // Sets the tile id and column id w.r.t that tile corresponding to // the specified tile group column id. -void Layout::LocateTileAndColumn(oid_t column_offset, - oid_t &tile_offset, oid_t &tile_column_offset) const { - - // Ensure that the column_offset is not out of bound - PELOTON_ASSERT(num_columns_ > column_offset); +void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, + oid_t &tile_column_offset) const { + // Ensure that the column_offset is not out of bound + PELOTON_ASSERT(num_columns_ > column_offset); - // For row store layout, tile id is always 0 and the tile - // column_id and tile column_id is the same. - if (layout_type_ == LayoutType::ROW) { - tile_offset = 0; - tile_column_offset = column_offset; - return; - } + // For row store layout, tile id is always 0 and the tile + // column_id and tile column_id is the same. + if (layout_type_ == LayoutType::ROW) { + tile_offset = 0; + tile_column_offset = column_offset; + return; + } // For column store layout, tile_id is always same as column_id // and the tile column_id is always 0. @@ -109,11 +104,11 @@ void Layout::LocateTileAndColumn(oid_t column_offset, return; } - // For other layouts, fetch the layout and - // get the entry in the column map - auto entry = column_layout_.at(column_offset); - tile_offset = entry.first; - tile_column_offset = entry.second; + // For other layouts, fetch the layout and + // get the entry in the column map + auto entry = column_layout_.at(column_offset); + tile_offset = entry.first; + tile_column_offset = entry.second; } double Layout::GetLayoutDifference(const storage::Layout &other) const { @@ -125,11 +120,9 @@ double Layout::GetLayoutDifference(const storage::Layout &other) const { PELOTON_ASSERT(this->num_columns_ == other.num_columns_); if ((this->layout_oid_ != other.layout_oid_)) { - for (oid_t col_itr = 0; col_itr < num_columns_; col_itr++) { - if (this->GetTileIdFromColumnId(col_itr) != - other.GetTileIdFromColumnId(col_itr)) { + other.GetTileIdFromColumnId(col_itr)) { diff++; } } @@ -153,8 +146,7 @@ oid_t Layout::GetTileColumnOffset(oid_t column_id) const { } std::vector Layout::GetLayoutSchemas( - catalog::Schema* const schema) const { - + catalog::Schema *const schema) const { std::vector schemas; // Build the schema tile at a time @@ -174,7 +166,7 @@ std::vector Layout::GetLayoutSchemas( // as the schema. This snipet was initially in abstract_table.cpp. for (auto column_info : column_layout_) { tile_schemas[column_info.second.first].push_back( - schema->GetColumn(column_info.first)); + schema->GetColumn(column_info.first)); } for (auto entry : tile_schemas) { @@ -267,7 +259,7 @@ std::string Layout::GetColumnMapInfo() const { for (auto column_info : column_layout_) { oid_t col_id = column_info.first; oid_t tile_id = column_info.second.first; - if(tile_column_map.find(tile_id) == tile_column_map.end()) { + if (tile_column_map.find(tile_id) == tile_column_map.end()) { tile_column_map[tile_id] = {}; } tile_column_map[tile_id].push_back(col_id); @@ -299,8 +291,7 @@ const std::string Layout::GetInfo() const { GETINFO_SPACER); } -bool operator==(const Layout& lhs, const Layout& rhs) { - +bool operator==(const Layout &lhs, const Layout &rhs) { // Check the equality of layout_type_ if (lhs.layout_type_ != rhs.layout_type_) { return false; @@ -317,21 +308,16 @@ bool operator==(const Layout& lhs, const Layout& rhs) { // Check for the equality of the column_layout_ for // LayoutType::HYBRID - if (lhs.layout_type_ == LayoutType::HYBRID) - { + if (lhs.layout_type_ == LayoutType::HYBRID) { return (lhs.column_layout_ == rhs.column_layout_); } // In case of LayoutType::ROW or LayoutType::COLUMN, // there is no need to check for column_layout_ equality return true; - -} - -bool operator!=(const Layout& lhs, const Layout& rhs) { - return !(lhs == rhs); } +bool operator!=(const Layout &lhs, const Layout &rhs) { return !(lhs == rhs); } -} // namespace storage -} // namespace peloton \ No newline at end of file +} // namespace storage +} // namespace peloton \ No newline at end of file diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index ada74b154a9..6213a5f540e 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -16,9 +16,9 @@ #include "catalog/manager.h" #include "common/container_tuple.h" +#include "common/internal_types.h" #include "common/logger.h" #include "common/platform.h" -#include "common/internal_types.h" #include "storage/abstract_table.h" #include "storage/layout.h" #include "storage/tile.h" @@ -40,7 +40,7 @@ TileGroup::TileGroup(BackendType backend_type, tile_group_header(tile_group_header), table(table), num_tuple_slots(tuple_count), - tile_group_layout_(layout){ + tile_group_layout_(layout) { tile_count_ = schemas.size(); for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { auto &manager = catalog::Manager::GetInstance(); @@ -288,7 +288,6 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, oid_t column_itr = 0; for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { - storage::Tile *tile = GetTile(tile_itr); PELOTON_ASSERT(tile); const catalog::Schema *schema = tile->GetSchema(); @@ -319,8 +318,8 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, type::Value TileGroup::GetValue(oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_->LocateTileAndColumn(column_id, - tile_offset, tile_column_id); + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, + tile_column_id); return GetTile(tile_offset)->GetValue(tuple_id, tile_column_id); } @@ -328,8 +327,8 @@ void TileGroup::SetValue(type::Value &value, oid_t tuple_id, oid_t column_id) { PELOTON_ASSERT(tuple_id < GetNextTupleSlot()); oid_t tile_column_id, tile_offset; - tile_group_layout_->LocateTileAndColumn(column_id, - tile_offset, tile_column_id); + tile_group_layout_->LocateTileAndColumn(column_id, tile_offset, + tile_column_id); GetTile(tile_offset)->SetValue(value, tuple_id, tile_column_id); } diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index db3e02fcf62..df64ea38bb4 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -116,8 +116,8 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto *txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - if (catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn) - == nullptr) { + if (catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn) == + nullptr) { txn_manager.AbortTransaction(txn); LOG_DEBUG("Layout Update to failed."); return; diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index 31283e80dd8..f99f11f1a40 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -22,9 +22,9 @@ #include "common/harness.h" #include "common/logger.h" #include "concurrency/transaction_manager_factory.h" +#include "sql/testing_sql_util.h" #include "storage/storage_manager.h" #include "type/ephemeral_pool.h" -#include "sql/testing_sql_util.h" namespace peloton { namespace test { @@ -236,7 +236,8 @@ TEST_F(CatalogTests, DroppingTable) { auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); // NOTE: everytime we create a database, there will be 9 catalog tables inside - EXPECT_EQ(CATALOG_TABLES_COUNT+3, + EXPECT_EQ( + CATALOG_TABLES_COUNT + 3, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); @@ -249,7 +250,8 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_NE(nullptr, database_object); auto department_table_object = database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); - EXPECT_EQ(CATALOG_TABLES_COUNT+2, + EXPECT_EQ( + CATALOG_TABLES_COUNT + 2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -261,7 +263,8 @@ TEST_F(CatalogTests, DroppingTable) { "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), CatalogException); // - EXPECT_EQ(CATALOG_TABLES_COUNT+2, + EXPECT_EQ( + CATALOG_TABLES_COUNT + 2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -270,7 +273,8 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), CatalogException); - EXPECT_EQ(CATALOG_TABLES_COUNT+2, + EXPECT_EQ( + CATALOG_TABLES_COUNT + 2, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -278,7 +282,8 @@ TEST_F(CatalogTests, DroppingTable) { txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "emp_table", txn); - EXPECT_EQ(CATALOG_TABLES_COUNT+1, + EXPECT_EQ( + CATALOG_TABLES_COUNT + 1, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); } @@ -315,27 +320,27 @@ TEST_F(CatalogTests, LayoutCatalogTest) { catalog->CreateDatabase(db_name, txn); // Create table. - auto val0 = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "val0", true); - auto val1 = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "val1", true); - auto val2 = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "val2", true); - auto val3 = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "val3", true); + auto val0 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val0", true); + auto val1 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val1", true); + auto val2 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val2", true); + auto val3 = catalog::Column(type::TypeId::INTEGER, + type::Type::GetTypeSize(type::TypeId::INTEGER), + "val3", true); std::unique_ptr table_schema( - new catalog::Schema({val0, val1, val2, val3})); - catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, - table_name, std::move(table_schema), txn); + new catalog::Schema({val0, val1, val2, val3})); + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, - table_name, txn); + auto table = + catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); auto table_oid = table->GetOid(); auto database_oid = table->GetDatabaseOid(); txn_manager.CommitTransaction(txn); @@ -349,8 +354,8 @@ TEST_F(CatalogTests, LayoutCatalogTest) { default_map[3] = std::make_pair(1, 1); txn = txn_manager.BeginTransaction(); - auto default_layout = catalog->CreateDefaultLayout(database_oid, table_oid, - default_map, txn); + auto default_layout = + catalog->CreateDefaultLayout(database_oid, table_oid, default_map, txn); txn_manager.CommitTransaction(txn); EXPECT_NE(nullptr, default_layout); @@ -362,8 +367,8 @@ TEST_F(CatalogTests, LayoutCatalogTest) { non_default_map[3] = std::make_pair(1, 1); txn = txn_manager.BeginTransaction(); - auto other_layout = catalog->CreateLayout(database_oid, table_oid, - non_default_map, txn); + auto other_layout = + catalog->CreateLayout(database_oid, table_oid, non_default_map, txn); txn_manager.CommitTransaction(txn); // Check that the default layout is still the same. @@ -372,8 +377,8 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Drop the default layout. auto default_layout_oid = default_layout->GetOid(); txn = txn_manager.BeginTransaction(); - EXPECT_EQ(ResultType::SUCCESS, catalog->DropLayout( - database_oid, table_oid, default_layout_oid, txn)); + EXPECT_EQ(ResultType::SUCCESS, catalog->DropLayout(database_oid, table_oid, + default_layout_oid, txn)); txn_manager.CommitTransaction(txn); // Check that default layout is reset and set to row_store. @@ -382,15 +387,15 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Query pg_layout to ensure that the entry is dropped txn = txn_manager.BeginTransaction(); - auto pg_layout = catalog-> - GetSystemCatalogs(database_oid)->GetLayoutCatalog(); + auto pg_layout = catalog->GetSystemCatalogs(database_oid)->GetLayoutCatalog(); EXPECT_EQ(nullptr, pg_layout->GetLayoutWithOid(table_oid, default_layout_oid, txn)); // The additional layout must be present in pg_layout auto other_layout_oid = other_layout->GetOid(); - EXPECT_EQ(*(other_layout.get()), *(pg_layout - ->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); + EXPECT_EQ( + *(other_layout.get()), + *(pg_layout->GetLayoutWithOid(table_oid, other_layout_oid, txn).get())); txn_manager.CommitTransaction(txn); // Drop database diff --git a/test/catalog/manager_test.cpp b/test/catalog/manager_test.cpp index f0556f01734..7b5ec2161ac 100644 --- a/test/catalog/manager_test.cpp +++ b/test/catalog/manager_test.cpp @@ -13,9 +13,9 @@ #include "common/harness.h" -#include "common/macros.h" #include "catalog/manager.h" #include "catalog/schema.h" +#include "common/macros.h" #include "storage/layout.h" #include "storage/tile_group.h" #include "storage/tile_group_factory.h" @@ -50,11 +50,13 @@ void AddTileGroup(UNUSED_ATTRIBUTE uint64_t thread_id) { schemas.push_back(*schema1); std::shared_ptr layout = - std::make_shared(columns.size()); + std::make_shared(columns.size()); for (oid_t txn_itr = 0; txn_itr < 100; txn_itr++) { - std::unique_ptr tile_group(storage::TileGroupFactory::GetTileGroup( - INVALID_OID, INVALID_OID, INVALID_OID, nullptr, schemas, layout, 3)); + std::unique_ptr tile_group( + storage::TileGroupFactory::GetTileGroup(INVALID_OID, INVALID_OID, + INVALID_OID, nullptr, schemas, + layout, 3)); } } diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 52b18d5d139..f0373745192 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -28,7 +28,7 @@ namespace test { class TableScanTranslatorTest : public PelotonCodeGenTest { std::string all_cols_table_name = "crazy_table"; -public: + public: TableScanTranslatorTest() : PelotonCodeGenTest(), num_rows_to_insert(64) { // Load test table LoadTestTable(TestTableId(), num_rows_to_insert); @@ -37,14 +37,12 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { } void ExecuteTileGroupTest(peloton::LayoutType layout_type) { - - const int tuples_per_tilegroup= 100; + const int tuples_per_tilegroup = 100; const int tile_group_count = 5; const int tuple_count = tuples_per_tilegroup * tile_group_count; const oid_t col_count = 100; const bool is_inlined = true; - ///////////////////////////////////////////////////////// // Define the schema. ///////////////////////////////////////////////////////// @@ -52,16 +50,15 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { std::vector columns; for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { - auto column = - catalog::Column(type::TypeId::INTEGER, - type::Type::GetTypeSize(type::TypeId::INTEGER), - "FIELD" + std::to_string(col_itr), is_inlined); + auto column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); columns.push_back(column); } std::unique_ptr table_schema = - std::unique_ptr(new catalog::Schema(columns)); + std::unique_ptr(new catalog::Schema(columns)); std::string table_name("TEST_TABLE"); ///////////////////////////////////////////////////////// @@ -79,8 +76,7 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { std::move(table_schema), txn, is_catalog, tuples_per_tilegroup, layout_type); // Get table reference - auto table = catalog->GetTableWithName(test_db_name, - DEFAULT_SCHEMA_NAME, + auto table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.EndTransaction(txn); @@ -100,13 +96,14 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { storage::Tuple tuple(table_schema_ptr, allocate); for (oid_t col_id = 0; col_id <= col_count; col_id++) { - auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + auto value = + type::ValueFactory::GetIntegerValue(populate_value + col_id); tuple.SetValue(col_id, value, testing_pool); } ItemPointer *index_entry_ptr = nullptr; ItemPointer tuple_slot_id = - table->InsertTuple(&tuple, txn, &index_entry_ptr); + table->InsertTuple(&tuple, txn, &index_entry_ptr); EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); @@ -122,11 +119,10 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { // Column ids to be scanned. std::vector column_ids; - for(oid_t col_id = 0 ; col_id < col_count; col_id++) { + for (oid_t col_id = 0; col_id < col_count; col_id++) { column_ids.push_back(col_id); } - // Setup the scan plan node planner::SeqScanPlan scan(table, nullptr, column_ids); @@ -137,7 +133,6 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { // Printing consumer codegen::BufferingConsumer buffer{column_ids, context}; - // COMPILE and execute CompileAndExecute(scan, buffer); @@ -149,8 +144,10 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { auto &tuple = results[tuple_id]; int tuple_id_value = tuple_id; for (oid_t col_id = 0; col_id < col_count; col_id++) { - auto value = type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); - EXPECT_EQ(CmpBool::CmpTrue, tuple.GetValue(col_id).CompareEquals(value)); + auto value = + type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); + EXPECT_EQ(CmpBool::CmpTrue, + tuple.GetValue(col_id).CompareEquals(value)); } } } @@ -730,12 +727,11 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { // invokes the TableScanTranslator // - const int tuples_per_tilegroup= 100; + const int tuples_per_tilegroup = 100; const int col_count = 6; const bool is_inlined = true; oid_t tuple_count = 100; - ///////////////////////////////////////////////////////// // Define the schema. ///////////////////////////////////////////////////////// @@ -743,16 +739,15 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { std::vector columns; for (oid_t col_itr = 0; col_itr < col_count; col_itr++) { - auto column = - catalog::Column(type::TypeId::INTEGER, - type::Type::GetTypeSize(type::TypeId::INTEGER), - "FIELD" + std::to_string(col_itr), is_inlined); + auto column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); columns.push_back(column); } std::unique_ptr table_schema = - std::unique_ptr(new catalog::Schema(columns)); + std::unique_ptr(new catalog::Schema(columns)); std::string table_name("MULTI_LAYOUT_TABLE"); ///////////////////////////////////////////////////////// @@ -770,8 +765,8 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { std::move(table_schema), txn, is_catalog, tuples_per_tilegroup, LayoutType::ROW); // Get table reference - auto table = catalog->GetTableWithName(test_db_name, - DEFAULT_SCHEMA_NAME, table_name, txn); + auto table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); txn_manager.EndTransaction(txn); ///////////////////////////////////////////////////////// @@ -798,7 +793,7 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { ItemPointer *index_entry_ptr = nullptr; ItemPointer tuple_slot_id = - table->InsertTuple(&tuple, txn, &index_entry_ptr); + table->InsertTuple(&tuple, txn, &index_entry_ptr); EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); @@ -824,8 +819,8 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { auto table_oid = table->GetOid(); txn = txn_manager.BeginTransaction(); - auto layout = catalog->CreateDefaultLayout(database_oid, table_oid, - column_map, txn); + auto layout = + catalog->CreateDefaultLayout(database_oid, table_oid, column_map, txn); EXPECT_NE(nullptr, layout); txn_manager.CommitTransaction(txn); @@ -847,7 +842,7 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { ItemPointer *index_entry_ptr = nullptr; ItemPointer tuple_slot_id = - table->InsertTuple(&tuple, txn, &index_entry_ptr); + table->InsertTuple(&tuple, txn, &index_entry_ptr); EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); @@ -865,11 +860,10 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { tuple_count = tuple_count + prev_tuple_count; // Column ids to be scanned. std::vector column_ids; - for(oid_t col_id = 0 ; col_id < col_count; col_id++) { + for (oid_t col_id = 0; col_id < col_count; col_id++) { column_ids.push_back(col_id); } - // Setup the scan plan node planner::SeqScanPlan scan(table, nullptr, column_ids); @@ -880,7 +874,6 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { // Printing consumer codegen::BufferingConsumer buffer{column_ids, context}; - // COMPILE and execute CompileAndExecute(scan, buffer); diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index f780b506b81..05f0a61d628 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -109,14 +109,16 @@ TEST_F(DropTests, DroppingTable) { // NOTE: everytime we create a database, there will be 8 catalog tables inside EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() - .size(), (CATALOG_TABLES_COUNT + 2)); + .size(), + (CATALOG_TABLES_COUNT + 2)); // Now dropping the table using the executor catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() - .size(), (CATALOG_TABLES_COUNT + 1)); + .size(), + (CATALOG_TABLES_COUNT + 1)); // free the database just created catalog->DropDatabaseWithName(TEST_DB_NAME, txn); @@ -213,9 +215,9 @@ TEST_F(DropTests, DroppingTrigger) { catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); EXPECT_EQ(CATALOG_TABLES_COUNT, (int)catalog::Catalog::GetInstance() - ->GetDatabaseObject(TEST_DB_NAME, txn) - ->GetTableObjects() - .size()); + ->GetDatabaseObject(TEST_DB_NAME, txn) + ->GetTableObjects() + .size()); txn_manager.CommitTransaction(txn); // free the database just created diff --git a/test/executor/logical_tile_test.cpp b/test/executor/logical_tile_test.cpp index ee2e47d26fe..0f225e65963 100644 --- a/test/executor/logical_tile_test.cpp +++ b/test/executor/logical_tile_test.cpp @@ -103,7 +103,6 @@ TEST_F(LogicalTileTests, TileMaterializationTest) { tile_schemas.push_back(tile_group->GetTile(i)->GetSchema()); } - // Create tuple schema from tile schemas. std::unique_ptr schema( catalog::Schema::AppendSchemaPtrList(tile_schemas)); diff --git a/test/executor/seq_scan_test.cpp b/test/executor/seq_scan_test.cpp index d9e9d0dbf27..0649aa38e63 100644 --- a/test/executor/seq_scan_test.cpp +++ b/test/executor/seq_scan_test.cpp @@ -84,8 +84,7 @@ storage::DataTable *CreateTable() { column_map1[2] = std::make_pair(1, 0); column_map1[3] = std::make_pair(1, 1); std::shared_ptr layout1 = - std::make_shared(column_map1); - + std::make_shared(column_map1); std::map> column_map2; column_map2[0] = std::make_pair(0, 0); @@ -93,8 +92,7 @@ storage::DataTable *CreateTable() { column_map2[2] = std::make_pair(1, 1); column_map2[3] = std::make_pair(1, 2); std::shared_ptr layout2 = - std::make_shared(column_map2); - + std::make_shared(column_map2); // Create tile groups. table->AddTileGroup(std::shared_ptr( diff --git a/test/executor/testing_executor_util.cpp b/test/executor/testing_executor_util.cpp index d592f58f6a7..cd923c11ed8 100644 --- a/test/executor/testing_executor_util.cpp +++ b/test/executor/testing_executor_util.cpp @@ -172,7 +172,8 @@ std::shared_ptr TestingExecutorUtil::CreateTileGroup( column_map[3] = std::make_pair(1, 1); std::shared_ptr layout = - std::shared_ptr(new const storage::Layout(column_map)); + std::shared_ptr( + new const storage::Layout(column_map)); std::shared_ptr tile_group_ptr( storage::TileGroupFactory::GetTileGroup( @@ -257,7 +258,7 @@ void TestingExecutorUtil::PopulateTable(storage::DataTable *table, int num_rows, */ void TestingExecutorUtil::PopulateTiles( std::shared_ptr tile_group, int num_rows) { - size_t tile_count = tile_group->GetTileCount(); + size_t tile_count = tile_group->GetTileCount(); // Create tuple schema from tile schemas. std::vector tile_schemas; for (oid_t tile_id = 0; tile_id < tile_count; tile_id++) { @@ -420,9 +421,10 @@ storage::DataTable *TestingExecutorUtil::CreateTable( } storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( - int tuples_per_tilegroup_count, std::string &db_name) { - auto table_schema = std::unique_ptr(new catalog::Schema( - {GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), GetColumnInfo(3)})); + int tuples_per_tilegroup_count, std::string &db_name) { + auto table_schema = std::unique_ptr( + new catalog::Schema({GetColumnInfo(0), GetColumnInfo(1), GetColumnInfo(2), + GetColumnInfo(3)})); std::string table_name("test_table"); bool is_catalog = false; @@ -436,8 +438,8 @@ storage::DataTable *TestingExecutorUtil::CreateTableUpdateCatalog( txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - auto table = catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, - table_name, txn); + auto table = + catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); txn_manager.CommitTransaction(txn); return table; diff --git a/test/executor/update_test.cpp b/test/executor/update_test.cpp index 6ade606efb4..04c42162cdc 100644 --- a/test/executor/update_test.cpp +++ b/test/executor/update_test.cpp @@ -91,8 +91,8 @@ storage::DataTable *CreateTable() { column_map1[3] = std::make_pair(1, 1); std::shared_ptr layout1 = - std::shared_ptr( - new const storage::Layout(column_map1)); + std::shared_ptr( + new const storage::Layout(column_map1)); std::map> column_map2; column_map2[0] = std::make_pair(0, 0); @@ -101,8 +101,8 @@ storage::DataTable *CreateTable() { column_map2[3] = std::make_pair(1, 2); std::shared_ptr layout2 = - std::shared_ptr( - new const storage::Layout(column_map1)); + std::shared_ptr( + new const storage::Layout(column_map1)); // Create tile groups. table->AddTileGroup(std::shared_ptr( diff --git a/test/include/executor/testing_executor_util.h b/test/include/executor/testing_executor_util.h index 10459c6c8fc..5d9dc14b5dd 100644 --- a/test/include/executor/testing_executor_util.h +++ b/test/include/executor/testing_executor_util.h @@ -81,7 +81,7 @@ class TestingExecutorUtil { * @return A pointer to the DataTable created. */ static storage::DataTable *CreateTableUpdateCatalog( - int tuples_per_tilegroup_count, std::string &db_name); + int tuples_per_tilegroup_count, std::string &db_name); /** @brief Creates a basic table with allocated and populated tuples */ static storage::DataTable *CreateAndPopulateTable(); diff --git a/test/statistics/stats_test.cpp b/test/statistics/stats_test.cpp index 77d4a65361d..ef3c7da6cba 100644 --- a/test/statistics/stats_test.cpp +++ b/test/statistics/stats_test.cpp @@ -136,9 +136,9 @@ TEST_F(StatsTests, MultiThreadStatsTest) { std::unique_ptr table_schema( new catalog::Schema({id_column, name_column})); catalog->CreateDatabase("emp_db", txn); - catalog::Catalog::GetInstance()->CreateTable( - "emp_db", DEFAULT_SCHEMA_NAME, "department_table", - std::move(table_schema), txn); + catalog::Catalog::GetInstance()->CreateTable("emp_db", DEFAULT_SCHEMA_NAME, + "department_table", + std::move(table_schema), txn); // Create multiple stat worker threads int num_threads = 8; diff --git a/test/storage/tile_group_test.cpp b/test/storage/tile_group_test.cpp index 36c7a998f77..4aca1ecb536 100644 --- a/test/storage/tile_group_test.cpp +++ b/test/storage/tile_group_test.cpp @@ -85,7 +85,7 @@ TEST_F(TileGroupTests, BasicTest) { column_map[3] = std::make_pair(1, 1); std::shared_ptr layout = - std::make_shared(column_map); + std::make_shared(column_map); std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( @@ -216,7 +216,7 @@ TEST_F(TileGroupTests, StressTest) { column_map[3] = std::make_pair(1, 1); std::shared_ptr layout = - std::make_shared(column_map); + std::make_shared(column_map); std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( @@ -238,11 +238,14 @@ TEST_F(TileGroupTests, StressTest) { // std::vector schemas; // // // SCHEMA -// catalog::Column column1(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), +// catalog::Column column1(type::TypeId::INTEGER, +// type::Type::GetTypeSize(type::TypeId::INTEGER), // "A", true); -// catalog::Column column2(type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), +// catalog::Column column2(type::TypeId::INTEGER, +// type::Type::GetTypeSize(type::TypeId::INTEGER), // "B", true); -// catalog::Column column3(type::TypeId::TINYINT, type::Type::GetTypeSize(type::TypeId::TINYINT), +// catalog::Column column3(type::TypeId::TINYINT, +// type::Type::GetTypeSize(type::TypeId::TINYINT), // "C", true); // catalog::Column column4(type::TypeId::VARCHAR, 50, "D", false); // @@ -381,9 +384,8 @@ TEST_F(TileGroupTests, TileCopyTest) { const int tuple_count = 4; - std::shared_ptr layout = - std::make_shared(schema->GetColumnCount()); + std::make_shared(schema->GetColumnCount()); std::shared_ptr tile_group( storage::TileGroupFactory::GetTileGroup( diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index 42ed404e112..adabf72a222 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -41,14 +41,14 @@ TEST_F(LayoutTunerTests, BasicTest) { std::string db_name = "test_db"; TestingExecutorUtil::InitializeDatabase(db_name); - auto data_table = TestingExecutorUtil::CreateTableUpdateCatalog(tuple_count, db_name); - + auto data_table = + TestingExecutorUtil::CreateTableUpdateCatalog(tuple_count, db_name); // Create a table and populate it auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, - false, true, txn); + TestingExecutorUtil::PopulateTable(data_table, tuple_count, false, false, + true, txn); txn_manager.CommitTransaction(txn); // Check column count @@ -63,7 +63,7 @@ TEST_F(LayoutTunerTests, BasicTest) { // Check old default tile group layout auto old_default_layout = data_table->GetDefaultLayout(); - LOG_INFO("Layout: %s",old_default_layout.GetColumnMapInfo().c_str()); + LOG_INFO("Layout: %s", old_default_layout.GetColumnMapInfo().c_str()); // Start layout tuner layout_tuner.Start(); @@ -118,12 +118,11 @@ TEST_F(LayoutTunerTests, BasicTest) { column_count = new_default_layout.GetColumnCount(); EXPECT_EQ(column_count, 4); - // Check the tile corresponding to each column. - EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(0),0); - EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(1),0); - EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2),0); - EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3),1); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(0), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(1), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(2), 0); + EXPECT_EQ(new_default_layout.GetTileIdFromColumnId(3), 1); // Check the per tile stats of the new layout // The layout must contain 2 tiles with the following stats From c219d7ba20035a779275e2a30d7d4f144ddeb953 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sat, 5 May 2018 04:01:20 -0400 Subject: [PATCH 36/42] Address review comments + minor clean-up in the layout.h API --- src/catalog/catalog.cpp | 29 +--------- src/catalog/table_catalog.cpp | 2 +- src/codegen/proxy/runtime_functions_proxy.cpp | 3 +- src/codegen/runtime_functions.cpp | 56 +++++++++---------- src/codegen/tile_group.cpp | 9 +-- src/include/catalog/catalog.h | 30 +++++++++- src/include/catalog/catalog_defaults.h | 2 +- src/include/catalog/layout_catalog.h | 5 -- .../codegen/proxy/runtime_functions_proxy.h | 1 - src/include/codegen/runtime_functions.h | 1 - src/include/common/internal_types.h | 5 ++ src/include/storage/layout.h | 5 +- src/storage/data_table.cpp | 2 +- src/storage/layout.cpp | 31 ++++++++++ test/catalog/catalog_test.cpp | 33 +++++++---- test/executor/drop_test.cpp | 10 +++- test/optimizer/optimizer_test.cpp | 10 +++- test/storage/database_test.cpp | 9 ++- test/tuning/layout_tuner_test.cpp | 2 +- 19 files changed, 146 insertions(+), 99 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index bf80063d580..d4eca4f03db 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -162,7 +162,7 @@ void Catalog::BootstrapSystemCatalogs(storage::Database *database, system_catalogs->GetSchemaCatalog()->InsertSchema( CATALOG_SCHEMA_OID, CATALOG_SCHEMA_NAME, pool_.get(), txn); system_catalogs->GetSchemaCatalog()->InsertSchema( - DEFUALT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn); + DEFAULT_SCHEMA_OID, DEFAULT_SCHEMA_NAME, pool_.get(), txn); // Insert catalog tables into pg_table // pg_database record is shared across different databases @@ -561,15 +561,6 @@ ResultType Catalog::CreateIndex( return ResultType::SUCCESS; } -/* - * @brief create a new layout for a table - * @param database_oid database to which the table belongs to - * @param table_oid table to which the layout has to be added - * @param column_map column_map of the new layout to be created - * @param txn TransactionContext - * @return shared_ptr shared_ptr to the newly created layout in case of - * success. nullptr in case of failure. - */ std::shared_ptr Catalog::CreateLayout( oid_t database_oid, oid_t table_oid, const column_map_type &column_map, concurrency::TransactionContext *txn) { @@ -594,16 +585,6 @@ std::shared_ptr Catalog::CreateLayout( return new_layout; } -/* - * @brief create a new layout for a table and make it the deafult if - * if the creating is successsful. - * @param database_oid database to which the table belongs to - * @param table_oid table to which the layout has to be added - * @param column_map column_map of the new layout to be created - * @param txn TransactionContext - * @return shared_ptr shared_ptr to the newly created layout in case of - * success. nullptr in case of failure. - */ std::shared_ptr Catalog::CreateDefaultLayout( oid_t database_oid, oid_t table_oid, const column_map_type &column_map, concurrency::TransactionContext *txn) { @@ -830,14 +811,6 @@ ResultType Catalog::DropIndex(oid_t database_oid, oid_t index_oid, return ResultType::SUCCESS; } -/*@brief Drop layout - * tile_groups - * @param database_oid the database to which the table belongs - * @param table_oid the table to which the layout belongs - * @param layout_oid the layout to be dropped - * @param txn TransactionContext - * @return TransactionContext ResultType(SUCCESS or FAILURE) - */ ResultType Catalog::DropLayout(oid_t database_oid, oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn) { diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 0a2229123ed..523ea8ae2a7 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -338,7 +338,7 @@ TableCatalog::TableCatalog( bool TableCatalogObject::InsertLayout( std::shared_ptr layout) { // Invalid object - if (layout == nullptr) { + if (!layout || (layout->GetOid() == INVALID_OID)) { return false; } diff --git a/src/codegen/proxy/runtime_functions_proxy.cpp b/src/codegen/proxy/runtime_functions_proxy.cpp index db37ae16c25..700e948d85c 100644 --- a/src/codegen/proxy/runtime_functions_proxy.cpp +++ b/src/codegen/proxy/runtime_functions_proxy.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "codegen/proxy/runtime_functions_proxy.h" + #include "codegen/proxy/data_table_proxy.h" #include "codegen/proxy/tile_group_proxy.h" #include "codegen/proxy/zone_map_proxy.h" @@ -24,8 +25,6 @@ DEFINE_TYPE(ColumnLayoutInfo, "peloton::ColumnLayoutInfo", DEFINE_TYPE(AbstractExpression, "peloton::expression::AbstractExpression", MEMBER(opaque)); -DEFINE_TYPE(Schema, "peloton::catalog::Schema", MEMBER(opaque)); - DEFINE_METHOD(peloton::codegen, RuntimeFunctions, HashCrc64); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, GetTileGroup); DEFINE_METHOD(peloton::codegen, RuntimeFunctions, GetTileGroupLayout); diff --git a/src/codegen/runtime_functions.cpp b/src/codegen/runtime_functions.cpp index d7e291a0298..40562516f75 100644 --- a/src/codegen/runtime_functions.cpp +++ b/src/codegen/runtime_functions.cpp @@ -102,38 +102,38 @@ void RuntimeFunctions::FillPredicateArray( // to skip over to find successive values of the column. //===----------------------------------------------------------------------===// void RuntimeFunctions::GetTileGroupLayout(const storage::TileGroup *tile_group, - const catalog::Schema *schema, ColumnLayoutInfo *infos, - uint32_t num_cols) { + UNUSED_ATTRIBUTE uint32_t num_cols) { const auto &layout = tile_group->GetLayout(); - // For LayoutType::ROW, the tile group contains a single tile - // and all the columns are in the same order as the table schema. - if (layout.IsRowStore()) { - auto tuple_location = tile_group->GetTile(0)->GetTupleLocation(0); - auto stride = schema->GetLength(); - for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { - infos[col_idx].column = tuple_location + schema->GetOffset(col_idx); - infos[col_idx].stride = stride; - infos[col_idx].is_columnar = false; - } - return; - } - for (uint32_t col_idx = 0; col_idx < num_cols; col_idx++) { - // Map the current column to a tile and a column offset in the tile - oid_t tile_offset, tile_column_offset; - layout.LocateTileAndColumn(col_idx, tile_offset, tile_column_offset); + UNUSED_ATTRIBUTE oid_t last_col_idx = INVALID_OID; - // Now grab the column information - auto *tile = tile_group->GetTile(tile_offset); - auto *tile_schema = tile->GetSchema(); - infos[col_idx].column = - tile->GetTupleLocation(0) + tile_schema->GetOffset(tile_column_offset); - infos[col_idx].stride = tile_schema->GetLength(); - infos[col_idx].is_columnar = tile_schema->GetColumnCount() == 1; - LOG_TRACE("Col [%u] start: %p, stride: %u, columnar: %s", col_idx, - infos[col_idx].column, infos[col_idx].stride, - infos[col_idx].is_columnar ? "true" : "false"); + auto tile_map = layout.GetTileMap(); + // Find the mapping for each tile in the layout. + for (auto tile_entry : tile_map) { + // Get the tile schema. + auto tile_idx = tile_entry.first; + auto *tile = tile_group->GetTile(tile_idx); + auto tile_schema = tile->GetSchema(); + // Map the current column to a tile and a column offset in the tile. + for (auto column_entry : tile_entry.second) { + // Now grab the column information + oid_t col_idx = column_entry.first; + oid_t tile_col_offset = column_entry.second; + // Ensure that the col_idx is within the num_cols range + PELOTON_ASSERT(col_idx < num_cols); + infos[col_idx].column = + tile->GetTupleLocation(0) + tile_schema->GetOffset(tile_col_offset); + infos[col_idx].stride = tile_schema->GetLength(); + infos[col_idx].is_columnar = tile_schema->GetColumnCount() == 1; + last_col_idx = col_idx; + LOG_TRACE("Col [%u] start: %p, stride: %u, columnar: %s", col_idx, + infos[col_idx].column, infos[col_idx].stride, + infos[col_idx].is_columnar ? "true" : "false"); + } } + // Ensure that ColumnLayoutInfo for each column has been populated. + PELOTON_ASSERT((last_col_idx != INVALID_OID) && + (last_col_idx == (num_cols - 1))); } void RuntimeFunctions::ThrowDivideByZeroException() { diff --git a/src/codegen/tile_group.cpp b/src/codegen/tile_group.cpp index 6c404b50a41..c26d78c0727 100644 --- a/src/codegen/tile_group.cpp +++ b/src/codegen/tile_group.cpp @@ -88,15 +88,12 @@ llvm::Value *TileGroup::GetTileGroupId(CodeGen &codegen, std::vector TileGroup::GetColumnLayouts( CodeGen &codegen, llvm::Value *tile_group_ptr, llvm::Value *column_layout_infos) const { - llvm::Value *schema_ptr = - codegen->CreateIntToPtr(codegen.Const64((int64_t)&schema_), - SchemaProxy::GetType(codegen)->getPointerTo()); // Call RuntimeFunctions::GetTileGroupLayout() uint32_t num_cols = schema_.GetColumnCount(); - codegen.Call(RuntimeFunctionsProxy::GetTileGroupLayout, - {tile_group_ptr, schema_ptr, column_layout_infos, - codegen.Const32(num_cols)}); + codegen.Call( + RuntimeFunctionsProxy::GetTileGroupLayout, + {tile_group_ptr, column_layout_infos, codegen.Const32(num_cols)}); // Collect triplets of all columns std::vector layouts; diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 6e3fd8e33bf..9a91da7479a 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -120,12 +120,29 @@ class Catalog { concurrency::TransactionContext *txn, bool is_catalog = false); - // Create a new layout + /** + * @brief create a new layout for a table + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ std::shared_ptr CreateLayout( oid_t database_oid, oid_t table_oid, const column_map_type &column_map, concurrency::TransactionContext *txn); - // Create a new layout and set it as the default for the table + /** + * @brief create a new layout for a table and make it the default if + * if the creating is successsful. + * @param database_oid database to which the table belongs to + * @param table_oid table to which the layout has to be added + * @param column_map column_map of the new layout to be created + * @param txn TransactionContext + * @return shared_ptr shared_ptr to the newly created layout in case of + * success. nullptr in case of failure. + */ std::shared_ptr CreateDefaultLayout( oid_t database_oid, oid_t table_oid, const column_map_type &column_map, concurrency::TransactionContext *txn); @@ -158,7 +175,14 @@ class Catalog { ResultType DropIndex(oid_t database_oid, oid_t index_oid, concurrency::TransactionContext *txn); - // Delete a layout using its database_oid, table_oid and layout_oid + /** @brief Drop layout + * tile_groups + * @param database_oid the database to which the table belongs + * @param table_oid the table to which the layout belongs + * @param layout_oid the layout to be dropped + * @param txn TransactionContext + * @return ResultType(SUCCESS or FAILURE) + */ ResultType DropLayout(oid_t database_oid, oid_t table_oid, oid_t layout_oid, concurrency::TransactionContext *txn); //===--------------------------------------------------------------------===// diff --git a/src/include/catalog/catalog_defaults.h b/src/include/catalog/catalog_defaults.h index f6343e85b8e..117a7e8ab6d 100644 --- a/src/include/catalog/catalog_defaults.h +++ b/src/include/catalog/catalog_defaults.h @@ -53,7 +53,7 @@ namespace catalog { // Reserved schema oid // "public" for default schema, and "pg_catalog" schema for catalog tables #define CATALOG_SCHEMA_OID (0 | SCHEMA_OID_MASK) -#define DEFUALT_SCHEMA_OID (1 | SCHEMA_OID_MASK) +#define DEFAULT_SCHEMA_OID (1 | SCHEMA_OID_MASK) #define CATALOG_SCHEMA_NAME "pg_catalog" #define DEFAULT_SCHEMA_NAME "public" diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 1819f36f53c..93baed80f46 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -39,11 +39,6 @@ namespace catalog { class LayoutCatalog : public AbstractCatalog { public: - // Global Singleton, only the first call requires passing parameters. - static LayoutCatalog *GetInstance( - storage::Database *pg_catalog = nullptr, - type::AbstractPool *pool = nullptr, - concurrency::TransactionContext *txn = nullptr); LayoutCatalog(storage::Database *pg_catalog, type::AbstractPool *pool, concurrency::TransactionContext *txn); diff --git a/src/include/codegen/proxy/runtime_functions_proxy.h b/src/include/codegen/proxy/runtime_functions_proxy.h index a61940cf3b3..fe73c0ba48a 100644 --- a/src/include/codegen/proxy/runtime_functions_proxy.h +++ b/src/include/codegen/proxy/runtime_functions_proxy.h @@ -51,7 +51,6 @@ PROXY(RuntimeFunctions) { TYPE_BUILDER(ColumnLayoutInfo, codegen::RuntimeFunctions::ColumnLayoutInfo); TYPE_BUILDER(AbstractExpression, expression::AbstractExpression); -TYPE_BUILDER(Schema, catalog::Schema); } // namespace codegen } // namespace peloton \ No newline at end of file diff --git a/src/include/codegen/runtime_functions.h b/src/include/codegen/runtime_functions.h index c67e5aaae0b..6b8ee29c980 100644 --- a/src/include/codegen/runtime_functions.h +++ b/src/include/codegen/runtime_functions.h @@ -70,7 +70,6 @@ class RuntimeFunctions { // Get the column configuration for every column in the tile group static void GetTileGroupLayout(const storage::TileGroup *tile_group, - const catalog::Schema *schema, ColumnLayoutInfo *infos, uint32_t num_cols); static void ThrowDivideByZeroException(); diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 0f12d14a49b..75ed9c12eda 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -1406,6 +1406,11 @@ typedef std::unordered_set> column_map_type; +/* tile_map_type used to store the mapping between a tile and its columns + * to vector{} + */ +typedef std::map>> tile_map_type; + //===--------------------------------------------------------------------===// // Wire protocol typedefs //===--------------------------------------------------------------------===// diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index 1995685ea8e..a4beb96f217 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -92,7 +92,10 @@ class Layout : public Printable { /** @brief Returns the number of columns in the layout. */ oid_t GetColumnCount() const { return num_columns_; } - /** @brief Constructs the schema for the given layout. Thid function + /** @brief Returns the tile-columns map for each tile in the TileGroup. */ + tile_map_type GetTileMap() const; + + /** @brief Constructs the schema for the given layout. This function * is used only in TempTables and LogicalTiles. */ std::vector GetLayoutSchemas( diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index e81ecc4a6ee..0894d99a67b 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1239,7 +1239,7 @@ void SetTransformedTileGroup(storage::TileGroup *orig_tile_group, // Check that both tile groups have the same schema // Currently done by checking that the number of columns are equal - // TODO Pooja: Handle schena equality for multiple schema versions. + // TODO Pooja: Handle schema equality for multiple schema versions. UNUSED_ATTRIBUTE auto new_column_count = new_layout.GetColumnCount(); UNUSED_ATTRIBUTE auto orig_column_count = orig_layout.GetColumnCount(); PELOTON_ASSERT(new_column_count == orig_column_count); diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index c8d96c4c382..e579a02c908 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -139,6 +139,37 @@ oid_t Layout::GetTileIdFromColumnId(oid_t column_id) const { return tile_offset; } +tile_map_type Layout::GetTileMap() const { + tile_map_type tile_map; + + if (layout_type_ == LayoutType::ROW) { + // Row store layout, hence all columns are contained in tile 0. + // The column_offset is always the same as column_id. + tile_map[0] = {}; + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + tile_map[0].emplace_back(column_id, column_id); + } + } else if (layout_type_ == LayoutType::COLUMN) { + // Column store layout, hence all columns are contained in separate tiles. + // The column_offset within the tile is always 0. + for (oid_t column_id = 0; column_id < num_columns_; column_id++) { + tile_map[column_id] = {std::make_pair(column_id, 0)}; + } + } else { + // Hybrid layout, hence figure out the mapping from the column_layout_. + for (auto entry : column_layout_) { + auto column_id = entry.first; + auto tile_id = entry.second.first; + auto column_offset = entry.second.second; + if (tile_map.find(tile_id) == tile_map.end()) { + tile_map[tile_id] = {}; + } + tile_map[tile_id].emplace_back(column_id, column_offset); + } + } + return tile_map; +} + oid_t Layout::GetTileColumnOffset(oid_t column_id) const { oid_t tile_column_id, tile_offset; LocateTileAndColumn(column_id, tile_offset, tile_column_id); diff --git a/test/catalog/catalog_test.cpp b/test/catalog/catalog_test.cpp index f99f11f1a40..36af866b958 100644 --- a/test/catalog/catalog_test.cpp +++ b/test/catalog/catalog_test.cpp @@ -235,9 +235,11 @@ TEST_F(CatalogTests, DroppingTable) { auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); auto catalog = catalog::Catalog::GetInstance(); - // NOTE: everytime we create a database, there will be 9 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables + // inside. Additionally, we create 3 tables for the test. + oid_t expected_table_count = CATALOG_TABLES_COUNT + 3; EXPECT_EQ( - CATALOG_TABLES_COUNT + 3, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); auto database_object = catalog::Catalog::GetInstance()->GetDatabaseObject("emp_db", txn); @@ -250,8 +252,10 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_NE(nullptr, database_object); auto department_table_object = database_object->GetTableObject("department_table", DEFAULT_SCHEMA_NAME); + // Decrement expected_table_count to account for the dropped table. + expected_table_count--; EXPECT_EQ( - CATALOG_TABLES_COUNT + 2, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -262,9 +266,8 @@ TEST_F(CatalogTests, DroppingTable) { EXPECT_THROW(catalog::Catalog::GetInstance()->DropTable( "emp_db", DEFAULT_SCHEMA_NAME, "department_table", txn), CatalogException); - // EXPECT_EQ( - CATALOG_TABLES_COUNT + 2, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -274,7 +277,7 @@ TEST_F(CatalogTests, DroppingTable) { "emp_db", DEFAULT_SCHEMA_NAME, "void_table", txn), CatalogException); EXPECT_EQ( - CATALOG_TABLES_COUNT + 2, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); @@ -282,8 +285,10 @@ TEST_F(CatalogTests, DroppingTable) { txn = txn_manager.BeginTransaction(); catalog::Catalog::GetInstance()->DropTable("emp_db", DEFAULT_SCHEMA_NAME, "emp_table", txn); + // Account for the dropped table. + expected_table_count--; EXPECT_EQ( - CATALOG_TABLES_COUNT + 1, + expected_table_count, (int)catalog->GetDatabaseObject("emp_db", txn)->GetTableObjects().size()); txn_manager.CommitTransaction(txn); } @@ -317,7 +322,7 @@ TEST_F(CatalogTests, LayoutCatalogTest) { // Create database. auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); - catalog->CreateDatabase(db_name, txn); + EXPECT_EQ(ResultType::SUCCESS, catalog->CreateDatabase(db_name, txn)); // Create table. auto val0 = catalog::Column(type::TypeId::INTEGER, @@ -334,15 +339,19 @@ TEST_F(CatalogTests, LayoutCatalogTest) { "val3", true); std::unique_ptr table_schema( new catalog::Schema({val0, val1, val2, val3})); - catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn); + EXPECT_EQ(ResultType::SUCCESS, + catalog->CreateTable(db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn)); txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); + auto database_oid = + catalog->GetDatabaseObject(db_name, txn)->GetDatabaseOid(); + auto table_object = + catalog->GetTableObject(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); + auto table_oid = table_object->GetTableOid(); auto table = catalog->GetTableWithName(db_name, DEFAULT_SCHEMA_NAME, table_name, txn); - auto table_oid = table->GetOid(); - auto database_oid = table->GetDatabaseOid(); txn_manager.CommitTransaction(txn); // Change default layout. diff --git a/test/executor/drop_test.cpp b/test/executor/drop_test.cpp index 05f0a61d628..d90f06cbcb1 100644 --- a/test/executor/drop_test.cpp +++ b/test/executor/drop_test.cpp @@ -106,19 +106,23 @@ TEST_F(DropTests, DroppingTable) { txn_manager.CommitTransaction(txn); txn = txn_manager.BeginTransaction(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables + // inside. In this test case, we have created two additional tables. + oid_t expeected_table_count = CATALOG_TABLES_COUNT + 2; EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size(), - (CATALOG_TABLES_COUNT + 2)); + expeected_table_count); // Now dropping the table using the executor catalog->DropTable(TEST_DB_NAME, DEFAULT_SCHEMA_NAME, "department_table", txn); + // Account for the dropped table. + expeected_table_count--; EXPECT_EQ((int)catalog->GetDatabaseObject(TEST_DB_NAME, txn) ->GetTableObjects() .size(), - (CATALOG_TABLES_COUNT + 1)); + expeected_table_count); // free the database just created catalog->DropDatabaseWithName(TEST_DB_NAME, txn); diff --git a/test/optimizer/optimizer_test.cpp b/test/optimizer/optimizer_test.cpp index dae410999d6..8b5ed1e0ec7 100644 --- a/test/optimizer/optimizer_test.cpp +++ b/test/optimizer/optimizer_test.cpp @@ -119,12 +119,14 @@ TEST_F(OptimizerTests, HashJoinTest) { LOG_INFO("Table Created"); traffic_cop.CommitQueryHelper(); - // NOTE: everytime we create a database, there will be 8 catalog tables inside + // NOTE: everytime we create a database, there will be 9 catalog tables inside + // Additionally, we also created a table for the test. + oid_t expected_table_count = CATALOG_TABLES_COUNT + 1; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) ->GetTableCount(), - 1 + CATALOG_TABLES_COUNT); + expected_table_count); traffic_cop.SetTcopTxnState(txn); LOG_INFO("Creating table"); @@ -155,11 +157,13 @@ TEST_F(OptimizerTests, HashJoinTest) { LOG_INFO("Table Created"); traffic_cop.CommitQueryHelper(); + // Account for table created. + expected_table_count++; txn = txn_manager.BeginTransaction(); EXPECT_EQ(catalog::Catalog::GetInstance() ->GetDatabaseWithName(DEFAULT_DB_NAME, txn) ->GetTableCount(), - 2 + CATALOG_TABLES_COUNT); + expected_table_count); // Inserting a tuple to table_a traffic_cop.SetTcopTxnState(txn); diff --git a/test/storage/database_test.cpp b/test/storage/database_test.cpp index b60919a79d3..427d4c220e3 100644 --- a/test/storage/database_test.cpp +++ b/test/storage/database_test.cpp @@ -63,11 +63,16 @@ TEST_F(DatabaseTests, AddDropTableTest) { database->AddTable(data_table.get()); // NOTE: everytime we create a database, there will be 8 catalog tables inside - EXPECT_TRUE(database->GetTableCount() == 1 + CATALOG_TABLES_COUNT); + // Addtionally, we created a table for the test + oid_t expected_table_count = CATALOG_TABLES_COUNT + 1; + + EXPECT_EQ(expected_table_count, database->GetTableCount()); database->DropTableWithOid(table_oid); - EXPECT_TRUE(database->GetTableCount() == CATALOG_TABLES_COUNT); + // Account for the dropped table. + expected_table_count--; + EXPECT_EQ(expected_table_count, database->GetTableCount()); data_table.release(); diff --git a/test/tuning/layout_tuner_test.cpp b/test/tuning/layout_tuner_test.cpp index adabf72a222..521ff92cc78 100644 --- a/test/tuning/layout_tuner_test.cpp +++ b/test/tuning/layout_tuner_test.cpp @@ -6,7 +6,7 @@ // // Identification: test/tuning/layout_tuner_test.cpp // -// Copyright (c) 2015-16, Carnegie Mellon University Databa e Group +// Copyright (c) 2015-16, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// From 2ea5d7efbc257b40bf15889d85410725a02ce0e8 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 6 May 2018 20:57:48 -0400 Subject: [PATCH 37/42] Change valid_layout_objects_ --- src/catalog/table_catalog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/catalog/table_catalog.cpp b/src/catalog/table_catalog.cpp index 523ea8ae2a7..9f181d4c0dc 100644 --- a/src/catalog/table_catalog.cpp +++ b/src/catalog/table_catalog.cpp @@ -349,7 +349,6 @@ bool TableCatalogObject::InsertLayout( return false; } - valid_layout_objects_ = true; layout_objects_.insert(std::make_pair(layout_id, layout)); return true; } From f25608284a9f918915514e6220abb512a4d4fa9f Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Sun, 6 May 2018 21:02:30 -0400 Subject: [PATCH 38/42] Revert unused changes --- src/include/codegen/proxy/runtime_functions_proxy.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/include/codegen/proxy/runtime_functions_proxy.h b/src/include/codegen/proxy/runtime_functions_proxy.h index fe73c0ba48a..e012501efc0 100644 --- a/src/include/codegen/proxy/runtime_functions_proxy.h +++ b/src/include/codegen/proxy/runtime_functions_proxy.h @@ -12,9 +12,8 @@ #pragma once -#include #include -#include "catalog/schema.h" +#include #include "codegen/proxy/proxy.h" #include "codegen/proxy/type_builder.h" #include "codegen/runtime_functions.h" @@ -35,11 +34,6 @@ PROXY(AbstractExpression) { DECLARE_TYPE; }; -PROXY(Schema) { - DECLARE_MEMBER(0, char[sizeof(catalog::Schema)], opaque); - DECLARE_TYPE; -}; - PROXY(RuntimeFunctions) { DECLARE_METHOD(HashCrc64); DECLARE_METHOD(GetTileGroup); From 0ff874dc535f78ce35053e2ed07430795e326844 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Mon, 7 May 2018 10:59:08 -0400 Subject: [PATCH 39/42] Addressed Prashanth's initial comments --- src/catalog/layout_catalog.cpp | 6 +++--- src/codegen/tile_group.cpp | 1 - src/codegen/updater.cpp | 2 +- src/include/catalog/layout_catalog.h | 15 --------------- src/include/catalog/table_catalog.h | 2 +- src/include/codegen/runtime_functions.h | 4 ---- src/include/codegen/tile_group.h | 2 +- src/include/common/internal_types.h | 5 ----- src/include/storage/data_table.h | 8 ++++---- src/include/storage/layout.h | 15 +++++++++------ src/storage/data_table.cpp | 1 + src/storage/layout.cpp | 2 +- 12 files changed, 21 insertions(+), 42 deletions(-) diff --git a/src/catalog/layout_catalog.cpp b/src/catalog/layout_catalog.cpp index d4173706020..bd72854455d 100644 --- a/src/catalog/layout_catalog.cpp +++ b/src/catalog/layout_catalog.cpp @@ -2,11 +2,11 @@ // // Peloton // -// layout_catalog.h +// layout_catalog.cpp // -// Identification: src/include/catalog/layout_catalog.cpp +// Identification: src/catalog/layout_catalog.cpp // -// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// diff --git a/src/codegen/tile_group.cpp b/src/codegen/tile_group.cpp index c26d78c0727..4831cee3437 100644 --- a/src/codegen/tile_group.cpp +++ b/src/codegen/tile_group.cpp @@ -88,7 +88,6 @@ llvm::Value *TileGroup::GetTileGroupId(CodeGen &codegen, std::vector TileGroup::GetColumnLayouts( CodeGen &codegen, llvm::Value *tile_group_ptr, llvm::Value *column_layout_infos) const { - // Call RuntimeFunctions::GetTileGroupLayout() uint32_t num_cols = schema_.GetColumnCount(); codegen.Call( diff --git a/src/codegen/updater.cpp b/src/codegen/updater.cpp index cb32ea51fb3..1782f219c83 100644 --- a/src/codegen/updater.cpp +++ b/src/codegen/updater.cpp @@ -46,7 +46,7 @@ char *Updater::GetDataPtr(uint32_t tile_group_id, uint32_t tuple_offset) { // Get the tile offset assuming that it is still a row store // Hence the Tile offset is 0. - auto layout = tile_group->GetLayout(); + UNUSED_ATTRIBUTE const auto &layout = tile_group->GetLayout(); PELOTON_ASSERT(layout.IsRowStore()); tile_ = tile_group->GetTileReference(0); return tile_->GetTupleLocation(tuple_offset); diff --git a/src/include/catalog/layout_catalog.h b/src/include/catalog/layout_catalog.h index 93baed80f46..0fc224f094d 100644 --- a/src/include/catalog/layout_catalog.h +++ b/src/include/catalog/layout_catalog.h @@ -10,21 +10,6 @@ // //===----------------------------------------------------------------------===// -//===----------------------------------------------------------------------===// -// pg_layout -// -// Schema: (column offset: column_name) -// 0: table_oid (pkey) -// 1: layout_oid (pkey) -// 2: num_columns (number of columns in the layout) -// 3: column_map (map column_oid to ) -// -// Indexes: (index offset: indexed columns) -// 0: table_oid & layout_oid (unique & primary key) -// 1: table_oid (non-unique) -// -//===----------------------------------------------------------------------===// - #pragma once #include "catalog/abstract_catalog.h" diff --git a/src/include/catalog/table_catalog.h b/src/include/catalog/table_catalog.h index bdf900357c8..9a01ee6e07f 100644 --- a/src/include/catalog/table_catalog.h +++ b/src/include/catalog/table_catalog.h @@ -39,7 +39,7 @@ namespace peloton { namespace storage { class Layout; -} +} // namespace storage namespace catalog { diff --git a/src/include/codegen/runtime_functions.h b/src/include/codegen/runtime_functions.h index 6b8ee29c980..4a7eafac1eb 100644 --- a/src/include/codegen/runtime_functions.h +++ b/src/include/codegen/runtime_functions.h @@ -17,10 +17,6 @@ namespace peloton { -namespace catalog { -class Schema; -} - namespace storage { class DataTable; class TileGroup; diff --git a/src/include/codegen/tile_group.h b/src/include/codegen/tile_group.h index 38d7192f62c..dcb8888fcde 100644 --- a/src/include/codegen/tile_group.h +++ b/src/include/codegen/tile_group.h @@ -137,7 +137,7 @@ class TileGroup { inline const TileGroup &GetTileGroup() const { return tile_group_; } - inline const ColumnLayout &GetLayout(uint32_t col_idx) const { + const ColumnLayout &GetLayout(uint32_t col_idx) const { return layout_[col_idx]; } diff --git a/src/include/common/internal_types.h b/src/include/common/internal_types.h index 75ed9c12eda..0f12d14a49b 100644 --- a/src/include/common/internal_types.h +++ b/src/include/common/internal_types.h @@ -1406,11 +1406,6 @@ typedef std::unordered_set> column_map_type; -/* tile_map_type used to store the mapping between a tile and its columns - * to vector{} - */ -typedef std::map>> tile_map_type; - //===--------------------------------------------------------------------===// // Wire protocol typedefs //===--------------------------------------------------------------------===// diff --git a/src/include/storage/data_table.h b/src/include/storage/data_table.h index 48f36dc075d..cfabec8f78e 100644 --- a/src/include/storage/data_table.h +++ b/src/include/storage/data_table.h @@ -249,12 +249,12 @@ class DataTable : public AbstractTable { void ClearLayoutSamples(); - inline void SetDefaultLayout(std::shared_ptr new_layout) { + void SetDefaultLayout(std::shared_ptr new_layout) { PELOTON_ASSERT(new_layout->GetColumnCount() == schema->GetColumnCount()); default_layout_ = new_layout; } - inline void ResetDefaultLayout(LayoutType type = LayoutType::ROW) { + void ResetDefaultLayout(LayoutType type = LayoutType::ROW) { PELOTON_ASSERT((type == LayoutType::ROW) || (type == LayoutType::COLUMN)); default_layout_ = std::shared_ptr( new const Layout(schema->GetColumnCount(), type)); @@ -438,8 +438,8 @@ class DataTable : public AbstractTable { bool dirty_ = false; // Last used layout_oid. Used while creating new layouts - // Initialized to COLUMN_STORE_OID since its the highest predfeined value. - std::atomic current_layout_oid_ = ATOMIC_VAR_INIT(COLUMN_STORE_OID); + // Initialized to COLUMN_STORE_OID since its the highest predefined value. + std::atomic current_layout_oid_; //===--------------------------------------------------------------------===// // TUNING MEMBERS diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index a4beb96f217..d2ec1f8072c 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -6,7 +6,7 @@ // // Identification: src/include/storage/layout.h // -// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// @@ -27,10 +27,15 @@ namespace peloton { namespace catalog { class Schema; -} +} // namespace catalog namespace storage { +/** @brief used to store the mapping between a tile and its columns + * to vector{} + */ +typedef std::map>> tile_map_type; + /** * @brief Class to store the physical layout of a TileGroup. */ @@ -62,12 +67,10 @@ class Layout : public Printable { Layout(const column_map_type &column_map, oid_t layout_oid); /** @brief Check whether this layout is a row store. */ - inline bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } + bool IsRowStore() const { return (layout_type_ == LayoutType::ROW); } /** @brief Check whether this layout is a column store. */ - inline bool IsColumnStore() const { - return (layout_type_ == LayoutType::COLUMN); - } + bool IsColumnStore() const { return (layout_type_ == LayoutType::COLUMN); } /** @brief Return the layout_oid_ of this object. */ oid_t GetOid() const { return layout_oid_; } diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 0894d99a67b..34ff155a0e6 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -67,6 +67,7 @@ DataTable::DataTable(catalog::Schema *schema, const std::string &table_name, database_oid(database_oid), table_name(table_name), tuples_per_tilegroup_(tuples_per_tilegroup), + current_layout_oid_(ATOMIC_VAR_INIT(COLUMN_STORE_OID)), adapt_table_(adapt_table), trigger_list_(new trigger::TriggerList()) { if (is_catalog == true) { diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index e579a02c908..dab6183b2b9 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -6,7 +6,7 @@ // // Identification: src/storage/layout.cpp // -// Copyright (c) 2015-18, Carnegie Mellon University Database Group +// Copyright (c) 2015-2018, Carnegie Mellon University Database Group // //===----------------------------------------------------------------------===// From ecb95bb9acd3a2ad546e130afa1f839fa3029a4c Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Tue, 8 May 2018 20:35:57 -0400 Subject: [PATCH 40/42] Modify CreateTable for tests + modify LayoutTuner --- src/include/storage/layout.h | 2 +- src/include/tuning/layout_tuner.h | 3 +- src/tuning/layout_tuner.cpp | 12 ++- test/codegen/table_scan_translator_test.cpp | 100 ++++---------------- test/codegen/testing_codegen_util.cpp | 87 ++++++++++++++++- test/include/codegen/testing_codegen_util.h | 19 +++- 6 files changed, 128 insertions(+), 95 deletions(-) diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index d2ec1f8072c..e05b1899e3f 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -141,4 +141,4 @@ class Layout : public Printable { }; } // namespace storage -} // namespace peloton \ No newline at end of file +} // namespace peloton diff --git a/src/include/tuning/layout_tuner.h b/src/include/tuning/layout_tuner.h index 5bcba99b6d7..14a6fe74e8d 100644 --- a/src/include/tuning/layout_tuner.h +++ b/src/include/tuning/layout_tuner.h @@ -87,8 +87,9 @@ class LayoutTuner { * Update layout of table * * @param table The table + * @return true if the update succeeds, false otherwise */ - void UpdateDefaultPartition(storage::DataTable *table); + bool UpdateDefaultPartition(storage::DataTable *table); private: /** diff --git a/src/tuning/layout_tuner.cpp b/src/tuning/layout_tuner.cpp index df64ea38bb4..a48fc64e564 100644 --- a/src/tuning/layout_tuner.cpp +++ b/src/tuning/layout_tuner.cpp @@ -76,7 +76,7 @@ Sample GetClustererSample(const Sample& sample, oid_t column_count) { return clusterer_sample; } -void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { +bool LayoutTuner::UpdateDefaultPartition(storage::DataTable *table) { oid_t column_count = table->GetSchema()->GetColumnCount(); // Set up clusterer @@ -87,7 +87,9 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { // Check if we have any samples if (samples.empty()) { - return; + LOG_DEBUG("Table[%u] contains no LayoutSamples. Layout not tuned.", + table->GetOid()); + return false; } for (auto sample : samples) { @@ -120,12 +122,13 @@ void LayoutTuner::UpdateDefaultPartition(storage::DataTable* table) { nullptr) { txn_manager.AbortTransaction(txn); LOG_DEBUG("Layout Update to failed."); - return; + return false; } txn_manager.CommitTransaction(txn); UNUSED_ATTRIBUTE auto layout = table->GetDefaultLayout(); LOG_TRACE("Updated Layout: %s", layout.GetInfo().c_str()); + return true; } void LayoutTuner::Tune() { @@ -142,7 +145,8 @@ void LayoutTuner::Tune() { table->TransformTileGroup(tile_group_offset, theta); // Update partitioning periodically - UpdateDefaultPartition(table); + // TODO Lin/Tianyu - Add Failure Handling/Retry logic. + UNUSED_ATTRIBUTE bool update_result = UpdateDefaultPartition(table); // Sleep a bit std::this_thread::sleep_for(std::chrono::microseconds(sleep_duration)); diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index f0373745192..3760745b48b 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -36,82 +36,10 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { CreateAndLoadAllColsTable(); } - void ExecuteTileGroupTest(peloton::LayoutType layout_type) { - const int tuples_per_tilegroup = 100; - const int tile_group_count = 5; - const int tuple_count = tuples_per_tilegroup * tile_group_count; - const oid_t col_count = 100; - const bool is_inlined = true; - - ///////////////////////////////////////////////////////// - // Define the schema. - ///////////////////////////////////////////////////////// - - std::vector columns; - - for (oid_t col_itr = 0; col_itr <= col_count; col_itr++) { - auto column = catalog::Column( - type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), - "FIELD" + std::to_string(col_itr), is_inlined); - - columns.push_back(column); - } - - std::unique_ptr table_schema = - std::unique_ptr(new catalog::Schema(columns)); - std::string table_name("TEST_TABLE"); - - ///////////////////////////////////////////////////////// - // Create table. - ///////////////////////////////////////////////////////// - - bool is_catalog = false; - auto *catalog = catalog::Catalog::GetInstance(); - auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); - const bool allocate = true; - auto txn = txn_manager.BeginTransaction(); - - // Insert table in catalog - catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, - std::move(table_schema), txn, is_catalog, - tuples_per_tilegroup, layout_type); - // Get table reference - auto table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, - table_name, txn); - txn_manager.EndTransaction(txn); - - ///////////////////////////////////////////////////////// - // Load in the data - ///////////////////////////////////////////////////////// - - // Insert tuples into tile_group. - - txn = txn_manager.BeginTransaction(); - auto table_schema_ptr = table->GetSchema(); - auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); - - for (oid_t row_id = 0; row_id < tuple_count; row_id++) { - int populate_value = row_id; - - storage::Tuple tuple(table_schema_ptr, allocate); - - for (oid_t col_id = 0; col_id <= col_count; col_id++) { - auto value = - type::ValueFactory::GetIntegerValue(populate_value + col_id); - tuple.SetValue(col_id, value, testing_pool); - } - - ItemPointer *index_entry_ptr = nullptr; - ItemPointer tuple_slot_id = - table->InsertTuple(&tuple, txn, &index_entry_ptr); - - EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); - EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); - - txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); - } - - txn_manager.CommitTransaction(txn); + void ScanLayoutTable(oid_t tuples_per_tilegroup, oid_t tilegroup_count, + oid_t column_count) { + auto table = GetLayoutTable(); + oid_t tuple_count = tuples_per_tilegroup * tilegroup_count; ///////////////////////////////////////////////////////// // Do a seq scan on the table with the given layout @@ -119,7 +47,7 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { // Column ids to be scanned. std::vector column_ids; - for (oid_t col_id = 0; col_id < col_count; col_id++) { + for (oid_t col_id = 0; col_id < column_count; col_id++) { column_ids.push_back(col_id); } @@ -143,7 +71,7 @@ class TableScanTranslatorTest : public PelotonCodeGenTest { for (oid_t tuple_id = 0; tuple_id < tuple_count; tuple_id++) { auto &tuple = results[tuple_id]; int tuple_id_value = tuple_id; - for (oid_t col_id = 0; col_id < col_count; col_id++) { + for (oid_t col_id = 0; col_id < column_count; col_id++) { auto value = type::ValueFactory::GetIntegerValue(tuple_id_value + col_id); EXPECT_EQ(CmpBool::CmpTrue, @@ -706,7 +634,13 @@ TEST_F(TableScanTranslatorTest, ScanRowLayout) { // Creates a table with LayoutType::ROW and // invokes the TableScanTranslator // - ExecuteTileGroupTest(LayoutType::ROW); + oid_t tuples_per_tilegroup = 100; + oid_t tilegroup_count = 5; + oid_t column_count = 100; + bool is_inlined = true; + CreateAndLoadTableWithLayout(LayoutType::ROW, tuples_per_tilegroup, + tilegroup_count, column_count, is_inlined); + ScanLayoutTable(tuples_per_tilegroup, tilegroup_count, column_count); } TEST_F(TableScanTranslatorTest, ScanColumnLayout) { @@ -714,7 +648,13 @@ TEST_F(TableScanTranslatorTest, ScanColumnLayout) { // Creates a table with LayoutType::COLUMN and // invokes the TableScanTranslator // - ExecuteTileGroupTest(LayoutType::COLUMN); + oid_t tuples_per_tilegroup = 100; + oid_t tilegroup_count = 5; + oid_t column_count = 100; + bool is_inlined = true; + CreateAndLoadTableWithLayout(LayoutType::COLUMN, tuples_per_tilegroup, + tilegroup_count, column_count, is_inlined); + ScanLayoutTable(tuples_per_tilegroup, tilegroup_count, column_count); } TEST_F(TableScanTranslatorTest, MultiLayoutScan) { diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index dc46599010b..57231c245c7 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -32,7 +32,8 @@ namespace test { // PELOTON CODEGEN TEST //===----------------------------------------------------------------------===// -PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup) { +PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { auto *catalog = catalog::Catalog::GetInstance(); auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); auto txn = txn_manager.BeginTransaction(); @@ -41,9 +42,10 @@ PelotonCodeGenTest::PelotonCodeGenTest(oid_t tuples_per_tilegroup) { catalog->CreateDatabase(test_db_name, txn); test_db = catalog->GetDatabaseWithName(test_db_name, txn); // Create test table - CreateTestTables(txn, tuples_per_tilegroup); + CreateTestTables(txn, tuples_per_tilegroup, layout_type); txn_manager.CommitTransaction(txn); + layout_table = nullptr; } PelotonCodeGenTest::~PelotonCodeGenTest() { @@ -103,13 +105,14 @@ std::unique_ptr PelotonCodeGenTest::CreateTestSchema( // Create all the test tables, but don't load any data void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, - oid_t tuples_per_tilegroup) { + oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type) { auto *catalog = catalog::Catalog::GetInstance(); for (int i = 0; i < 4; i++) { auto table_schema = CreateTestSchema(); catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, - tuples_per_tilegroup); + tuples_per_tilegroup, layout_type); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, DEFAULT_SCHEMA_NAME, @@ -120,7 +123,7 @@ void PelotonCodeGenTest::CreateTestTables(concurrency::TransactionContext *txn, auto table_schema = CreateTestSchema(true); catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, test_table_names[i], std::move(table_schema), txn, false, - tuples_per_tilegroup); + tuples_per_tilegroup, layout_type); test_table_oids.push_back(catalog ->GetTableObject(test_db_name, DEFAULT_SCHEMA_NAME, @@ -175,6 +178,80 @@ void PelotonCodeGenTest::LoadTestTable(oid_t table_id, uint32_t num_rows, txn_manager.CommitTransaction(txn); } +void PelotonCodeGenTest::CreateAndLoadTableWithLayout( + peloton::LayoutType layout_type, oid_t tuples_per_tilegroup, + oid_t tile_group_count, oid_t column_count, bool is_inlined) { + oid_t tuple_count = tuples_per_tilegroup * tile_group_count; + ///////////////////////////////////////////////////////// + // Define the schema. + ///////////////////////////////////////////////////////// + + std::vector columns; + + for (oid_t col_itr = 0; col_itr <= column_count; col_itr++) { + auto column = catalog::Column( + type::TypeId::INTEGER, type::Type::GetTypeSize(type::TypeId::INTEGER), + "FIELD" + std::to_string(col_itr), is_inlined); + + columns.push_back(column); + } + + std::unique_ptr table_schema = + std::unique_ptr(new catalog::Schema(columns)); + std::string table_name("LAYOUT_TABLE"); + + ///////////////////////////////////////////////////////// + // Create table. + ///////////////////////////////////////////////////////// + + bool is_catalog = false; + auto *catalog = catalog::Catalog::GetInstance(); + auto &txn_manager = concurrency::TransactionManagerFactory::GetInstance(); + const bool allocate = true; + auto txn = txn_manager.BeginTransaction(); + + // Insert table in catalog + catalog->CreateTable(test_db_name, DEFAULT_SCHEMA_NAME, table_name, + std::move(table_schema), txn, is_catalog, + tuples_per_tilegroup, layout_type); + // Get table reference + layout_table = catalog->GetTableWithName(test_db_name, DEFAULT_SCHEMA_NAME, + table_name, txn); + txn_manager.EndTransaction(txn); + + ///////////////////////////////////////////////////////// + // Load in the data + ///////////////////////////////////////////////////////// + + // Insert tuples into tile_group. + + txn = txn_manager.BeginTransaction(); + auto table_schema_ptr = layout_table->GetSchema(); + auto testing_pool = TestingHarness::GetInstance().GetTestingPool(); + + for (oid_t row_id = 0; row_id < tuple_count; row_id++) { + int populate_value = row_id; + + storage::Tuple tuple(table_schema_ptr, allocate); + + for (oid_t col_id = 0; col_id <= column_count; col_id++) { + auto value = type::ValueFactory::GetIntegerValue(populate_value + col_id); + tuple.SetValue(col_id, value, testing_pool); + } + + ItemPointer *index_entry_ptr = nullptr; + ItemPointer tuple_slot_id = + layout_table->InsertTuple(&tuple, txn, &index_entry_ptr); + + EXPECT_TRUE(tuple_slot_id.block != INVALID_OID); + EXPECT_TRUE(tuple_slot_id.offset != INVALID_OID); + + txn_manager.PerformInsert(txn, tuple_slot_id, index_entry_ptr); + } + + txn_manager.CommitTransaction(txn); +} + void PelotonCodeGenTest::ExecuteSync( codegen::Query &query, std::unique_ptr executor_context, diff --git a/test/include/codegen/testing_codegen_util.h b/test/include/codegen/testing_codegen_util.h index 7e5cd3ee20f..8234dfb0d2b 100644 --- a/test/include/codegen/testing_codegen_util.h +++ b/test/include/codegen/testing_codegen_util.h @@ -52,7 +52,8 @@ class PelotonCodeGenTest : public PelotonTest { "table4", "table5"}; std::vector test_table_oids; - PelotonCodeGenTest(oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + PelotonCodeGenTest(oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + peloton::LayoutType layout_type = LayoutType::ROW); virtual ~PelotonCodeGenTest(); @@ -64,6 +65,9 @@ class PelotonCodeGenTest : public PelotonTest { return *GetDatabase().GetTableWithOid(static_cast(table_id)); } + // Get the layout table + storage::DataTable *GetLayoutTable() const { return layout_table; } + // Create the schema (common among all tables) catalog::Column GetTestColumn(uint32_t col_id) const; @@ -71,14 +75,20 @@ class PelotonCodeGenTest : public PelotonTest { bool add_primary = false) const; // Create the test tables - void CreateTestTables( - concurrency::TransactionContext *txn, - oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP); + void CreateTestTables(concurrency::TransactionContext *txn, + oid_t tuples_per_tilegroup, + peloton::LayoutType layout_type); // Load the given table with the given number of rows void LoadTestTable(oid_t table_id, uint32_t num_rows, bool insert_nulls = false); + // Load tables with the specified layout + void CreateAndLoadTableWithLayout(peloton::LayoutType layout_type, + oid_t tuples_per_tilegroup, + oid_t tile_group_count, oid_t column_count, + bool is_inlined); + static void ExecuteSync( codegen::Query &query, std::unique_ptr executor_context, @@ -117,6 +127,7 @@ class PelotonCodeGenTest : public PelotonTest { private: storage::Database *test_db; + storage::DataTable *layout_table; }; //===----------------------------------------------------------------------===// From 66308054a6fe906b869542e862151a17282b39c7 Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Wed, 9 May 2018 16:36:04 -0400 Subject: [PATCH 41/42] Address Pervaze's review comments. --- src/catalog/catalog.cpp | 2 +- src/gc/gc_manager.cpp | 4 +-- src/include/catalog/catalog.h | 2 +- src/include/storage/layout.h | 14 ++++++---- src/include/storage/tile_group.h | 8 +++--- src/optimizer/stats/tuple_sampler.cpp | 2 +- src/storage/data_table.cpp | 2 +- src/storage/layout.cpp | 30 ++++++++++----------- src/storage/tile_group.cpp | 10 +++---- test/codegen/table_scan_translator_test.cpp | 14 +++++----- test/codegen/testing_codegen_util.cpp | 6 ++--- 11 files changed, 48 insertions(+), 46 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index d4eca4f03db..fe91b340ed3 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -299,7 +299,7 @@ ResultType Catalog::CreateTable(const std::string &database_name, const std::string &table_name, std::unique_ptr schema, concurrency::TransactionContext *txn, - bool is_catalog, oid_t tuples_per_tilegroup, + bool is_catalog, uint32_t tuples_per_tilegroup, peloton::LayoutType layout_type) { if (txn == nullptr) throw CatalogException("Do not have transaction to create table " + diff --git a/src/gc/gc_manager.cpp b/src/gc/gc_manager.cpp index 76fb59966ce..9357ff1496e 100644 --- a/src/gc/gc_manager.cpp +++ b/src/gc/gc_manager.cpp @@ -25,8 +25,8 @@ namespace gc { // Check a tuple and reclaim all varlen field void GCManager::CheckAndReclaimVarlenColumns(storage::TileGroup *tile_group, oid_t tuple_id) { - oid_t tile_count = tile_group->tile_count_; - oid_t tile_col_count; + uint32_t tile_count = tile_group->tile_count_; + uint32_t tile_col_count; type::TypeId type_id; char *tuple_location; char *field_location; diff --git a/src/include/catalog/catalog.h b/src/include/catalog/catalog.h index 9a91da7479a..febcb82be4e 100644 --- a/src/include/catalog/catalog.h +++ b/src/include/catalog/catalog.h @@ -100,7 +100,7 @@ class Catalog { const std::string &database_name, const std::string &schema_name, const std::string &table_name, std::unique_ptr, concurrency::TransactionContext *txn, bool is_catalog = false, - oid_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, + uint32_t tuples_per_tilegroup = DEFAULT_TUPLES_PER_TILEGROUP, peloton::LayoutType layout_type = LayoutType::ROW); // Create index for a table diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index e05b1899e3f..d06fe93427e 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -75,11 +75,15 @@ class Layout : public Printable { /** @brief Return the layout_oid_ of this object. */ oid_t GetOid() const { return layout_oid_; } - /** @brief Sets the tile id and column id w.r.t. of the tile corresponding - * to the specified tile group column id. + /** @brief Locates the tile and column to which the specified + * specified tile group column_id. + * It updates the tile_id and tile_column_id references. + * @param column_id The Id of the column to be located. + * @param tile_id A reference to update the tile_id. + * @param tile_column_id A reference to update the tile_column_id. */ - void LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, - oid_t &tile_column_offset) const; + void LocateTileAndColumn(oid_t column_id, oid_t &tile_id, + oid_t &tile_column_id) const; /** @brief Returns the layout difference w.r.t. the other Layout. * @double The delta between the layouts. Used by LayoutTuner class. @@ -93,7 +97,7 @@ class Layout : public Printable { oid_t GetTileColumnOffset(oid_t column_id) const; /** @brief Returns the number of columns in the layout. */ - oid_t GetColumnCount() const { return num_columns_; } + uint32_t GetColumnCount() const { return num_columns_; } /** @brief Returns the tile-columns map for each tile in the TileGroup. */ tile_map_type GetTileMap() const; diff --git a/src/include/storage/tile_group.h b/src/include/storage/tile_group.h index dbf35e9f281..ea9218f06a8 100644 --- a/src/include/storage/tile_group.h +++ b/src/include/storage/tile_group.h @@ -118,9 +118,9 @@ class TileGroup : public Printable { // this function is called only when building tile groups for aggregation // operations. // FIXME: GC has recycled some of the tuples, so this count is not accurate - oid_t GetActiveTupleCount() const; + uint32_t GetActiveTupleCount() const; - oid_t GetAllocatedTupleCount() const { return num_tuple_slots; } + uint32_t GetAllocatedTupleCount() const { return num_tuple_slots_; } TileGroupHeader *GetHeader() const { return tile_group_header; } @@ -187,10 +187,10 @@ class TileGroup : public Printable { AbstractTable *table; // this design is fantastic!!! // number of tuple slots allocated - oid_t num_tuple_slots; + uint32_t num_tuple_slots_; // number of tiles - oid_t tile_count_; + uint32_t tile_count_; std::mutex tile_group_mutex; diff --git a/src/optimizer/stats/tuple_sampler.cpp b/src/optimizer/stats/tuple_sampler.cpp index 9bb91540c9c..26831079add 100644 --- a/src/optimizer/stats/tuple_sampler.cpp +++ b/src/optimizer/stats/tuple_sampler.cpp @@ -98,7 +98,7 @@ bool TupleSampler::GetTupleInTileGroup(storage::TileGroup *tile_group, storage::Tile *tile = tile_group->GetTile(tile_itr); const catalog::Schema &schema = *(tile->GetSchema()); - oid_t tile_column_count = schema.GetColumnCount(); + uint32_t tile_column_count = schema.GetColumnCount(); char *tile_tuple_location = tile->GetTupleLocation(tuple_offset); storage::Tuple tile_tuple(&schema, tile_tuple_location); diff --git a/src/storage/data_table.cpp b/src/storage/data_table.cpp index 34ff155a0e6..2cc158eb837 100644 --- a/src/storage/data_table.cpp +++ b/src/storage/data_table.cpp @@ -1203,7 +1203,7 @@ std::vector TransformTileGroupSchema( // First, get info from the original tile group's schema std::map> schemas; - oid_t column_count = layout.GetColumnCount(); + uint32_t column_count = layout.GetColumnCount(); for (oid_t col_id = 0; col_id < column_count; col_id++) { // Get TileGroup layout's tile and offset for col_id. tile_group_layout.LocateTileAndColumn(col_id, orig_tile_offset, diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index dab6183b2b9..37664a42f77 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -81,34 +81,32 @@ Layout::Layout(const column_map_type &column_map, oid_t layout_id) } } -// Sets the tile id and column id w.r.t that tile corresponding to -// the specified tile group column id. -void Layout::LocateTileAndColumn(oid_t column_offset, oid_t &tile_offset, - oid_t &tile_column_offset) const { +void Layout::LocateTileAndColumn(oid_t column_id, oid_t &tile_id, + oid_t &tile_column_id) const { // Ensure that the column_offset is not out of bound - PELOTON_ASSERT(num_columns_ > column_offset); + PELOTON_ASSERT(num_columns_ > column_id); - // For row store layout, tile id is always 0 and the tile - // column_id and tile column_id is the same. + // For row store layout, tile_id is always 0 and the + // column_id and tile column_id are the same. if (layout_type_ == LayoutType::ROW) { - tile_offset = 0; - tile_column_offset = column_offset; + tile_id = 0; + tile_column_id = column_id; return; } - // For column store layout, tile_id is always same as column_id - // and the tile column_id is always 0. + // For column store layout, tile_id is same as column_id + // and the tile_column_id is always 0. if (layout_type_ == LayoutType::COLUMN) { - tile_offset = column_offset; - tile_column_offset = 0; + tile_id = column_id; + tile_column_id = 0; return; } // For other layouts, fetch the layout and // get the entry in the column map - auto entry = column_layout_.at(column_offset); - tile_offset = entry.first; - tile_column_offset = entry.second; + auto entry = column_layout_.at(column_id); + tile_id = entry.first; + tile_column_id = entry.second; } double Layout::GetLayoutDifference(const storage::Layout &other) const { diff --git a/src/storage/tile_group.cpp b/src/storage/tile_group.cpp index 6213a5f540e..69dac07e157 100644 --- a/src/storage/tile_group.cpp +++ b/src/storage/tile_group.cpp @@ -39,7 +39,7 @@ TileGroup::TileGroup(BackendType backend_type, backend_type(backend_type), tile_group_header(tile_group_header), table(table), - num_tuple_slots(tuple_count), + num_tuple_slots_(tuple_count), tile_group_layout_(layout) { tile_count_ = schemas.size(); for (oid_t tile_itr = 0; tile_itr < tile_count_; tile_itr++) { @@ -102,7 +102,7 @@ oid_t TileGroup::GetActiveTupleCount() const { */ void TileGroup::CopyTuple(const Tuple *tuple, const oid_t &tuple_slot_id) { LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; @@ -136,7 +136,7 @@ oid_t TileGroup::InsertTuple(const Tuple *tuple) { oid_t tuple_slot_id = tile_group_header->GetNextEmptyTupleSlot(); LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); // No more slots if (tuple_slot_id == INVALID_OID) { @@ -183,7 +183,7 @@ oid_t TileGroup::InsertTupleFromRecovery(cid_t commit_id, oid_t tuple_slot_id, } LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; @@ -282,7 +282,7 @@ oid_t TileGroup::InsertTupleFromCheckpoint(oid_t tuple_slot_id, if (status == false) return INVALID_OID; LOG_TRACE("Tile Group Id :: %u status :: %u out of %u slots ", tile_group_id, - tuple_slot_id, num_tuple_slots); + tuple_slot_id, num_tuple_slots_); oid_t tile_column_count; oid_t column_itr = 0; diff --git a/test/codegen/table_scan_translator_test.cpp b/test/codegen/table_scan_translator_test.cpp index 3760745b48b..99fda04bb2b 100644 --- a/test/codegen/table_scan_translator_test.cpp +++ b/test/codegen/table_scan_translator_test.cpp @@ -634,9 +634,9 @@ TEST_F(TableScanTranslatorTest, ScanRowLayout) { // Creates a table with LayoutType::ROW and // invokes the TableScanTranslator // - oid_t tuples_per_tilegroup = 100; - oid_t tilegroup_count = 5; - oid_t column_count = 100; + uint32_t tuples_per_tilegroup = 100; + uint32_t tilegroup_count = 5; + uint32_t column_count = 100; bool is_inlined = true; CreateAndLoadTableWithLayout(LayoutType::ROW, tuples_per_tilegroup, tilegroup_count, column_count, is_inlined); @@ -648,9 +648,9 @@ TEST_F(TableScanTranslatorTest, ScanColumnLayout) { // Creates a table with LayoutType::COLUMN and // invokes the TableScanTranslator // - oid_t tuples_per_tilegroup = 100; - oid_t tilegroup_count = 5; - oid_t column_count = 100; + uint32_t tuples_per_tilegroup = 100; + uint32_t tilegroup_count = 5; + uint32_t column_count = 100; bool is_inlined = true; CreateAndLoadTableWithLayout(LayoutType::COLUMN, tuples_per_tilegroup, tilegroup_count, column_count, is_inlined); @@ -670,7 +670,7 @@ TEST_F(TableScanTranslatorTest, MultiLayoutScan) { const int tuples_per_tilegroup = 100; const int col_count = 6; const bool is_inlined = true; - oid_t tuple_count = 100; + uint32_t tuple_count = 100; ///////////////////////////////////////////////////////// // Define the schema. diff --git a/test/codegen/testing_codegen_util.cpp b/test/codegen/testing_codegen_util.cpp index 57231c245c7..ff05144978b 100644 --- a/test/codegen/testing_codegen_util.cpp +++ b/test/codegen/testing_codegen_util.cpp @@ -179,9 +179,9 @@ void PelotonCodeGenTest::LoadTestTable(oid_t table_id, uint32_t num_rows, } void PelotonCodeGenTest::CreateAndLoadTableWithLayout( - peloton::LayoutType layout_type, oid_t tuples_per_tilegroup, - oid_t tile_group_count, oid_t column_count, bool is_inlined) { - oid_t tuple_count = tuples_per_tilegroup * tile_group_count; + peloton::LayoutType layout_type, uint32_t tuples_per_tilegroup, + uint32_t tile_group_count, uint32_t column_count, bool is_inlined) { + uint32_t tuple_count = tuples_per_tilegroup * tile_group_count; ///////////////////////////////////////////////////////// // Define the schema. ///////////////////////////////////////////////////////// From cc8751a752b6a5d878e1044509fbd89bd436fc9c Mon Sep 17 00:00:00 2001 From: poojanilangekar Date: Fri, 11 May 2018 21:12:42 -0400 Subject: [PATCH 42/42] Changed after Prashanth's review --- src/catalog/catalog.cpp | 2 +- src/include/common/exception.h | 13 ++++++++++++- src/include/storage/layout.h | 4 ++-- src/storage/layout.cpp | 4 ++-- src/storage/tile_group_factory.cpp | 4 ++++ 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/catalog/catalog.cpp b/src/catalog/catalog.cpp index fe91b340ed3..a36eb71c4bd 100644 --- a/src/catalog/catalog.cpp +++ b/src/catalog/catalog.cpp @@ -579,7 +579,7 @@ std::shared_ptr Catalog::CreateLayout( bool result = pg_layout->InsertLayout(table_oid, new_layout, pool_.get(), txn); if (!result) { - LOG_DEBUG("Failed to create a new layout for table %u", table_oid); + LOG_ERROR("Failed to create a new layout for table %u", table_oid); return nullptr; } return new_layout; diff --git a/src/include/common/exception.h b/src/include/common/exception.h index 4c201891751..7f25ec20e9f 100644 --- a/src/include/common/exception.h +++ b/src/include/common/exception.h @@ -59,7 +59,8 @@ enum class ExceptionType { SETTINGS = 23, // settings related BINDER = 24, // binder related NETWORK = 25, // network related - OPTIMIZER = 26 // optimizer related + OPTIMIZER = 26, // optimizer related + NULL_POINTER = 27 // nullptr exception }; class Exception : public std::runtime_error { @@ -132,6 +133,8 @@ class Exception : public std::runtime_error { return "Settings"; case ExceptionType::OPTIMIZER: return "Optimizer"; + case ExceptionType::NULL_POINTER: + return "NullPointer"; default: return "Unknown"; } @@ -467,4 +470,12 @@ class OptimizerException : public Exception { : Exception(ExceptionType::OPTIMIZER, msg) {} }; +class NullPointerException : public Exception { + NullPointerException() = delete; + + public: + NullPointerException(std::string msg) + : Exception(ExceptionType::NULL_POINTER, msg) {} +}; + } // namespace peloton diff --git a/src/include/storage/layout.h b/src/include/storage/layout.h index d06fe93427e..ce726d5d729 100644 --- a/src/include/storage/layout.h +++ b/src/include/storage/layout.h @@ -34,7 +34,7 @@ namespace storage { /** @brief used to store the mapping between a tile and its columns * to vector{} */ -typedef std::map>> tile_map_type; +typedef std::map>> TileToColumnMap; /** * @brief Class to store the physical layout of a TileGroup. @@ -100,7 +100,7 @@ class Layout : public Printable { uint32_t GetColumnCount() const { return num_columns_; } /** @brief Returns the tile-columns map for each tile in the TileGroup. */ - tile_map_type GetTileMap() const; + TileToColumnMap GetTileMap() const; /** @brief Constructs the schema for the given layout. This function * is used only in TempTables and LogicalTiles. diff --git a/src/storage/layout.cpp b/src/storage/layout.cpp index 37664a42f77..c03b721087f 100644 --- a/src/storage/layout.cpp +++ b/src/storage/layout.cpp @@ -137,8 +137,8 @@ oid_t Layout::GetTileIdFromColumnId(oid_t column_id) const { return tile_offset; } -tile_map_type Layout::GetTileMap() const { - tile_map_type tile_map; +TileToColumnMap Layout::GetTileMap() const { + TileToColumnMap tile_map; if (layout_type_ == LayoutType::ROW) { // Row store layout, hence all columns are contained in tile 0. diff --git a/src/storage/tile_group_factory.cpp b/src/storage/tile_group_factory.cpp index c1fd897c09b..2c0da483619 100644 --- a/src/storage/tile_group_factory.cpp +++ b/src/storage/tile_group_factory.cpp @@ -31,6 +31,10 @@ TileGroup *TileGroupFactory::GetTileGroup( // Allocate the data on appropriate backend BackendType backend_type = BackendType::MM; // logging::LoggingUtil::GetBackendType(peloton_logging_mode); + // Ensure that the layout of the new TileGroup is not null. + if (layout == nullptr) { + throw NullPointerException("Layout of the TileGroup must be non-null."); + } TileGroupHeader *tile_header = new TileGroupHeader(backend_type, tuple_count); TileGroup *tile_group = new TileGroup(backend_type, tile_header, table,