diff --git a/src/storage/root_page.cc b/src/storage/root_page.cc index f92731a..3238eca 100644 --- a/src/storage/root_page.cc +++ b/src/storage/root_page.cc @@ -24,6 +24,7 @@ #include "root_page.h" #include +#include #include #include "data_page.h" @@ -35,13 +36,19 @@ using vsql::preview_storage::Error; // Define the thread_local static member thread_local RootPage::LastSlotInfo RootPage::s_last_slot_info; -void RootPage::init(Space::Ref space_ref, uint16_t col_len) { +void RootPage::init(Space::Ref space_ref, uint16_t col_len, + uint8_t num_segments, uint8_t num_root_pages, + uint8_t storage_metadata_len) { + // Store N, K, and metadata length first — offset functions depend on them. + m_num_segments = num_segments; + m_num_root_pages = num_root_pages; + m_storage_metadata_len = storage_metadata_len; m_column_size = col_len; // Calculate maximum free slots based on available page space uint32_t page_size = Page::get_size(space_ref); uint32_t available_space = - page_size - FREE_SLOT_ARRAY_OFF - Page::TRAILER_SIZE; + page_size - free_slot_array_off() - Page::TRAILER_SIZE; uint16_t max_slots = static_cast(available_space / FREE_SLOT_LEN); // Cap at the configured maximum @@ -52,56 +59,76 @@ void RootPage::init(Space::Ref space_ref, uint16_t col_len) { m_max_free_slots = max_slots; } -bool RootPage::format(Page &root_page, MtrCtx::Ref mtr, - uint8_t format_version) { +bool RootPage::format(Page &root_page, MtrCtx::Ref mtr, uint8_t format_version, + std::string_view metadata) { assert(root_page.is_loaded(Page::Latch::EXCLUSIVE)); - if (m_column_size == 0 || m_max_free_slots == 0) { + if (m_column_size == 0 || m_max_free_slots == 0 || + metadata.size() > UINT8_MAX) { return true; } // Write version - root_page.write_integer_1(VERSION_OFF, format_version, mtr); + root_page.write_integer_1(version_off(), format_version, mtr); // Write page type root_page.write_integer_1( - PAGE_TYPE_OFF, static_cast(ColumnPageType::ROOT_PAGE), mtr); - - // Write creator name - root_page.write_string(CREATOR_NAME_OFF, CREATOR, CREATOR_NAME_LEN, mtr); + page_type_off(), static_cast(ColumnPageType::ROOT_PAGE), mtr); + + // Write storage metadata: 1-byte length followed by data bytes + root_page.write_integer_1(storage_metadata_len_off(), + static_cast(metadata.size()), mtr); + root_page.write_string( + storage_metadata_off(), + reinterpret_cast(metadata.data()), metadata.size(), + mtr); + + // Write K and initialize the K-1 other root page REFs to INVALID_REF. + root_page.write_integer_1(num_root_pages_off(), m_num_root_pages, mtr); + for (uint8_t i = 0; i < m_num_root_pages - 1; ++i) { + Page::Offset ref_off = other_root_pages_off() + i * ROOT_PAGE_REF_LEN; + root_page.write_integer_4(ref_off, Page::INVALID_REF, mtr); + } // Write column size - root_page.write_integer_2(COLUMN_SIZE_OFF, m_column_size, mtr); + root_page.write_integer_2(column_size_off(), m_column_size, mtr); // Write data page head (initially invalid - no pages yet) - root_page.write_integer_4(ALL_SLOT_HEAD_OFF, Page::INVALID_REF, mtr); + root_page.write_integer_4(all_slot_head_off(), Page::INVALID_REF, mtr); // Write data page tail (initially invalid - no pages yet) - root_page.write_integer_4(ALL_SLOT_TAIL_OFF, Page::INVALID_REF, mtr); + root_page.write_integer_4(all_slot_tail_off(), Page::INVALID_REF, mtr); // Total data pages counter is 0. Page data is zero filled by default. - assert(0 == root_page.read_integer_4(TOTAL_DATA_PAGES_OFF)); + assert(0 == root_page.read_integer_4(total_data_pages_off())); // Total free pages counter is 0. - assert(0 == root_page.read_integer_4(TOTAL_FREE_PAGES_OFF)); + assert(0 == root_page.read_integer_4(total_free_pages_off())); // Write free slot array max size - root_page.write_integer_2(FREE_SLOT_ARRAY_MAX_SIZE_OFF, m_max_free_slots, + root_page.write_integer_2(free_slot_array_max_size_off(), m_max_free_slots, mtr); // Write free slot array current size - root_page.write_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF, + root_page.write_integer_2(free_slot_array_cur_size_off(), NUM_FREE_SLOTS_INITIAL, mtr); // Initialize NUM_FREE_SLOTS_INITIAL slots to NULL_FREE_PAGE_REF for (uint16_t i = 0; i < NUM_FREE_SLOTS_INITIAL; i++) { - Page::Offset slot_offset = FREE_SLOT_ARRAY_OFF + (i * FREE_SLOT_LEN); + Page::Offset slot_offset = free_slot_array_off() + i * FREE_SLOT_LEN; root_page.write_integer_4(slot_offset, NULL_FREE_PAGE_REF, mtr); } return false; } +std::string RootPage::read_metadata(const Page &root_page) const { + uint8_t len = root_page.read_integer_1(storage_metadata_len_off()); + const char *raw = reinterpret_cast(root_page.get_data() + + storage_metadata_off()); + return std::string(raw, len); +} + uint16_t RootPage::get_random_slot(uint16_t max_size) { if (max_size == 0) { return 0; @@ -144,7 +171,7 @@ void RootPage::grow_free_slots(Page &root_page, MtrCtx::Ref mtr) { // Step 1: Read current free slots count uint16_t cur_free_slots = - root_page.read_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF); + root_page.read_integer_2(free_slot_array_cur_size_off()); // Step 2: We should only be called if we can grow assert(cur_free_slots < m_max_free_slots); @@ -161,12 +188,13 @@ void RootPage::grow_free_slots(Page &root_page, MtrCtx::Ref mtr) { // Step 4: Initialize new slots to NULL_FREE_PAGE_REF for (uint16_t i = cur_free_slots; i < new_free_slots; i++) { - Page::Offset slot_offset = FREE_SLOT_ARRAY_OFF + (i * FREE_SLOT_LEN); + Page::Offset slot_offset = free_slot_array_off() + (i * FREE_SLOT_LEN); root_page.write_integer_4(slot_offset, NULL_FREE_PAGE_REF, mtr); } // Step 5: Update current free slots count - root_page.write_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF, new_free_slots, mtr); + root_page.write_integer_2(free_slot_array_cur_size_off(), new_free_slots, + mtr); } void RootPage::page_select(Page &root_page, Space::Ref space_ref, @@ -175,7 +203,7 @@ void RootPage::page_select(Page &root_page, Space::Ref space_ref, root_page.is_loaded(Page::Latch::SHARED)); // Step 1: Extract current size of free slot array from root page - cur_free_slots = root_page.read_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF); + cur_free_slots = root_page.read_integer_2(free_slot_array_cur_size_off()); // Free slot array size should always be at least 1 assert(cur_free_slots > 0); @@ -200,7 +228,7 @@ void RootPage::page_select(Page &root_page, Space::Ref space_ref, // Step 3: Read data page reference from the slot Page::Offset slot_offset = - FREE_SLOT_ARRAY_OFF + (slot_number * FREE_SLOT_LEN); + free_slot_array_off() + (slot_number * FREE_SLOT_LEN); data_page_ref = root_page.read_integer_4(slot_offset); if (data_page_ref == NULL_FREE_PAGE_REF) { @@ -219,10 +247,11 @@ bool RootPage::add_free_page(Page &root_page, Page &data_page, Page::Ref root_page_ref = root_page.get_ref(); // Step 1: Validate slot_number - assert(slot_number < root_page.read_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF)); + assert(slot_number < + root_page.read_integer_2(free_slot_array_cur_size_off())); Page::Offset slot_offset = - FREE_SLOT_ARRAY_OFF + (slot_number * FREE_SLOT_LEN); + free_slot_array_off() + (slot_number * FREE_SLOT_LEN); Page::Ref slot_page_ref = root_page.read_integer_4(slot_offset); // Step 2: Setup links for the current slot page. @@ -247,8 +276,9 @@ bool RootPage::add_free_page(Page &root_page, Page &data_page, data_page_info->set_free_slot_number(data_page, slot_number, mtr); // Step 5: Increment total free pages counter - uint32_t total_free_pages = root_page.read_integer_4(TOTAL_FREE_PAGES_OFF); - root_page.write_integer_4(TOTAL_FREE_PAGES_OFF, total_free_pages + 1, mtr); + Page::Offset free_pages_offset = total_free_pages_off(); + uint32_t total_free_pages = root_page.read_integer_4(free_pages_offset); + root_page.write_integer_4(free_pages_offset, total_free_pages + 1, mtr); return false; } @@ -264,10 +294,11 @@ bool RootPage::remove_free_page(Page &root_page, Page &data_page, // Step 1: Get free slot number from data page. uint16_t slot_number = data_page_info->get_free_slot_number(data_page); assert(slot_number != DataPage::INVALID_SLOT); - assert(slot_number < root_page.read_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF)); + assert(slot_number < + root_page.read_integer_2(free_slot_array_cur_size_off())); Page::Offset slot_offset = - FREE_SLOT_ARRAY_OFF + (slot_number * FREE_SLOT_LEN); + free_slot_array_off() + (slot_number * FREE_SLOT_LEN); #ifndef NDEBUG Page::Ref data_page_ref = data_page.get_ref(); @@ -321,9 +352,9 @@ bool RootPage::remove_free_page(Page &root_page, Page &data_page, data_page_info->set_free_links(data_page, &prev_ref, &prev_ref, mtr); // Step 6: Decrement total free pages counter - uint32_t total_free_pages = root_page.read_integer_4(TOTAL_FREE_PAGES_OFF); + uint32_t total_free_pages = root_page.read_integer_4(total_free_pages_off()); assert(total_free_pages > 0); - root_page.write_integer_4(TOTAL_FREE_PAGES_OFF, total_free_pages - 1, mtr); + root_page.write_integer_4(total_free_pages_off(), total_free_pages - 1, mtr); return false; } @@ -337,16 +368,16 @@ bool RootPage::add_data_page(Page &root_page, Page &data_page, Page::Ref data_page_ref = data_page.get_ref(); Page::Ref invalid_ref = Page::INVALID_REF; - Page::Ref head_ref = root_page.read_integer_4(ALL_SLOT_HEAD_OFF); + Page::Ref head_ref = root_page.read_integer_4(all_slot_head_off()); #ifndef NDEBUG - Page::Ref tail_ref = root_page.read_integer_4(ALL_SLOT_TAIL_OFF); + Page::Ref tail_ref = root_page.read_integer_4(all_slot_tail_off()); #endif // NDEBUG // 1. Set Current head page links. if (head_ref == Page::INVALID_REF) { // Current head is empty. It is the first page being inserted. assert(tail_ref == Page::INVALID_REF); - root_page.write_integer_4(ALL_SLOT_TAIL_OFF, data_page_ref, mtr); + root_page.write_integer_4(all_slot_tail_off(), data_page_ref, mtr); } else { Page head_page; Error error = @@ -362,11 +393,11 @@ bool RootPage::add_data_page(Page &root_page, Page &data_page, data_page.write_links(invalid_ref, head_ref, mtr); // 3. Set root page head and tails links. - root_page.write_integer_4(ALL_SLOT_HEAD_OFF, data_page_ref, mtr); + root_page.write_integer_4(all_slot_head_off(), data_page_ref, mtr); // 4. Increment total data pages counter - uint32_t total_data_pages = root_page.read_integer_4(TOTAL_DATA_PAGES_OFF); - root_page.write_integer_4(TOTAL_DATA_PAGES_OFF, total_data_pages + 1, mtr); + uint32_t total_data_pages = root_page.read_integer_4(total_data_pages_off()); + root_page.write_integer_4(total_data_pages_off(), total_data_pages + 1, mtr); return false; } @@ -383,8 +414,8 @@ bool RootPage::remove_data_page(Page &root_page, Page &data_page, #ifndef NDEBUG Page::Ref data_page_ref = data_page.get_ref(); - Page::Ref head_ref = root_page.read_integer_4(ALL_SLOT_HEAD_OFF); - Page::Ref tail_ref = root_page.read_integer_4(ALL_SLOT_TAIL_OFF); + Page::Ref head_ref = root_page.read_integer_4(all_slot_head_off()); + Page::Ref tail_ref = root_page.read_integer_4(all_slot_tail_off()); #endif // NDEBUG Page prev_page; @@ -412,7 +443,7 @@ bool RootPage::remove_data_page(Page &root_page, Page &data_page, if (prev_ref == Page::INVALID_REF) { // This is the head page, update root's head pointer assert(head_ref == data_page_ref); - root_page.write_integer_4(ALL_SLOT_HEAD_OFF, next_ref, mtr); + root_page.write_integer_4(all_slot_head_off(), next_ref, mtr); } else { // Update the previous page's next link @@ -423,7 +454,7 @@ bool RootPage::remove_data_page(Page &root_page, Page &data_page, if (next_ref == Page::INVALID_REF) { // This is the tail page, update root's tail pointer assert(tail_ref == data_page_ref); - root_page.write_integer_4(ALL_SLOT_TAIL_OFF, prev_ref, mtr); + root_page.write_integer_4(all_slot_tail_off(), prev_ref, mtr); } else { // Update the next page's previous link @@ -434,9 +465,9 @@ bool RootPage::remove_data_page(Page &root_page, Page &data_page, data_page.write_links(Page::INVALID_REF, Page::INVALID_REF, mtr); // 6. Decrement total data pages counter - uint32_t total_data_pages = root_page.read_integer_4(TOTAL_DATA_PAGES_OFF); + uint32_t total_data_pages = root_page.read_integer_4(total_data_pages_off()); assert(total_data_pages > 0); - root_page.write_integer_4(TOTAL_DATA_PAGES_OFF, total_data_pages - 1, mtr); + root_page.write_integer_4(total_data_pages_off(), total_data_pages - 1, mtr); return false; } @@ -447,12 +478,13 @@ uint16_t RootPage::get_free_slot(Page &root_page) { // Step 1: Get current number of free slots uint16_t cur_free_slots = - root_page.read_integer_2(FREE_SLOT_ARRAY_CUR_SIZE_OFF); + root_page.read_integer_2(free_slot_array_cur_size_off()); assert(cur_free_slots > 0); // Step 2: Search for an empty slot (NULL_FREE_PAGE_REF) for (uint16_t slot_idx = 0; slot_idx < cur_free_slots; slot_idx++) { - Page::Offset slot_offset = FREE_SLOT_ARRAY_OFF + (slot_idx * FREE_SLOT_LEN); + Page::Offset slot_offset = + free_slot_array_off() + (slot_idx * FREE_SLOT_LEN); Page::Ref slot_page_ref = root_page.read_integer_4(slot_offset); if (slot_page_ref == NULL_FREE_PAGE_REF) { diff --git a/src/storage/root_page.h b/src/storage/root_page.h index 68e339a..ff346fd 100644 --- a/src/storage/root_page.h +++ b/src/storage/root_page.h @@ -25,6 +25,8 @@ #define VILLAGESQL_EXAMPLES_VSQL_SVECTOR_SRC_ROOT_PAGE_H #include +#include +#include #include @@ -36,7 +38,6 @@ using vsql::preview_storage::Segment; using vsql::preview_storage::Space; // Forward declarations -struct ColumnStorageContext; struct DataPage; // Page type identifiers for SVECTOR column storage @@ -65,13 +66,16 @@ struct RootPage { // Thread-local storage for last slot information static thread_local LastSlotInfo s_last_slot_info; - public: - // We create one segment for storing the column data. - static constexpr uint8_t NUM_SEGMENTS = 1; - +public: // Root Page format - Version-1 - // [Page Header] [N] [Segment Header] [Version] [Type] [Creator Name] - // |-----38-----|-1-|-------10-------|----1----|--1--|------8-------| + // [Page Header] [N] [Segment Header] [Version] [Type] + // |-----38-----|-1-|-----N x 10-----|----1----|--1--| + // + // [Metadata Len(M)] [Storage Metadata] + // |--------1-------|--------M---------| + // + // [Number of root pages (K)] [Other root page REFs] + // |------------1------------|----4 x (K - 1)------| // // [Column Size] [Data Page Head] [Data Page Tail] // |-----2------|-------4--------|-------4-------| @@ -85,70 +89,98 @@ struct RootPage { // [Free Slot Array: Max 2k slots] [Left Over] [Page Trailer] // |------------M * 4-------------|-----L-----|-----8-------| - static constexpr Page::Offset VERSION_OFF = - Page::HEADER_SIZE + Segment::NUM_SEGMENTS_SIZE + - Segment::HEADER_SIZE * NUM_SEGMENTS; + // Fixed-size field lengths. All offsets are dynamic based on number of + // segments (N) and Number of root pages(K). static constexpr Page::Offset VERSION_LEN = 1; - - static constexpr Page::Offset PAGE_TYPE_OFF = VERSION_OFF + VERSION_LEN; static constexpr Page::Offset PAGE_TYPE_LEN = 1; - - static constexpr Page::Offset CREATOR_NAME_OFF = - PAGE_TYPE_OFF + PAGE_TYPE_LEN; - static constexpr Page::Offset CREATOR_NAME_LEN = 8; - - static constexpr Page::Offset COLUMN_SIZE_OFF = - CREATOR_NAME_OFF + CREATOR_NAME_LEN; + static constexpr Page::Offset STORAGE_METADATA_LEN_SIZE = 1; + static constexpr Page::Offset NUM_ROOT_PAGES_LEN = 1; + static constexpr Page::Offset ROOT_PAGE_REF_LEN = 4; static constexpr Page::Offset COLUMN_SIZE_LEN = 2; - - // Head of all partially full and completely full pages. - static constexpr Page::Offset ALL_SLOT_HEAD_OFF = - COLUMN_SIZE_OFF + COLUMN_SIZE_LEN; static constexpr Page::Offset ALL_SLOT_HEAD_LEN = 4; - - // Tail of all partially full and completely full pages. - static constexpr Page::Offset ALL_SLOT_TAIL_OFF = - ALL_SLOT_HEAD_OFF + ALL_SLOT_HEAD_LEN; static constexpr Page::Offset ALL_SLOT_TAIL_LEN = 4; - - // Total number of data pages (both free and full) - static constexpr Page::Offset TOTAL_DATA_PAGES_OFF = - ALL_SLOT_TAIL_OFF + ALL_SLOT_TAIL_LEN; static constexpr Page::Offset TOTAL_DATA_PAGES_LEN = 4; - - // Total number of free pages (partially full pages) - static constexpr Page::Offset TOTAL_FREE_PAGES_OFF = - TOTAL_DATA_PAGES_OFF + TOTAL_DATA_PAGES_LEN; static constexpr Page::Offset TOTAL_FREE_PAGES_LEN = 4; - - static constexpr Page::Offset FREE_SLOT_ARRAY_MAX_SIZE_OFF = - TOTAL_FREE_PAGES_OFF + TOTAL_FREE_PAGES_LEN; static constexpr Page::Offset FREE_SLOT_ARRAY_MAX_SIZE_LEN = 2; - - static constexpr Page::Offset FREE_SLOT_ARRAY_CUR_SIZE_OFF = - FREE_SLOT_ARRAY_MAX_SIZE_OFF + FREE_SLOT_ARRAY_MAX_SIZE_LEN; static constexpr Page::Offset FREE_SLOT_ARRAY_CUR_SIZE_LEN = 2; - // Each free slot holds the free (partially full) page reference. - // Zero, if no free page. - static constexpr Page::Offset FREE_SLOT_ARRAY_OFF = - FREE_SLOT_ARRAY_CUR_SIZE_OFF + FREE_SLOT_ARRAY_CUR_SIZE_LEN; static constexpr Page::Offset FREE_SLOT_LEN = 4; - // Start with a single free slot an increase dynamically. + inline Page::Offset version_off() const { + return Page::HEADER_SIZE + Segment::NUM_SEGMENTS_SIZE + + static_cast(Segment::HEADER_SIZE) * m_num_segments; + } + inline Page::Offset page_type_off() const { + return version_off() + VERSION_LEN; + } + inline Page::Offset storage_metadata_len_off() const { + return page_type_off() + PAGE_TYPE_LEN; + } + inline Page::Offset storage_metadata_off() const { + return storage_metadata_len_off() + STORAGE_METADATA_LEN_SIZE; + } + inline Page::Offset num_root_pages_off() const { + return storage_metadata_off() + m_storage_metadata_len; + } + inline Page::Offset other_root_pages_off() const { + return num_root_pages_off() + NUM_ROOT_PAGES_LEN; + } + + inline Page::Offset column_size_off() const { + return other_root_pages_off() + ROOT_PAGE_REF_LEN * (m_num_root_pages - 1); + } + inline Page::Offset all_slot_head_off() const { + return column_size_off() + COLUMN_SIZE_LEN; + } + inline Page::Offset all_slot_tail_off() const { + return all_slot_head_off() + ALL_SLOT_HEAD_LEN; + } + inline Page::Offset total_data_pages_off() const { + return all_slot_tail_off() + ALL_SLOT_TAIL_LEN; + } + inline Page::Offset total_free_pages_off() const { + return total_data_pages_off() + TOTAL_DATA_PAGES_LEN; + } + inline Page::Offset free_slot_array_max_size_off() const { + return total_free_pages_off() + TOTAL_FREE_PAGES_LEN; + } + inline Page::Offset free_slot_array_cur_size_off() const { + return free_slot_array_max_size_off() + FREE_SLOT_ARRAY_MAX_SIZE_LEN; + } + inline Page::Offset free_slot_array_off() const { + return free_slot_array_cur_size_off() + FREE_SLOT_ARRAY_CUR_SIZE_LEN; + } + + // Start with a single free slot and increase dynamically. static constexpr uint16_t NUM_FREE_SLOTS_INITIAL = 1; // NULL reference for free page list. Use same value as Page::INVALID_REF. static constexpr uint32_t NULL_FREE_PAGE_REF = Page::INVALID_REF; - // Creator name - static constexpr unsigned char CREATOR[CREATOR_NAME_LEN] = "SVECTOR"; - - // Initialize root page parameters (calculate capacity). - void init(Space::Ref space_ref, uint16_t col_len); + // Set N, M, and K without full initialization. Used during load() to + // bootstrap offset functions before col_len is known. Must be followed by + // init(). Call in three stages during load: + // 1. set_layout(N, 0, 1) — enables storage_metadata_len_off() + // 2. set_layout(N, M, 1) — enables num_root_pages_off() + // 3. set_layout(N, M, K) — final layout + void set_layout(uint8_t num_segments, uint8_t storage_metadata_len, + uint8_t num_root_pages) { + m_num_segments = num_segments; + m_storage_metadata_len = storage_metadata_len; + m_num_root_pages = num_root_pages; + } + + // Initialize root page parameters. num_segments (N), num_root_pages (K), and + // storage_metadata_len are fixed at creation time and must not change. + void init(Space::Ref space_ref, uint16_t col_len, uint8_t num_segments, + uint8_t num_root_pages, uint8_t storage_metadata_len); // Format root page. - bool format(Page &root_page, MtrCtx::Ref mtr, uint8_t format_version); + bool format(Page &root_page, MtrCtx::Ref mtr, uint8_t format_version, + std::string_view metadata); + + // Read the storage metadata from a formatted root page. + std::string read_metadata(const Page &root_page) const; // Select a data page for insert with latch on root. This function examines // the root page to find a suitable data page. @@ -201,6 +233,7 @@ struct RootPage { MtrCtx::Ref mtr); // Getters for member variables + uint8_t get_num_segments() const { return m_num_segments; } uint16_t get_column_size() const { return m_column_size; } uint16_t get_max_free_slots() const { return m_max_free_slots; } @@ -215,11 +248,17 @@ struct RootPage { static bool get_cached_slot_number(Space::Ref space_ref, Page::Ref page_ref, uint16_t max_slots, uint16_t &slot_number); + // Set by init() and fixed for the lifetime of this RootPage. Must be + // initialized before any offset function is called. + uint8_t m_num_segments = 0; + uint8_t m_num_root_pages = 0; + uint8_t m_storage_metadata_len = 0; + // Maximum number of free slots for concurrent insert. Might need to reduce // based on page size. uint16_t m_max_free_slots = 2048; - // Initialized by create_storage and not changed later. + // Initialized by init() and not changed later. uint16_t m_column_size = 0; }; diff --git a/src/storage/storage.cc b/src/storage/storage.cc index 9a7e1b2..1505252 100644 --- a/src/storage/storage.cc +++ b/src/storage/storage.cc @@ -53,8 +53,8 @@ class ConcurrencyGuard { std::atomic &m_counter; }; -void ColumnStorageContext::fill_error(const char *info, char *msg, uint32_t len, - bool local) { +void MultiColumnStore::fill_error(const char *info, char *msg, uint32_t len, + bool local) { if (local) { snprintf(msg, len, "SVECTOR: %s", info); } else { @@ -64,21 +64,43 @@ void ColumnStorageContext::fill_error(const char *info, char *msg, uint32_t len, } } -bool ColumnStorageContext::create(Space::Ref space_ref, Segment::TrxRef trx_ref, - uint16_t col_len, char *error_msg, - uint32_t error_msg_len) { - m_root.init(space_ref, col_len); - m_data.init(space_ref, col_len); +void ColumnStore::fill_error(const char *info, char *msg, uint32_t len, + bool local) { + MultiColumnStore::fill_error(info, msg, len, local); +} + +bool MultiColumnStore::create(Space::Ref space_ref, Segment::TrxRef trx_ref, + const std::vector &storages, + uint8_t num_segments, char *error_msg, + uint32_t error_msg_len) { + assert(!storages.empty()); + size_t num_storages = storages.size(); + assert(m_stores.empty()); + m_stores.resize(num_storages); + + ColumnStore &primary = m_stores[0]; + assert(num_storages >= (size_t)num_segments); Page::Ref root_page_ref; - if (Segment::create(space_ref, RootPage::NUM_SEGMENTS, trx_ref, - root_page_ref) != Error::SUCCESS) { + if (Segment::create(space_ref, num_segments, trx_ref, root_page_ref) != + Error::SUCCESS) { fill_error("create: failed to create segment", error_msg, error_msg_len, false); return true; } + auto num_root_pages = static_cast(num_storages); + primary.init(space_ref, root_page_ref, storages[0].col_len, num_segments, + num_root_pages, storages[0].metadata); + + // Additional stores do not have root pages yet. Their root pages will be + // allocated and assigned on first use. + for (size_t i = 1; i < storages.size(); ++i) { + m_stores[i].init(space_ref, Page::INVALID_REF, storages[i].col_len, + num_segments, num_root_pages, storages[i].metadata); + } + { MtrCtx mtr_ctx; auto mtr = mtr_ctx.start(); @@ -92,7 +114,8 @@ bool ColumnStorageContext::create(Space::Ref space_ref, Segment::TrxRef trx_ref, return true; } - bool err = m_root.format(root_page, mtr, FORMAT_VERSION); + bool err = primary.m_root.format(root_page, mtr, FORMAT_VERSION, + primary.m_metadata); mtr_ctx.commit(); if (err) { @@ -106,8 +129,8 @@ bool ColumnStorageContext::create(Space::Ref space_ref, Segment::TrxRef trx_ref, return false; } -bool ColumnStorageContext::drop(Segment::TrxRef trx_ref, char *error_msg, - uint32_t error_msg_len) { +bool MultiColumnStore::drop(Segment::TrxRef trx_ref, char *error_msg, + uint32_t error_msg_len) { Space::Ref space_ref; Page::Ref page_ref; decode_ref(space_ref, page_ref); @@ -119,9 +142,161 @@ bool ColumnStorageContext::drop(Segment::TrxRef trx_ref, char *error_msg, return false; } -bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, - Column::Data col_data, Column::Ref &col_ref, - char *error_msg, uint32_t error_msg_len) { +bool MultiColumnStore::init_root_page(uint8_t seg_idx, uint8_t root_idx, + char *error_msg, uint32_t error_msg_len) { + assert(root_idx > 0 && root_idx < m_stores.size()); + + ColumnStore &primary = m_stores[0]; + ColumnStore &target = m_stores[root_idx]; + Space::Ref space_ref = primary.m_space_ref; + + MtrCtx mtr_ctx; + auto mtr = mtr_ctx.start(); + + Page primary_root; + if (primary_root.load(space_ref, primary.m_root_page_ref, + Page::Latch::EXCLUSIVE, mtr) != Error::SUCCESS) { + fill_error("init_root_page: failed to load primary root page", error_msg, + error_msg_len, false); + mtr_ctx.commit(); + return true; + } + + Segment::Ref seg_head = Segment::get_header(primary_root, seg_idx); + + Page new_page; + if (new_page.load_new(seg_head, mtr) != Error::SUCCESS) { + fill_error("init_root_page: failed to allocate page", error_msg, + error_msg_len, false); + mtr_ctx.commit(); + return true; + } + + if (target.m_root.format(new_page, mtr, FORMAT_VERSION, target.m_metadata)) { + fill_error("init_root_page: failed to format root page", error_msg, + error_msg_len, true); + mtr_ctx.commit(); + return true; + } + + Page::Ref new_ref = new_page.get_ref(); + Page::Offset ref_off = + primary.m_root.other_root_pages_off() + + static_cast(root_idx - 1) * RootPage::ROOT_PAGE_REF_LEN; + primary_root.write_integer_4(ref_off, new_ref, mtr); + + mtr_ctx.commit(); + + target.m_root_page_ref = new_ref; + return false; +} + +bool MultiColumnStore::load(Column::StorageRef storage_ref, + const std::vector &specs, + char *error_msg, uint32_t error_msg_len) { + m_ref = storage_ref; + Space::Ref space_ref; + Page::Ref root_page_ref; + decode_ref(space_ref, root_page_ref); + + // Bootstraps a RootPage from a latched page and extracts its metadata. + // Returns the bootstrapped RootPage so the caller can use it to compute + // further offsets before the MTR is committed. + struct PageInfo { + uint8_t num_segs; + uint8_t num_root_pages; + uint16_t col_len; + std::string metadata; + RootPage rp; + }; + auto read_page_info = [](const Page &page) -> PageInfo { + RootPage rp; + // Stage 1: set N only (enables storage_metadata_len_off()). + uint8_t ns = page.read_integer_1(Page::HEADER_SIZE); + rp.set_layout(ns, 0, 1); + // Stage 2: read metadata length (enables num_root_pages_off()). + uint8_t ml = page.read_integer_1(rp.storage_metadata_len_off()); + rp.set_layout(ns, ml, 1); + // Stage 3: read K and finalize layout. + uint8_t nrp = page.read_integer_1(rp.num_root_pages_off()); + rp.set_layout(ns, ml, nrp); + return {ns, nrp, page.read_integer_2(rp.column_size_off()), + rp.read_metadata(page), std::move(rp)}; + }; + + MtrCtx mtr_ctx; + auto mtr = mtr_ctx.start(); + Page primary_root; + if (primary_root.load(space_ref, root_page_ref, Page::Latch::SHARED, mtr) != + Error::SUCCESS) { + fill_error("load: failed to load primary root page", error_msg, + error_msg_len, false); + mtr_ctx.commit(); + return true; + } + auto info0 = read_page_info(primary_root); + + // Collect all K-1 other root page refs before releasing the latch. + std::vector other_refs(static_cast(info0.num_root_pages) - + 1); + for (uint8_t i = 0; i < info0.num_root_pages - 1; ++i) { + Page::Offset off = + info0.rp.other_root_pages_off() + + static_cast(i) * RootPage::ROOT_PAGE_REF_LEN; + other_refs[i] = primary_root.read_integer_4(off); + } + mtr_ctx.commit(); + + // Single resize to avoid triggering the assert(false) move constructor. + assert(m_stores.empty()); + m_stores.resize(info0.num_root_pages); + m_stores[0].init(space_ref, root_page_ref, info0.col_len, info0.num_segs, + info0.num_root_pages, info0.metadata); + + assert(info0.num_root_pages == specs.size()); + + for (uint8_t i = 1; i < info0.num_root_pages; ++i) { + Page::Ref ref = other_refs[i - 1]; + if (ref == Page::INVALID_REF) { + m_stores[i].init(space_ref, Page::INVALID_REF, specs[i].col_len, + info0.num_segs, info0.num_root_pages, specs[i].metadata); + continue; + } + + MtrCtx sec_mtr_ctx; + auto sec_mtr = sec_mtr_ctx.start(); + Page sec_root; + if (sec_root.load(space_ref, ref, Page::Latch::SHARED, sec_mtr) != + Error::SUCCESS) { + fill_error("load: failed to load secondary root page", error_msg, + error_msg_len, false); + sec_mtr_ctx.commit(); + return true; + } + auto info = read_page_info(sec_root); + sec_mtr_ctx.commit(); + m_stores[i].init(space_ref, ref, info.col_len, info.num_segs, + info.num_root_pages, info.metadata); + } + + encode_ref(space_ref, root_page_ref); + return false; +} + +void ColumnStore::init(Space::Ref space_ref, Page::Ref root_page_ref, + uint16_t col_len, uint8_t num_segments, + uint8_t num_root_pages, std::string_view metadata) { + m_space_ref = space_ref; + m_root_page_ref = root_page_ref; + m_metadata = metadata; + m_root.init(space_ref, col_len, num_segments, num_root_pages, + static_cast(metadata.size())); + m_data.init(space_ref, col_len); +} + +bool ColumnStore::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Data col_data, Column::Ref &col_ref, + char *error_msg, uint32_t error_msg_len) { // Track concurrency: increment on entry, decrement on exit ConcurrencyGuard concurrency_guard(m_insert_concurrency_counter); @@ -139,48 +314,43 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, return true; } - // Step 1: Decode Space and Page reference (root page) - Space::Ref space_ref; - Page::Ref root_page_ref; - decode_ref(space_ref, root_page_ref); - - // Step 2: Use the provided mtr context + // Step 1: Use the provided mtr context auto mtr = static_cast(mctx); - // Step 3: Load and latch the root page (SHARED latch for optimistic insert) + // Step 2: Load and latch the root page (SHARED latch for optimistic insert) Page root_page; - if (root_page.load(space_ref, root_page_ref, Page::Latch::SHARED, mtr) != + if (root_page.load(m_space_ref, m_root_page_ref, Page::Latch::SHARED, mtr) != Error::SUCCESS) { fill_error("insert: failed to load root page", error_msg, error_msg_len, false); return true; } - // Step 4: Try optimistic page selection + // Step 3: Try optimistic page selection Page::Ref data_page_ref; uint16_t cur_free_slots = 0; - m_root.page_select(root_page, space_ref, data_page_ref, cur_free_slots); + m_root.page_select(root_page, m_space_ref, data_page_ref, cur_free_slots); - // Step 4a: Check if concurrency exceeds available slots and we can grow + // Step 3a: Check if concurrency exceeds available slots and we can grow uint32_t concurrency = m_insert_concurrency_counter.load(std::memory_order_relaxed); bool need_grow = (concurrency > cur_free_slots) && (cur_free_slots < m_root.get_max_free_slots()); - // Step 5: Optimistic path - check if we got a valid page + // Step 4: Optimistic path - check if we got a valid page Page data_page; bool need_pessimistic = need_grow; if (!need_pessimistic && data_page_ref != Page::INVALID_REF) { - // Step 5a: Load data page with X latch - if (data_page.load(space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != - Error::SUCCESS) { + // Step 4a: Load data page with X latch + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::EXCLUSIVE, + mtr) != Error::SUCCESS) { fill_error("insert: failed to load data page", error_msg, error_msg_len, false); return true; } - // Step 5b: Check if data page has its last free slot + // Step 4b: Check if data page has its last free slot // If the page has its last free slot, it will become full after insert // and needs to be removed from the root page's free list. This requires // X latch on root, so we must fall back to pessimistic path. @@ -196,7 +366,7 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, need_pessimistic = true; } - // Step 6: Pessimistic path if needed + // Step 5: Pessimistic path if needed if (need_pessimistic) { // Release S latch on root page if (root_page.release(mtr) != Error::SUCCESS) { @@ -206,14 +376,14 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, } // Acquire X latch on root page - if (root_page.load(space_ref, root_page_ref, Page::Latch::EXCLUSIVE, mtr) != - Error::SUCCESS) { + if (root_page.load(m_space_ref, m_root_page_ref, Page::Latch::EXCLUSIVE, + mtr) != Error::SUCCESS) { fill_error("insert: failed to load root page", error_msg, error_msg_len, false); return true; } - // Step 6a: Grow free slots if needed + // Step 5a: Grow free slots if needed if (need_grow) { m_root.grow_free_slots(root_page, mtr); } @@ -221,7 +391,7 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, // Re-read the cached slot under X latch. Between releasing S and acquiring // X, another thread may have allocated a page in this slot or removed it. // We must re-check to get the current state. - m_root.page_select(root_page, space_ref, data_page_ref, cur_free_slots); + m_root.page_select(root_page, m_space_ref, data_page_ref, cur_free_slots); if (data_page_ref == Page::INVALID_REF) { Segment::Ref seg_head = Segment::get_header(root_page, 0); @@ -231,9 +401,9 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, return true; } - m_data.format(data_page, mtr, FORMAT_VERSION); + m_data.format(data_page, mtr, MultiColumnStore::FORMAT_VERSION); - if (m_root.add_data_page(root_page, data_page, space_ref, mtr)) { + if (m_root.add_data_page(root_page, data_page, m_space_ref, mtr)) { fill_error("insert: failed to register new data page", error_msg, error_msg_len, true); return true; @@ -244,14 +414,14 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, // single vector occupies the whole page; the page will be full after // this insert and must never appear in the free list. if (!m_data.has_last_free_slot(data_page) && - m_root.add_free_page(root_page, data_page, &m_data, space_ref, + m_root.add_free_page(root_page, data_page, &m_data, m_space_ref, RootPage::s_last_slot_info.slot_number, mtr)) { fill_error("insert: failed to register new data page", error_msg, error_msg_len, true); return true; } } else { - if (data_page.load(space_ref, data_page_ref, Page::Latch::EXCLUSIVE, + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != Error::SUCCESS) { fill_error("insert: failed to load data page", error_msg, error_msg_len, false); @@ -259,7 +429,7 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, } if (m_data.has_last_free_slot(data_page)) { - if (m_root.remove_free_page(root_page, data_page, &m_data, space_ref, + if (m_root.remove_free_page(root_page, data_page, &m_data, m_space_ref, mtr)) { fill_error("insert: failed to remove full page from free list", error_msg, error_msg_len, true); @@ -268,7 +438,7 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, } } } else { - // Step 7: Release root page latch. We haven't modified root page. + // Step 6: Release root page latch. We haven't modified root page. if (root_page.release(mtr) != Error::SUCCESS) { fill_error("insert: failed to release root page", error_msg, error_msg_len, false); @@ -276,41 +446,35 @@ bool ColumnStorageContext::insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, } } - // Step 8: Insert into data page (hint will be updated) + // Step 7: Insert into data page (hint will be updated) assert(data_page.is_loaded()); m_data.insert(data_page, mtr, trx_ref, col_data, col_ref); return false; } -bool ColumnStorageContext::fetch(MtrCtx::Ref mctx, Column::Ref col_ref, - Column::Data &col_data, - Column::Data &rowid_prefix, - Segment::TrxRef &trx_ref, bool &delete_marked, - char *error_msg, uint32_t error_msg_len) { +bool ColumnStore::fetch(MtrCtx::Ref mctx, Column::Ref col_ref, + Column::Data &col_data, Column::Data &rowid_prefix, + Segment::TrxRef &trx_ref, bool &delete_marked, + char *error_msg, uint32_t error_msg_len) { // Step 1: Decode column reference to get page and slot Page::Ref data_page_ref; uint16_t slot_index; DataPage::decode_column_ref(col_ref, data_page_ref, slot_index); - // Step 2: Get space reference from storage context - Space::Ref space_ref; - Page::Ref root_page_ref; - decode_ref(space_ref, root_page_ref); - - // Step 3: Use the provided mtr context + // Step 2: Use the provided mtr context auto mtr = static_cast(mctx); - // Step 4: Load data page with S latch + // Step 3: Load data page with S latch Page data_page; - if (data_page.load(space_ref, data_page_ref, Page::Latch::SHARED, mtr) != + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::SHARED, mtr) != Error::SUCCESS) { fill_error("fetch: failed to load data page", error_msg, error_msg_len, false); return true; } - // Step 5: Get record status (delete_marked and is_free) + // Step 4: Get record status (delete_marked and is_free) bool is_free = true; std::tie(delete_marked, is_free) = m_data.get_record_status(data_page, slot_index); @@ -322,50 +486,43 @@ bool ColumnStorageContext::fetch(MtrCtx::Ref mctx, Column::Ref col_ref, return true; } - // Step 7: Read transaction reference from record + // Step 5: Read transaction reference from record Page::Offset rec_offset = m_data.get_record_offset(slot_index); trx_ref = data_page.read_integer_8(rec_offset); - // Step 8: Set column data pointer to the column data in the page + // Step 6: Set column data pointer to the column data in the page Page::Offset col_data_offset = rec_offset + DataPage::TRX_REF_SIZE; col_data.data = data_page.get_data() + col_data_offset; col_data.length = m_root.get_column_size(); - // Step 9: We don't store rowid_prefix for SVECTOR + // Step 7: We don't store rowid_prefix for SVECTOR rowid_prefix.data = nullptr; rowid_prefix.length = 0; return false; } -bool ColumnStorageContext::mark_delete(MtrCtx::Ref mctx, - Segment::TrxRef trx_ref, - Column::Ref col_ref, bool delete_mark, - char *error_msg, - uint32_t error_msg_len) { +bool ColumnStore::mark_delete(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Ref col_ref, bool delete_mark, + char *error_msg, uint32_t error_msg_len) { // Step 1: Decode column reference to get page and slot Page::Ref data_page_ref; uint16_t slot_index; DataPage::decode_column_ref(col_ref, data_page_ref, slot_index); - // Step 2: Get space reference from storage context - Space::Ref space_ref; - Page::Ref root_page_ref; - decode_ref(space_ref, root_page_ref); - - // Step 3: Use the provided mtr context + // Step 2: Use the provided mtr context auto mtr = static_cast(mctx); - // Step 4: Load data page with X latch + // Step 3: Load data page with X latch Page data_page; - if (data_page.load(space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != Error::SUCCESS) { fill_error("mark_delete: failed to load data page", error_msg, error_msg_len, false); return true; } - // Step 5: Mark or unmark the record as deleted based on delete_mark parameter + // Step 4: Mark or unmark the record as deleted based on delete_mark parameter Page::Offset rec_offset = m_data.get_record_offset(slot_index); Segment::TrxRef old_trx_ref = data_page.read_integer_8(rec_offset); bool trx_id_match = (old_trx_ref == trx_ref); @@ -376,7 +533,7 @@ bool ColumnStorageContext::mark_delete(MtrCtx::Ref mctx, m_data.set_record_undelete(data_page, slot_index, trx_id_match, mtr); } - // Step 6: Update the transaction reference + // Step 5: Update the transaction reference if (!trx_id_match) { data_page.write_integer_8(rec_offset, trx_ref, mtr); } @@ -384,36 +541,31 @@ bool ColumnStorageContext::mark_delete(MtrCtx::Ref mctx, return false; } -bool ColumnStorageContext::purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, - Column::Ref col_ref, char *error_msg, - uint32_t error_msg_len) { +bool ColumnStore::purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Ref col_ref, char *error_msg, + uint32_t error_msg_len) { // Step 1: Decode column reference to get page and slot Page::Ref data_page_ref; uint16_t slot_index; DataPage::decode_column_ref(col_ref, data_page_ref, slot_index); - // Step 2: Get space reference from storage context - Space::Ref space_ref; - Page::Ref root_page_ref; - decode_ref(space_ref, root_page_ref); - - // Step 3: Use the provided mtr context + // Step 2: Use the provided mtr context auto mtr = static_cast(mctx); - // Step 4: Load data page with X latch + // Step 3: Load data page with X latch Page data_page; - if (data_page.load(space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != Error::SUCCESS) { fill_error("purge: failed to load data page", error_msg, error_msg_len, false); return true; } - // Step 5: Check if page will need to be added to free list after purge + // Step 4: Check if page will need to be added to free list after purge bool need_pessimistic = m_data.needs_add_to_free_list(data_page, true); Page root_page; - // Step 6: If page needs to be added to free list, follow pessimistic path + // Step 5: If page needs to be added to free list, follow pessimistic path if (need_pessimistic) { // Release data page X latch if (data_page.release(mtr) != Error::SUCCESS) { @@ -423,23 +575,23 @@ bool ColumnStorageContext::purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, } // Load root page with X latch - if (root_page.load(space_ref, root_page_ref, Page::Latch::EXCLUSIVE, mtr) != - Error::SUCCESS) { + if (root_page.load(m_space_ref, m_root_page_ref, Page::Latch::EXCLUSIVE, + mtr) != Error::SUCCESS) { fill_error("purge: failed to load root page", error_msg, error_msg_len, false); return true; } // Re-load data page with X latch - if (data_page.load(space_ref, data_page_ref, Page::Latch::EXCLUSIVE, mtr) != - Error::SUCCESS) { + if (data_page.load(m_space_ref, data_page_ref, Page::Latch::EXCLUSIVE, + mtr) != Error::SUCCESS) { fill_error("purge: failed to reload data page", error_msg, error_msg_len, false); return true; } } - // Step 7: Purge the record + // Step 6: Purge the record bool purged = false; if (m_data.purge(data_page, mtr, slot_index, trx_ref, purged)) { char info[64]; @@ -449,11 +601,11 @@ bool ColumnStorageContext::purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, return true; } - // Step 8: Add page to free list, after Re-check + // Step 7: Add page to free list, after Re-check if (need_pessimistic && purged && m_data.needs_add_to_free_list(data_page)) { // Get a free slot from root page to add this page uint16_t slot_number = m_root.get_free_slot(root_page); - if (m_root.add_free_page(root_page, data_page, &m_data, space_ref, + if (m_root.add_free_page(root_page, data_page, &m_data, m_space_ref, slot_number, mtr)) { char info[64]; snprintf(info, sizeof(info), "purge: failed to add page %u to free list", @@ -482,10 +634,11 @@ bool ColumnStorage::create(Ctx *storage, Space::Ref space, // Subtract the storage_ref prefix from the column length stored per record. uint16_t store_len = static_cast(col_len - sizeof(Column::Ref)); + constexpr uint8_t NUM_SEGMENTS = 1; auto *col_store = storage->user(); - bool err = - col_store->create(space, trx_ref, store_len, error_msg, error_msg_len); + bool err = col_store->create(space, trx_ref, {{store_len, "SVECTOR"}}, + NUM_SEGMENTS, error_msg, error_msg_len); if (!err) storage->set_ref(col_store->m_ref); return err; } @@ -497,32 +650,8 @@ bool ColumnStorage::drop(Ctx *storage, Segment::TrxRef trx_ref, char *error_msg, bool ColumnStorage::load(Ctx *storage, Column::StorageRef storage_ref, char *error_msg, uint32_t error_msg_len) { - auto *col_store = storage->user(); - col_store->m_ref = storage_ref; - - Space::Ref space_ref; - Page::Ref root_page_ref; - col_store->decode_ref(space_ref, root_page_ref); - - MtrCtx mtr_ctx; - auto mtr = mtr_ctx.start(); - - Page root_page; - if (root_page.load(space_ref, root_page_ref, Page::Latch::SHARED, mtr) != - Error::SUCCESS) { - snprintf(error_msg, error_msg_len, - "SVECTOR: load: failed to load root page %u", root_page_ref); - mtr_ctx.commit(); - return true; - } - - uint16_t col_len = root_page.read_integer_2(RootPage::COLUMN_SIZE_OFF); - mtr_ctx.commit(); - - col_store->m_root.init(space_ref, col_len); - col_store->m_data.init(space_ref, col_len); - - return false; + return storage->user()->load(storage_ref, {{0, ""}}, error_msg, + error_msg_len); } bool ColumnStorage::insert(Ctx *storage, MtrCtx::Ref mctx, @@ -532,32 +661,32 @@ bool ColumnStorage::insert(Ctx *storage, MtrCtx::Ref mctx, // Ignore rowid prefix. Currently we don't support fetching the record back // from column reference. (void)rowid_prefix; - return storage->user()->insert(mctx, trx_ref, col_data, *col_ref, error_msg, - error_msg_len); + return storage->user()->m_stores[0].insert(mctx, trx_ref, col_data, *col_ref, + error_msg, error_msg_len); } bool ColumnStorage::select(Ctx *storage, MtrCtx::Ref mctx, Column::Ref col_ref, Column::Data *col_data, Column::Data *rowid_prefix, Segment::TrxRef *trx_ref, bool *delete_marked, char *error_msg, uint32_t error_msg_len) { - return storage->user()->fetch(mctx, col_ref, *col_data, *rowid_prefix, - *trx_ref, *delete_marked, error_msg, - error_msg_len); + return storage->user()->m_stores[0].fetch( + mctx, col_ref, *col_data, *rowid_prefix, *trx_ref, *delete_marked, + error_msg, error_msg_len); } bool ColumnStorage::mark_delete(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Ref col_ref, bool delete_mark, char *error_msg, uint32_t error_msg_len) { - return storage->user()->mark_delete(mctx, trx_ref, col_ref, delete_mark, - error_msg, error_msg_len); + return storage->user()->m_stores[0].mark_delete( + mctx, trx_ref, col_ref, delete_mark, error_msg, error_msg_len); } bool ColumnStorage::purge(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Ref col_ref, char *error_msg, uint32_t error_msg_len) { - return storage->user()->purge(mctx, trx_ref, col_ref, error_msg, - error_msg_len); + return storage->user()->m_stores[0].purge(mctx, trx_ref, col_ref, error_msg, + error_msg_len); } } // namespace svector diff --git a/src/storage/storage.h b/src/storage/storage.h index ef6c8d2..6df2ba6 100644 --- a/src/storage/storage.h +++ b/src/storage/storage.h @@ -25,7 +25,11 @@ #define VILLAGESQL_EXAMPLES_VSQL_SVECTOR_SRC_STORAGE_H #include +#include #include +#include +#include +#include #include @@ -40,20 +44,73 @@ using vsql::preview_storage::Page; using vsql::preview_storage::Segment; using vsql::preview_storage::Space; -struct ColumnStorageContext { - static constexpr uint8_t FORMAT_VERSION = 1; +// Describes one column store to be created within a segment. +struct Storage_spec { + uint16_t col_len; + std::string metadata; +}; +struct ColumnStore { // Atomic counter to measure insert concurrency for this storage context. // This counter is incremented when entering insert() and decremented when // leaving, allowing us to track the peak number of concurrent inserts. std::atomic m_insert_concurrency_counter{0}; + RootPage m_root; + DataPage m_data; + + // Space and root page for this store. + Space::Ref m_space_ref{0}; + Page::Ref m_root_page_ref{Page::INVALID_REF}; + + std::string m_metadata; + + ColumnStore() = default; + + // std::atomic is not movable so std::vector requires a move + // constructor to compile. Callers must reserve() before emplace_back() so + // reallocation never happens and this constructor is never actually called. + ColumnStore(ColumnStore &&) noexcept { assert(false); } + + ColumnStore(const ColumnStore &) = delete; + ColumnStore &operator=(const ColumnStore &) = delete; + ColumnStore &operator=(ColumnStore &&) = delete; + + void init(Space::Ref space_ref, Page::Ref root_page_ref, uint16_t col_len, + uint8_t num_segments, uint8_t num_root_pages, + std::string_view metadata); + + bool initialized() const { return (m_root_page_ref != Page::INVALID_REF); } + + static void fill_error(const char *info, char *msg, uint32_t len, bool local); + + // Storage operations. All return false on success, true on error. + bool insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Data col_data, + Column::Ref &col_ref, char *error_msg, uint32_t error_msg_len); + + bool fetch(MtrCtx::Ref mctx, Column::Ref col_ref, Column::Data &col_data, + Column::Data &rowid_prefix, Segment::TrxRef &trx_ref, + bool &delete_marked, char *error_msg, uint32_t error_msg_len); + + bool mark_delete(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Ref col_ref, bool delete_mark, char *error_msg, + uint32_t error_msg_len); + + bool purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Ref col_ref, + char *error_msg, uint32_t error_msg_len); +}; + +struct MultiColumnStore { + static constexpr uint8_t FORMAT_VERSION = 1; + // Persistent storage reference: encodes (space_ref, root_page_ref). // Set on create/load; used by decode_ref throughout DML operations. Column::StorageRef m_ref{0}; - RootPage m_root; - DataPage m_data; + // One entry per column store. First entry is the primary store whose ref is + // encoded in m_ref; its root page also holds segments and root page refs + // for any additional stores. + std::vector m_stores; // Encode space_ref and root page ref into m_ref. void encode_ref(Space::Ref space_ref, Page::Ref page_ref) { @@ -68,26 +125,22 @@ struct ColumnStorageContext { } // Storage operations. All return false on success, true on error. - bool create(Space::Ref space_ref, Segment::TrxRef trx_ref, uint16_t col_len, + bool create(Space::Ref space_ref, Segment::TrxRef trx_ref, + const std::vector &storages, uint8_t num_segments, char *error_msg, uint32_t error_msg_len); bool drop(Segment::TrxRef trx_ref, char *error_msg, uint32_t error_msg_len); - bool insert(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Data col_data, - Column::Ref &col_ref, char *error_msg, uint32_t error_msg_len); + // Allocate a new root page for the store at root_idx from segment seg_idx, + // format it, and record its ref in the primary root page's other-refs array. + // Reused when creating higher-level stores on demand. + bool init_root_page(uint8_t seg_idx, uint8_t root_idx, char *error_msg, + uint32_t error_msg_len); - bool fetch(MtrCtx::Ref mctx, Column::Ref col_ref, Column::Data &col_data, - Column::Data &rowid_prefix, Segment::TrxRef &trx_ref, - bool &delete_marked, char *error_msg, uint32_t error_msg_len); + bool load(Column::StorageRef storage_ref, + const std::vector &specs, char *error_msg, + uint32_t error_msg_len); - bool mark_delete(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, - Column::Ref col_ref, bool delete_mark, char *error_msg, - uint32_t error_msg_len); - - bool purge(MtrCtx::Ref mctx, Segment::TrxRef trx_ref, Column::Ref col_ref, - char *error_msg, uint32_t error_msg_len); - - private: // Format an error message into msg/len. // Writes "SVECTOR: " when local is true, or // "SVECTOR: : " when local is false. @@ -98,35 +151,36 @@ struct ColumnStorageContext { // All methods are static and correspond to the VEF storage interface. class ColumnStorage { public: - using Ctx = Column::StorageCtx; + using Ctx = Column::StorageCtx; - static bool create(Ctx *storage, Space::Ref space, Segment::TrxRef trx_ref, - uint32_t col_len, char *error_msg, uint32_t error_msg_len); + static bool create(Ctx *storage, Space::Ref space, Segment::TrxRef trx_ref, + uint32_t col_len, char *error_msg, + uint32_t error_msg_len); - static bool drop(Ctx *storage, Segment::TrxRef trx_ref, char *error_msg, - uint32_t error_msg_len); + static bool drop(Ctx *storage, Segment::TrxRef trx_ref, char *error_msg, + uint32_t error_msg_len); - static bool load(Ctx *storage, Column::StorageRef storage_ref, - char *error_msg, uint32_t error_msg_len); + static bool load(Ctx *storage, Column::StorageRef storage_ref, + char *error_msg, uint32_t error_msg_len); - static bool insert(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, - Column::Data col_data, Column::Data rowid_prefix, - Column::Ref *col_ref, char *error_msg, - uint32_t error_msg_len); + static bool insert(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Data col_data, Column::Data rowid_prefix, + Column::Ref *col_ref, char *error_msg, + uint32_t error_msg_len); - static bool select(Ctx *storage, MtrCtx::Ref mctx, Column::Ref col_ref, - Column::Data *col_data, Column::Data *rowid_prefix, - Segment::TrxRef *trx_ref, bool *delete_marked, - char *error_msg, uint32_t error_msg_len); + static bool select(Ctx *storage, MtrCtx::Ref mctx, Column::Ref col_ref, + Column::Data *col_data, Column::Data *rowid_prefix, + Segment::TrxRef *trx_ref, bool *delete_marked, + char *error_msg, uint32_t error_msg_len); - static bool mark_delete(Ctx *storage, MtrCtx::Ref mctx, - Segment::TrxRef trx_ref, Column::Ref col_ref, - bool delete_mark, char *error_msg, - uint32_t error_msg_len); + static bool mark_delete(Ctx *storage, MtrCtx::Ref mctx, + Segment::TrxRef trx_ref, Column::Ref col_ref, + bool delete_mark, char *error_msg, + uint32_t error_msg_len); - static bool purge(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, - Column::Ref col_ref, char *error_msg, - uint32_t error_msg_len); + static bool purge(Ctx *storage, MtrCtx::Ref mctx, Segment::TrxRef trx_ref, + Column::Ref col_ref, char *error_msg, + uint32_t error_msg_len); }; } // namespace svector diff --git a/src/storage/tools/root_page_parser.cc b/src/storage/tools/root_page_parser.cc index 680248b..2d9b63a 100644 --- a/src/storage/tools/root_page_parser.cc +++ b/src/storage/tools/root_page_parser.cc @@ -26,6 +26,8 @@ #include #include +#include "page_reader.h" + namespace svector { namespace tool { @@ -62,36 +64,64 @@ std::string RootPageParser::read_string(const std::vector &data, bool RootPageParser::parse(const std::vector &page_data, RootPageInfo &info, std::string &error) { + constexpr uint32_t PAGE_HEADER_SIZE = PageReader::FIL_PAGE_DATA; + + if (page_data.size() <= PAGE_HEADER_SIZE) { + error = "Page too small to contain root page header"; + return false; + } + + RootPage rp; + uint8_t num_segments = read_uint8(page_data, PAGE_HEADER_SIZE); + // Three-stage bootstrap: each stage unlocks the next offset function. + // Stage 1: set N only (enables storage_metadata_len_off()). + rp.set_layout(num_segments, 0, 1); + // Validate page size - if (page_data.size() < RootPage::FREE_SLOT_ARRAY_OFF) { + if (page_data.size() <= rp.storage_metadata_len_off()) { error = "Page too small to contain root page header"; return false; } - // Parse fields using offsets from RootPage - info.version = read_uint8(page_data, RootPage::VERSION_OFF); - info.page_type = read_uint8(page_data, RootPage::PAGE_TYPE_OFF); - info.creator_name = read_string(page_data, RootPage::CREATOR_NAME_OFF, - RootPage::CREATOR_NAME_LEN); - info.column_size = read_uint16(page_data, RootPage::COLUMN_SIZE_OFF); - info.all_slot_head = read_uint32(page_data, RootPage::ALL_SLOT_HEAD_OFF); - info.all_slot_tail = read_uint32(page_data, RootPage::ALL_SLOT_TAIL_OFF); - info.total_data_pages = - read_uint32(page_data, RootPage::TOTAL_DATA_PAGES_OFF); - info.total_free_pages = - read_uint32(page_data, RootPage::TOTAL_FREE_PAGES_OFF); - info.free_slot_array_max_size = - read_uint16(page_data, RootPage::FREE_SLOT_ARRAY_MAX_SIZE_OFF); - info.free_slot_array_cur_size = - read_uint16(page_data, RootPage::FREE_SLOT_ARRAY_CUR_SIZE_OFF); + // Stage 2: read metadata length (enables num_root_pages_off()). + uint8_t metadata_len = read_uint8(page_data, rp.storage_metadata_len_off()); + rp.set_layout(num_segments, metadata_len, 1); + + if (page_data.size() <= rp.num_root_pages_off()) { + error = "Page too small to contain root page header"; + return false; + } + + // Stage 3: read K and finalize layout. + uint8_t num_root_pages = read_uint8(page_data, rp.num_root_pages_off()); + rp.set_layout(num_segments, metadata_len, num_root_pages); - // Validate creator name - if (info.creator_name != reinterpret_cast(RootPage::CREATOR)) { - error = "Invalid creator name: expected 'SVECTOR', got '" + - info.creator_name + "'"; + if (page_data.size() < rp.free_slot_array_off()) { + error = "Page too small to contain root page header"; return false; } + // Parse fields using offsets from RootPage + info.version = read_uint8(page_data, rp.version_off()); + info.page_type = read_uint8(page_data, rp.page_type_off()); + info.storage_metadata = + read_string(page_data, rp.storage_metadata_off(), metadata_len); + info.num_root_pages = num_root_pages; + info.other_root_page_refs.clear(); + for (uint8_t i = 0; i < num_root_pages - 1; ++i) { + uint32_t off = rp.other_root_pages_off() + i * RootPage::ROOT_PAGE_REF_LEN; + info.other_root_page_refs.push_back(read_uint32(page_data, off)); + } + info.column_size = read_uint16(page_data, rp.column_size_off()); + info.all_slot_head = read_uint32(page_data, rp.all_slot_head_off()); + info.all_slot_tail = read_uint32(page_data, rp.all_slot_tail_off()); + info.total_data_pages = read_uint32(page_data, rp.total_data_pages_off()); + info.total_free_pages = read_uint32(page_data, rp.total_free_pages_off()); + info.free_slot_array_max_size = + read_uint16(page_data, rp.free_slot_array_max_size_off()); + info.free_slot_array_cur_size = + read_uint16(page_data, rp.free_slot_array_cur_size_off()); + // Validate page type if (info.page_type != static_cast(ColumnPageType::ROOT_PAGE)) { error = "Invalid page type: expected ROOT_PAGE (1), got " + @@ -104,8 +134,7 @@ bool RootPageParser::parse(const std::vector &page_data, info.free_slots.reserve(info.free_slot_array_cur_size); for (uint16_t i = 0; i < info.free_slot_array_cur_size; ++i) { - uint32_t offset = - RootPage::FREE_SLOT_ARRAY_OFF + (i * RootPage::FREE_SLOT_LEN); + uint32_t offset = rp.free_slot_array_off() + (i * RootPage::FREE_SLOT_LEN); if (offset + RootPage::FREE_SLOT_LEN > page_data.size()) { error = "Free slot array extends beyond page boundary"; return false; @@ -118,13 +147,26 @@ bool RootPageParser::parse(const std::vector &page_data, } void RootPageParser::display(const RootPageInfo &info, bool verbose) { - std::cout << "SVECTOR Root Page\n"; + std::cout << info.storage_metadata << " Root Page\n"; std::cout << "=================\n\n"; std::cout << "Version: " << static_cast(info.version) << "\n"; std::cout << "Page Type: " << static_cast(info.page_type) << " (ROOT_PAGE)\n"; - std::cout << "Creator: " << info.creator_name << "\n"; + std::cout << "Root Pages: " << static_cast(info.num_root_pages) + << "\n"; + if (!info.other_root_page_refs.empty()) { + std::cout << "Other Root Pages:"; + for (uint32_t ref : info.other_root_page_refs) { + if (ref == RootPage::NULL_FREE_PAGE_REF) { + std::cout << " (NULL)"; + } else { + std::cout << " Page #" << ref; + } + } + std::cout << "\n"; + } + std::cout << "Column Size: " << info.column_size << " bytes"; // Calculate vector dimensions (assuming float32) diff --git a/src/storage/tools/root_page_parser.h b/src/storage/tools/root_page_parser.h index 9739cf4..b4d47ef 100644 --- a/src/storage/tools/root_page_parser.h +++ b/src/storage/tools/root_page_parser.h @@ -40,7 +40,9 @@ class RootPageParser { struct RootPageInfo { uint8_t version; uint8_t page_type; - std::string creator_name; + std::string storage_metadata; + uint8_t num_root_pages; + std::vector other_root_page_refs; uint16_t column_size; uint32_t all_slot_head; uint32_t all_slot_tail; diff --git a/src/vector.cc b/src/vector.cc index dccfe5c..efedfd6 100644 --- a/src/vector.cc +++ b/src/vector.cc @@ -606,7 +606,7 @@ constexpr auto SVECTOR = vsql::make_type() .build(); static constexpr auto kSVectorStorageIntf = - make_column_store(SVECTOR) + make_column_store(SVECTOR) .create<&svector::ColumnStorage::create>() .drop<&svector::ColumnStorage::drop>() .load<&svector::ColumnStorage::load>()