From 15928dd14f7200467b932c0a2c3c31d9c2145821 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Sun, 21 Jun 2026 15:11:20 -0700 Subject: [PATCH 1/9] cloud_storage: split offset_index into its own library remote_segment_index.{h,cc} contained two independent things sharing a TU: the read/write `offset_index` (used by readers) and `remote_segment_index_builder`, a storage::batch_consumer that pulls in raft (raft::offset_translator_batch_types). offset_index itself has none of that coupling -- only model, bytes, utils/delta_for and serde. Move `offset_index` into cloud_storage/offset_index.{h,cc} behind a new //src/v/cloud_storage:offset_index cc_library. remote_segment_index.{h,cc} keeps segment_record_stats + the builder, now including offset_index.h; the cloud_storage monolith depends on :offset_index. Co-Authored-By: Claude Opus 4.8 --- src/v/cloud_storage/BUILD | 27 ++ src/v/cloud_storage/offset_index.cc | 435 ++++++++++++++++++++ src/v/cloud_storage/offset_index.h | 178 ++++++++ src/v/cloud_storage/remote_segment_index.cc | 415 ------------------- src/v/cloud_storage/remote_segment_index.h | 159 +------ 5 files changed, 641 insertions(+), 573 deletions(-) create mode 100644 src/v/cloud_storage/offset_index.cc create mode 100644 src/v/cloud_storage/offset_index.h diff --git a/src/v/cloud_storage/BUILD b/src/v/cloud_storage/BUILD index e5b1a1aebc49f..91e97d8e8afb6 100644 --- a/src/v/cloud_storage/BUILD +++ b/src/v/cloud_storage/BUILD @@ -86,6 +86,32 @@ redpanda_cc_library( ], ) +redpanda_cc_library( + name = "offset_index", + srcs = [ + "offset_index.cc", + ], + hdrs = [ + "offset_index.h", + ], + implementation_deps = [ + ":logger", + "//src/v/base", + "//src/v/bytes:iobuf_parser", + "//src/v/serde", + "//src/v/serde:iobuf", + "//src/v/serde:vector", + "@seastar", + ], + visibility = ["//visibility:public"], + deps = [ + "//src/v/bytes:iobuf", + "//src/v/model", + "//src/v/utils:delta_for", + "@abseil-cpp//absl/container:btree", + ], +) + redpanda_cc_library( name = "cloud_storage", srcs = [ @@ -183,6 +209,7 @@ redpanda_cc_library( visibility = ["//visibility:public"], deps = [ ":logger", + ":offset_index", ":remote_label", ":segment_meta_cstore", ":topic_mount_manifest_path", diff --git a/src/v/cloud_storage/offset_index.cc b/src/v/cloud_storage/offset_index.cc new file mode 100644 index 0000000000000..f72a8ea0dab56 --- /dev/null +++ b/src/v/cloud_storage/offset_index.cc @@ -0,0 +1,435 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/vectorizedio/redpanda/blob/master/licenses/rcl.md + */ + +#include "cloud_storage/offset_index.h" + +#include "base/vassert.h" +#include "bytes/iobuf_parser.h" +#include "cloud_storage/logger.h" +#include "serde/rw/envelope.h" +#include "serde/rw/iobuf.h" +#include "serde/rw/vector.h" + +#include + +namespace cloud_storage { + +offset_index::offset_index( + model::offset initial_rp, + kafka::offset initial_kaf, + int64_t initial_file_pos, + int64_t file_pos_step, + model::timestamp initial_time) + : _rp_offsets{} + , _kaf_offsets{} + , _file_offsets{} + , _time_offsets{} + , _pos{} + , _initial_rp(initial_rp) + , _initial_kaf(initial_kaf) + , _initial_file_pos(initial_file_pos) + , _initial_time(initial_time) + , _rp_index(initial_rp) + , _kaf_index(initial_kaf) + , _file_index(initial_file_pos, delta_delta_t(file_pos_step)) + , _time_index(initial_time.value()) + , _min_file_pos_step(file_pos_step) {} + +size_t offset_index::estimate_memory_use() const { + return _file_index.mem_use() + _rp_index.mem_use() + _kaf_index.mem_use() + + _time_index.mem_use(); +} + +void offset_index::add( + model::offset rp_offset, + kafka::offset kaf_offset, + int64_t file_offset, + model::timestamp max_timestamp) { + auto ix = index_mask & _pos++; + _rp_offsets.at(ix) = rp_offset(); + _kaf_offsets.at(ix) = kaf_offset(); + _file_offsets.at(ix) = file_offset; + _time_offsets.at(ix) = max_timestamp.value(); + try { + if ((_pos & index_mask) == 0) { + _rp_index.add(_rp_offsets); + _kaf_index.add(_kaf_offsets); + _file_index.add(_file_offsets); + _time_index.add(_time_offsets); + } + } catch (...) { + // Get rid of the corrupted state in the encoders. + // If the exception is thrown out of 'add' method + // the invariant of the index is broken. + _pos = 0; + _rp_offsets = {}; + _kaf_offsets = {}; + _file_offsets = {}; + _rp_index = encoder_t(_initial_rp); + _kaf_index = encoder_t(_initial_kaf); + _file_index = foffset_encoder_t( + _initial_file_pos, delta_delta_t(_min_file_pos_step)); + _time_index = encoder_t(_initial_time.value()); + throw; + } +} + +std:: + variant + offset_index::maybe_find_offset( + int64_t upper_bound, + deltafor_encoder& encoder, + const std::array& write_buffer) { + deltafor_decoder decoder( + encoder.get_initial_value(), encoder.get_row_count(), encoder.share()); + auto max_index = encoder.get_row_count() * details::FOR_buffer_depth - 1; + auto maybe_ix = _find_under(std::move(decoder), upper_bound); + if (!maybe_ix || maybe_ix->ix == max_index) { + auto ixend = _pos & index_mask; + std::optional candidate; + for (size_t i = 0; i < ixend; i++) { + if (write_buffer.at(i) < upper_bound) { + candidate = find_result{ + .rp_offset = model::offset(_rp_offsets.at(i)), + .kaf_offset = kafka::offset(_kaf_offsets.at(i)), + .file_pos = _file_offsets.at(i), + }; + } else { + break; + } + } + // maybe_ix can point to the last element of the compressed + // chunk if all elements inside it are less than the offset that + // we're looking for. In this case we should use it even if + // we can't find anything inside the buffer. + // maybe_ix will be null if the compressed chunk is empty. + if (candidate) { + return *candidate; + } + if (!maybe_ix) { + return std::monostate(); + } + } + // Invariant: maybe_ix here can't be nullopt + return *maybe_ix; +} + +std::optional +offset_index::find_rp_offset(model::offset upper_bound) { + auto search_result = maybe_find_offset(upper_bound, _rp_index, _rp_offsets); + + return ss::visit( + search_result, + [](std::monostate) -> std::optional { return std::nullopt; }, + [](find_result result) -> std::optional { return result; }, + [this](index_value index_result) -> std::optional { + find_result res{}; + + size_t ix = index_result.ix; + res.rp_offset = model::offset(index_result.value); + + decoder_t kaf_dec( + _kaf_index.get_initial_value(), + _kaf_index.get_row_count(), + _kaf_index.copy()); + auto kaf_offset = _fetch_ix(std::move(kaf_dec), ix); + vassert(kaf_offset.has_value(), "Inconsistent index state"); + res.kaf_offset = kafka::offset(*kaf_offset); + foffset_decoder_t file_dec( + _file_index.get_initial_value(), + _file_index.get_row_count(), + _file_index.copy(), + delta_delta_t(_min_file_pos_step)); + auto file_pos = _fetch_ix(std::move(file_dec), ix); + res.file_pos = *file_pos; + return res; + }); +} + +std::optional +offset_index::find_kaf_offset(kafka::offset upper_bound) { + auto search_result = maybe_find_offset( + upper_bound, _kaf_index, _kaf_offsets); + return ss::visit( + search_result, + [](std::monostate) -> std::optional { return std::nullopt; }, + [](find_result result) -> std::optional { return result; }, + [this](index_value index_result) -> std::optional { + find_result res{}; + + size_t ix = index_result.ix; + res.kaf_offset = kafka::offset(index_result.value); + + decoder_t rp_dec( + _rp_index.get_initial_value(), + _rp_index.get_row_count(), + _rp_index.copy()); + auto rp_offset = _fetch_ix(std::move(rp_dec), ix); + vassert(rp_offset.has_value(), "Inconsistent index state"); + res.rp_offset = model::offset(*rp_offset); + foffset_decoder_t file_dec( + _file_index.get_initial_value(), + _file_index.get_row_count(), + _file_index.copy(), + delta_delta_t(_min_file_pos_step)); + auto file_pos = _fetch_ix(std::move(file_dec), ix); + res.file_pos = *file_pos; + return res; + }); +} + +std::optional +offset_index::find_timestamp(model::timestamp upper_bound) { + if (_initial_time == model::timestamp::missing()) { + // Bail out early if this is a version 1 index + // that does not index timestamps. + return std::nullopt; + } + + auto search_result = maybe_find_offset( + upper_bound.value(), _time_index, _time_offsets); + + return ss::visit( + search_result, + [](std::monostate) -> std::optional { return std::nullopt; }, + [](find_result result) -> std::optional { return result; }, + [this](index_value index_result) -> std::optional { + size_t ix = index_result.ix; + + // Decode all offset indices to build up the result. + decoder_t rp_dec( + _rp_index.get_initial_value(), + _rp_index.get_row_count(), + _rp_index.copy()); + auto rp_offset = _fetch_ix(std::move(rp_dec), ix); + vassert(rp_offset.has_value(), "Inconsistent index state"); + + decoder_t kaf_dec( + _kaf_index.get_initial_value(), + _kaf_index.get_row_count(), + _kaf_index.copy()); + auto kaf_offset = _fetch_ix(std::move(kaf_dec), ix); + vassert(kaf_offset.has_value(), "Inconsistent index state"); + + foffset_decoder_t file_dec( + _file_index.get_initial_value(), + _file_index.get_row_count(), + _file_index.copy(), + delta_delta_t(_min_file_pos_step)); + auto file_pos = _fetch_ix(std::move(file_dec), ix); + vassert(file_pos.has_value(), "Inconsistent index state"); + + return offset_index::find_result{ + .rp_offset = model::offset(*rp_offset), + .kaf_offset = kafka::offset(*kaf_offset), + .file_pos = *file_pos}; + }); +} + +offset_index::coarse_index_t offset_index::build_coarse_index( + uint64_t step_size, std::string_view index_path) const { + vlog( + cst_log.trace, + "{}: building coarse index from file offset index with {} rows", + index_path, + _file_index.get_row_count()); + vassert( + step_size > static_cast(_min_file_pos_step), + "{}: step size {} cannot be less than or equal to index step size {}", + index_path, + step_size, + _min_file_pos_step); + + foffset_decoder_t file_dec( + _file_index.get_initial_value(), + _file_index.get_row_count(), + _file_index.copy(), + delta_delta_t(_min_file_pos_step)); + std::array file_row{}; + + decoder_t kaf_dec( + _kaf_index.get_initial_value(), + _kaf_index.get_row_count(), + _kaf_index.copy()); + std::array kafka_row{}; + + coarse_index_t index; + auto populate_index = [step_size, &index, index_path]( + const auto& file_offsets, + const auto& kafka_offsets, + auto& span_start, + auto& span_end) { + for (auto it = file_offsets.cbegin(), kit = kafka_offsets.cbegin(); + it != file_offsets.cend() && kit != kafka_offsets.cend(); + ++it, ++kit) { + span_end = *it; + auto delta = span_end - span_start + 1; + if (span_end > span_start && delta >= step_size) { + vlog( + cst_log.trace, + "{}: adding entry to coarse index, current file pos: {}, " + "step size: {}, span size: {}", + index_path, + span_end, + step_size, + delta); + index[kafka::offset{*kit}] = span_end; + span_start = span_end + 1; + } + } + }; + + size_t start{0}; + size_t end{0}; + while (file_dec.read(file_row) && kaf_dec.read(kafka_row)) { + populate_index(file_row, kafka_row, start, end); + file_row = {}; + kafka_row = {}; + } + + populate_index(_file_offsets, _kaf_offsets, start, end); + return index; +} + +struct offset_index_header + : serde::envelope< + offset_index_header, + serde::version<2>, + serde::compat_version<1>> { + int64_t min_file_pos_step; + uint64_t num_elements; + int64_t base_rp; + int64_t last_rp; + int64_t base_kaf; + int64_t last_kaf; + int64_t base_file; + int64_t last_file; + std::vector rp_write_buf; + std::vector kaf_write_buf; + std::vector file_write_buf; + iobuf rp_index; + iobuf kaf_index; + iobuf file_index; + + // Version 2 fields + int64_t base_time{model::timestamp::missing().value()}; + int64_t last_time{model::timestamp::missing().value()}; + std::vector time_write_buf; + iobuf time_index; + + auto serde_fields() { + return std::tie( + min_file_pos_step, + num_elements, + base_rp, + last_rp, + base_kaf, + last_kaf, + base_file, + last_file, + rp_write_buf, + kaf_write_buf, + file_write_buf, + rp_index, + kaf_index, + file_index, + base_time, + last_time, + time_write_buf, + time_index); + } +}; + +iobuf offset_index::to_iobuf() const { + offset_index_header hdr{ + .min_file_pos_step = _min_file_pos_step, + .num_elements = _pos, + .base_rp = _initial_rp, + .last_rp = _rp_index.get_last_value(), + .base_kaf = _initial_kaf, + .last_kaf = _kaf_index.get_last_value(), + .base_file = _initial_file_pos, + .last_file = _file_index.get_last_value(), + .rp_write_buf = std::vector( + _rp_offsets.begin(), _rp_offsets.end()), + .kaf_write_buf = std::vector( + _kaf_offsets.begin(), _kaf_offsets.end()), + .file_write_buf = std::vector( + _file_offsets.begin(), _file_offsets.end()), + .rp_index = _rp_index.copy(), + .kaf_index = _kaf_index.copy(), + .file_index = _file_index.copy(), + .base_time = _initial_time.value(), + .last_time = _time_index.get_last_value(), + .time_write_buf = std::vector( + _time_offsets.begin(), _time_offsets.end()), + .time_index = _time_index.copy()}; + return serde::to_iobuf(std::move(hdr)); +} + +void offset_index::from_iobuf(iobuf b) { + iobuf_parser parser(std::move(b)); + auto hdr = serde::read(parser); + auto num_rows = hdr.num_elements / buffer_depth; + _pos = hdr.num_elements; + _initial_rp = model::offset(hdr.base_rp); + _initial_kaf = kafka::offset(hdr.base_kaf); + _initial_file_pos = hdr.base_file; + std::copy( + hdr.rp_write_buf.begin(), hdr.rp_write_buf.end(), _rp_offsets.begin()); + std::copy( + hdr.kaf_write_buf.begin(), hdr.kaf_write_buf.end(), _kaf_offsets.begin()); + std::copy( + hdr.file_write_buf.begin(), + hdr.file_write_buf.end(), + _file_offsets.begin()); + _rp_index = encoder_t( + _initial_rp, num_rows, hdr.last_rp, std::move(hdr.rp_index)); + _kaf_index = encoder_t( + _initial_kaf, num_rows, hdr.last_kaf, std::move(hdr.kaf_index)); + _file_index = foffset_encoder_t( + _initial_file_pos, + num_rows, + hdr.last_file, + std::move(hdr.file_index), + delta_delta_t(_min_file_pos_step)); + _min_file_pos_step = hdr.min_file_pos_step; + + _initial_time = model::timestamp(hdr.base_time); + _time_index = encoder_t( + _initial_time.value(), + num_rows, + hdr.last_time, + std::move(hdr.time_index)); + std::copy( + hdr.time_write_buf.begin(), + hdr.time_write_buf.end(), + _time_offsets.begin()); +} + +std::optional +offset_index::_find_under(deltafor_decoder decoder, int64_t offset) { + size_t ix = 0; + std::array rp_buf{}; + std::optional candidate; + while (decoder.read(rp_buf)) { + for (auto o : rp_buf) { + if (o >= offset) { + return candidate; + } + candidate = {.ix = size_t(ix), .value = o}; + ix++; + } + rp_buf = {}; + } + return candidate; +} + +} // namespace cloud_storage diff --git a/src/v/cloud_storage/offset_index.h b/src/v/cloud_storage/offset_index.h new file mode 100644 index 0000000000000..e9e5d70201dff --- /dev/null +++ b/src/v/cloud_storage/offset_index.h @@ -0,0 +1,178 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ + +#pragma once + +#include "absl/container/btree_map.h" +#include "bytes/iobuf.h" +#include "model/fundamental.h" +#include "model/timestamp.h" +#include "utils/delta_for.h" + +#include +#include +#include +#include + +namespace cloud_storage { + +/// Offset index for remote_segment +/// +/// The object indexes tuples that contain three elements: +/// - redpanda offset +/// - kafka offset +/// - file offset +/// +/// The search is linear. The underlying data structure is a +/// fragmented buffer (iobuf). It is possible to search by redpanda +/// and kafka offsets, but not by file offset. +/// +/// The invariant of the offset_index is that all three encoders +/// have the same number of elements. All three buffers should also +/// have the same number of elements. +class offset_index { + static constexpr uint32_t buffer_depth = details::FOR_buffer_depth; + static constexpr uint32_t index_mask = buffer_depth - 1; + static_assert( + (buffer_depth & index_mask) == 0, + "buffer_depth have to be a power of two"); + +public: + offset_index( + model::offset initial_rp, + kafka::offset initial_kaf, + int64_t initial_file_pos, + int64_t file_pos_step, + model::timestamp initial_time); + + /// Add new tuple to the index. + void add( + model::offset rp_offset, + kafka::offset kaf_offset, + int64_t file_offset, + model::timestamp); + + struct find_result { + model::offset rp_offset; + kafka::offset kaf_offset; + int64_t file_pos; + }; + + /// Estimate memory usage by the index + size_t estimate_memory_use() const; + + /// Find index entry which is strictly lower than the redpanda offset + /// + /// The returned value has rp_offset less than upper_bound. + /// If all elements are larger than 'upper_bound' nullopt is returned. + /// If all elements are smaller than 'upper_bound' the last value is + /// returned. + std::optional find_rp_offset(model::offset upper_bound); + + /// Find index entry which is strictly lower than the kafka offset + /// + /// The returned value has kaf_offset less than upper_bound. + /// If all elements are larger than 'upper_bound' nullopt is returned. + /// If all elements are smaller than 'upper_bound' the last value is + /// returned. + std::optional find_kaf_offset(kafka::offset upper_bound); + + /// Find index entry which is strictly lower than the timestamp + /// + /// The returned value has timestamp less than upper_bound. + /// If all elements are larger than 'upper_bound' nullopt is returned. + /// If all elements are smaller than 'upper_bound' the last value is + /// returned. + std::optional find_timestamp(model::timestamp upper_bound); + + /// Builds a coarse index mapping kafka offsets to file positions. The step + /// size is the resolution of the index. So given a step size of 16MiB, the + /// result contains mappings of kafka offset to file position from the index + /// where entries are _roughly_ 16MiB apart in terms of file position. + using coarse_index_t = absl::btree_map; + coarse_index_t + build_coarse_index(uint64_t step_size, std::string_view index_path) const; + + /// Serialize offset_index + iobuf to_iobuf() const; + + /// Deserialize offset_index + void from_iobuf(iobuf in); + +private: + struct index_value { + size_t ix; + int64_t value; + }; + + /// Find index entry which is strictly lower than the provided value + /// + /// The encoder and the write buffer have to be provided via parameters. + /// The returned value is a variant which contains a monostate if no + /// value can be found; index_value if the value is found in the + /// encoder; find_result if the value is found in the write buffer (in + /// this case no further search is needed). + std::variant maybe_find_offset( + int64_t upper_bound, + deltafor_encoder& encoder, + const std::array& write_buffer); + + /// Find element inside the offset range stored in the decoder which is + /// less than offset. Return last element if no such element can be + /// found. Return nullopt if all emlements are larger or equal than + /// offset. + static std::optional + _find_under(deltafor_decoder decoder, int64_t offset); + + /// Return element by index. + template + static std::optional + _fetch_ix(DecoderT decoder, size_t target_ix) { + size_t ix = 0; + std::array buffer{}; + while (decoder.read(buffer)) { + for (auto o : buffer) { + if (ix == target_ix) { + return o; + } + ix++; + } + buffer = {}; + } + return std::nullopt; + } + +private: + std::array _rp_offsets; + std::array _kaf_offsets; + std::array _file_offsets; + std::array _time_offsets; + uint64_t _pos; + model::offset _initial_rp; + kafka::offset _initial_kaf; + int64_t _initial_file_pos; + model::timestamp _initial_time; + + using encoder_t = deltafor_encoder; + using decoder_t = deltafor_decoder; + using delta_delta_t = details::delta_delta; + using foffset_encoder_t = deltafor_encoder; + using foffset_decoder_t = deltafor_decoder; + + encoder_t _rp_index; + encoder_t _kaf_index; + foffset_encoder_t _file_index; + encoder_t _time_index; + int64_t _min_file_pos_step; + + friend class offset_index_accessor; +}; + +} // namespace cloud_storage diff --git a/src/v/cloud_storage/remote_segment_index.cc b/src/v/cloud_storage/remote_segment_index.cc index b715c55ba8660..1348add173247 100644 --- a/src/v/cloud_storage/remote_segment_index.cc +++ b/src/v/cloud_storage/remote_segment_index.cc @@ -10,425 +10,10 @@ #include "cloud_storage/remote_segment_index.h" -#include "cloud_storage/logger.h" #include "raft/consensus.h" -#include "serde/rw/envelope.h" -#include "serde/rw/iobuf.h" -#include "serde/rw/vector.h" namespace cloud_storage { -offset_index::offset_index( - model::offset initial_rp, - kafka::offset initial_kaf, - int64_t initial_file_pos, - int64_t file_pos_step, - model::timestamp initial_time) - : _rp_offsets{} - , _kaf_offsets{} - , _file_offsets{} - , _time_offsets{} - , _pos{} - , _initial_rp(initial_rp) - , _initial_kaf(initial_kaf) - , _initial_file_pos(initial_file_pos) - , _initial_time(initial_time) - , _rp_index(initial_rp) - , _kaf_index(initial_kaf) - , _file_index(initial_file_pos, delta_delta_t(file_pos_step)) - , _time_index(initial_time.value()) - , _min_file_pos_step(file_pos_step) {} - -size_t offset_index::estimate_memory_use() const { - return _file_index.mem_use() + _rp_index.mem_use() + _kaf_index.mem_use() - + _time_index.mem_use(); -} - -void offset_index::add( - model::offset rp_offset, - kafka::offset kaf_offset, - int64_t file_offset, - model::timestamp max_timestamp) { - auto ix = index_mask & _pos++; - _rp_offsets.at(ix) = rp_offset(); - _kaf_offsets.at(ix) = kaf_offset(); - _file_offsets.at(ix) = file_offset; - _time_offsets.at(ix) = max_timestamp.value(); - try { - if ((_pos & index_mask) == 0) { - _rp_index.add(_rp_offsets); - _kaf_index.add(_kaf_offsets); - _file_index.add(_file_offsets); - _time_index.add(_time_offsets); - } - } catch (...) { - // Get rid of the corrupted state in the encoders. - // If the exception is thrown out of 'add' method - // the invariant of the index is broken. - _pos = 0; - _rp_offsets = {}; - _kaf_offsets = {}; - _file_offsets = {}; - _rp_index = encoder_t(_initial_rp); - _kaf_index = encoder_t(_initial_kaf); - _file_index = foffset_encoder_t( - _initial_file_pos, delta_delta_t(_min_file_pos_step)); - _time_index = encoder_t(_initial_time.value()); - throw; - } -} - -std:: - variant - offset_index::maybe_find_offset( - int64_t upper_bound, - deltafor_encoder& encoder, - const std::array& write_buffer) { - deltafor_decoder decoder( - encoder.get_initial_value(), encoder.get_row_count(), encoder.share()); - auto max_index = encoder.get_row_count() * details::FOR_buffer_depth - 1; - auto maybe_ix = _find_under(std::move(decoder), upper_bound); - if (!maybe_ix || maybe_ix->ix == max_index) { - auto ixend = _pos & index_mask; - std::optional candidate; - for (size_t i = 0; i < ixend; i++) { - if (write_buffer.at(i) < upper_bound) { - candidate = find_result{ - .rp_offset = model::offset(_rp_offsets.at(i)), - .kaf_offset = kafka::offset(_kaf_offsets.at(i)), - .file_pos = _file_offsets.at(i), - }; - } else { - break; - } - } - // maybe_ix can point to the last element of the compressed - // chunk if all elements inside it are less than the offset that - // we're looking for. In this case we should use it even if - // we can't find anything inside the buffer. - // maybe_ix will be null if the compressed chunk is empty. - if (candidate) { - return *candidate; - } - if (!maybe_ix) { - return std::monostate(); - } - } - // Invariant: maybe_ix here can't be nullopt - return *maybe_ix; -} - -std::optional -offset_index::find_rp_offset(model::offset upper_bound) { - auto search_result = maybe_find_offset(upper_bound, _rp_index, _rp_offsets); - - return ss::visit( - search_result, - [](std::monostate) -> std::optional { return std::nullopt; }, - [](find_result result) -> std::optional { return result; }, - [this](index_value index_result) -> std::optional { - find_result res{}; - - size_t ix = index_result.ix; - res.rp_offset = model::offset(index_result.value); - - decoder_t kaf_dec( - _kaf_index.get_initial_value(), - _kaf_index.get_row_count(), - _kaf_index.copy()); - auto kaf_offset = _fetch_ix(std::move(kaf_dec), ix); - vassert(kaf_offset.has_value(), "Inconsistent index state"); - res.kaf_offset = kafka::offset(*kaf_offset); - foffset_decoder_t file_dec( - _file_index.get_initial_value(), - _file_index.get_row_count(), - _file_index.copy(), - delta_delta_t(_min_file_pos_step)); - auto file_pos = _fetch_ix(std::move(file_dec), ix); - res.file_pos = *file_pos; - return res; - }); -} - -std::optional -offset_index::find_kaf_offset(kafka::offset upper_bound) { - auto search_result = maybe_find_offset( - upper_bound, _kaf_index, _kaf_offsets); - return ss::visit( - search_result, - [](std::monostate) -> std::optional { return std::nullopt; }, - [](find_result result) -> std::optional { return result; }, - [this](index_value index_result) -> std::optional { - find_result res{}; - - size_t ix = index_result.ix; - res.kaf_offset = kafka::offset(index_result.value); - - decoder_t rp_dec( - _rp_index.get_initial_value(), - _rp_index.get_row_count(), - _rp_index.copy()); - auto rp_offset = _fetch_ix(std::move(rp_dec), ix); - vassert(rp_offset.has_value(), "Inconsistent index state"); - res.rp_offset = model::offset(*rp_offset); - foffset_decoder_t file_dec( - _file_index.get_initial_value(), - _file_index.get_row_count(), - _file_index.copy(), - delta_delta_t(_min_file_pos_step)); - auto file_pos = _fetch_ix(std::move(file_dec), ix); - res.file_pos = *file_pos; - return res; - }); -} - -std::optional -offset_index::find_timestamp(model::timestamp upper_bound) { - if (_initial_time == model::timestamp::missing()) { - // Bail out early if this is a version 1 index - // that does not index timestamps. - return std::nullopt; - } - - auto search_result = maybe_find_offset( - upper_bound.value(), _time_index, _time_offsets); - - return ss::visit( - search_result, - [](std::monostate) -> std::optional { return std::nullopt; }, - [](find_result result) -> std::optional { return result; }, - [this](index_value index_result) -> std::optional { - size_t ix = index_result.ix; - - // Decode all offset indices to build up the result. - decoder_t rp_dec( - _rp_index.get_initial_value(), - _rp_index.get_row_count(), - _rp_index.copy()); - auto rp_offset = _fetch_ix(std::move(rp_dec), ix); - vassert(rp_offset.has_value(), "Inconsistent index state"); - - decoder_t kaf_dec( - _kaf_index.get_initial_value(), - _kaf_index.get_row_count(), - _kaf_index.copy()); - auto kaf_offset = _fetch_ix(std::move(kaf_dec), ix); - vassert(kaf_offset.has_value(), "Inconsistent index state"); - - foffset_decoder_t file_dec( - _file_index.get_initial_value(), - _file_index.get_row_count(), - _file_index.copy(), - delta_delta_t(_min_file_pos_step)); - auto file_pos = _fetch_ix(std::move(file_dec), ix); - vassert(file_pos.has_value(), "Inconsistent index state"); - - return offset_index::find_result{ - .rp_offset = model::offset(*rp_offset), - .kaf_offset = kafka::offset(*kaf_offset), - .file_pos = *file_pos}; - }); -} - -offset_index::coarse_index_t offset_index::build_coarse_index( - uint64_t step_size, std::string_view index_path) const { - vlog( - cst_log.trace, - "{}: building coarse index from file offset index with {} rows", - index_path, - _file_index.get_row_count()); - vassert( - step_size > static_cast(_min_file_pos_step), - "{}: step size {} cannot be less than or equal to index step size {}", - index_path, - step_size, - _min_file_pos_step); - - foffset_decoder_t file_dec( - _file_index.get_initial_value(), - _file_index.get_row_count(), - _file_index.copy(), - delta_delta_t(_min_file_pos_step)); - std::array file_row{}; - - decoder_t kaf_dec( - _kaf_index.get_initial_value(), - _kaf_index.get_row_count(), - _kaf_index.copy()); - std::array kafka_row{}; - - coarse_index_t index; - auto populate_index = [step_size, &index, index_path]( - const auto& file_offsets, - const auto& kafka_offsets, - auto& span_start, - auto& span_end) { - for (auto it = file_offsets.cbegin(), kit = kafka_offsets.cbegin(); - it != file_offsets.cend() && kit != kafka_offsets.cend(); - ++it, ++kit) { - span_end = *it; - auto delta = span_end - span_start + 1; - if (span_end > span_start && delta >= step_size) { - vlog( - cst_log.trace, - "{}: adding entry to coarse index, current file pos: {}, " - "step size: {}, span size: {}", - index_path, - span_end, - step_size, - delta); - index[kafka::offset{*kit}] = span_end; - span_start = span_end + 1; - } - } - }; - - size_t start{0}; - size_t end{0}; - while (file_dec.read(file_row) && kaf_dec.read(kafka_row)) { - populate_index(file_row, kafka_row, start, end); - file_row = {}; - kafka_row = {}; - } - - populate_index(_file_offsets, _kaf_offsets, start, end); - return index; -} - -struct offset_index_header - : serde::envelope< - offset_index_header, - serde::version<2>, - serde::compat_version<1>> { - int64_t min_file_pos_step; - uint64_t num_elements; - int64_t base_rp; - int64_t last_rp; - int64_t base_kaf; - int64_t last_kaf; - int64_t base_file; - int64_t last_file; - std::vector rp_write_buf; - std::vector kaf_write_buf; - std::vector file_write_buf; - iobuf rp_index; - iobuf kaf_index; - iobuf file_index; - - // Version 2 fields - int64_t base_time{model::timestamp::missing().value()}; - int64_t last_time{model::timestamp::missing().value()}; - std::vector time_write_buf; - iobuf time_index; - - auto serde_fields() { - return std::tie( - min_file_pos_step, - num_elements, - base_rp, - last_rp, - base_kaf, - last_kaf, - base_file, - last_file, - rp_write_buf, - kaf_write_buf, - file_write_buf, - rp_index, - kaf_index, - file_index, - base_time, - last_time, - time_write_buf, - time_index); - } -}; - -iobuf offset_index::to_iobuf() const { - offset_index_header hdr{ - .min_file_pos_step = _min_file_pos_step, - .num_elements = _pos, - .base_rp = _initial_rp, - .last_rp = _rp_index.get_last_value(), - .base_kaf = _initial_kaf, - .last_kaf = _kaf_index.get_last_value(), - .base_file = _initial_file_pos, - .last_file = _file_index.get_last_value(), - .rp_write_buf = std::vector( - _rp_offsets.begin(), _rp_offsets.end()), - .kaf_write_buf = std::vector( - _kaf_offsets.begin(), _kaf_offsets.end()), - .file_write_buf = std::vector( - _file_offsets.begin(), _file_offsets.end()), - .rp_index = _rp_index.copy(), - .kaf_index = _kaf_index.copy(), - .file_index = _file_index.copy(), - .base_time = _initial_time.value(), - .last_time = _time_index.get_last_value(), - .time_write_buf = std::vector( - _time_offsets.begin(), _time_offsets.end()), - .time_index = _time_index.copy()}; - return serde::to_iobuf(std::move(hdr)); -} - -void offset_index::from_iobuf(iobuf b) { - iobuf_parser parser(std::move(b)); - auto hdr = serde::read(parser); - auto num_rows = hdr.num_elements / buffer_depth; - _pos = hdr.num_elements; - _initial_rp = model::offset(hdr.base_rp); - _initial_kaf = kafka::offset(hdr.base_kaf); - _initial_file_pos = hdr.base_file; - std::copy( - hdr.rp_write_buf.begin(), hdr.rp_write_buf.end(), _rp_offsets.begin()); - std::copy( - hdr.kaf_write_buf.begin(), hdr.kaf_write_buf.end(), _kaf_offsets.begin()); - std::copy( - hdr.file_write_buf.begin(), - hdr.file_write_buf.end(), - _file_offsets.begin()); - _rp_index = encoder_t( - _initial_rp, num_rows, hdr.last_rp, std::move(hdr.rp_index)); - _kaf_index = encoder_t( - _initial_kaf, num_rows, hdr.last_kaf, std::move(hdr.kaf_index)); - _file_index = foffset_encoder_t( - _initial_file_pos, - num_rows, - hdr.last_file, - std::move(hdr.file_index), - delta_delta_t(_min_file_pos_step)); - _min_file_pos_step = hdr.min_file_pos_step; - - _initial_time = model::timestamp(hdr.base_time); - _time_index = encoder_t( - _initial_time.value(), - num_rows, - hdr.last_time, - std::move(hdr.time_index)); - std::copy( - hdr.time_write_buf.begin(), - hdr.time_write_buf.end(), - _time_offsets.begin()); -} - -std::optional -offset_index::_find_under(deltafor_decoder decoder, int64_t offset) { - size_t ix = 0; - std::array rp_buf{}; - std::optional candidate; - while (decoder.read(rp_buf)) { - for (auto o : rp_buf) { - if (o >= offset) { - return candidate; - } - candidate = {.ix = size_t(ix), .value = o}; - ix++; - } - rp_buf = {}; - } - return candidate; -} - remote_segment_index_builder::remote_segment_index_builder( const model::ntp& ntp, offset_index& ix, diff --git a/src/v/cloud_storage/remote_segment_index.h b/src/v/cloud_storage/remote_segment_index.h index 7fd2cdab40274..d6e8d43f042d3 100644 --- a/src/v/cloud_storage/remote_segment_index.h +++ b/src/v/cloud_storage/remote_segment_index.h @@ -10,174 +10,17 @@ #pragma once -#include "absl/container/btree_map.h" #include "base/seastarx.h" -#include "base/units.h" #include "bytes/iobuf.h" -#include "bytes/iobuf_parser.h" +#include "cloud_storage/offset_index.h" #include "model/fundamental.h" #include "model/record_batch_types.h" #include "storage/parser.h" -#include "utils/delta_for.h" #include -#include - namespace cloud_storage { -/// Offset index for remote_segment -/// -/// The object indexes tuples that contain three elements: -/// - redpanda offset -/// - kafka offset -/// - file offset -/// -/// The search is linear. The underlying data structure is a -/// fragmented buffer (iobuf). It is possible to search by redpanda -/// and kafka offsets, but not by file offset. -/// -/// The invariant of the offset_index is that all three encoders -/// have the same number of elements. All three buffers should also -/// have the same number of elements. -class offset_index { - static constexpr uint32_t buffer_depth = details::FOR_buffer_depth; - static constexpr uint32_t index_mask = buffer_depth - 1; - static_assert( - (buffer_depth & index_mask) == 0, - "buffer_depth have to be a power of two"); - -public: - offset_index( - model::offset initial_rp, - kafka::offset initial_kaf, - int64_t initial_file_pos, - int64_t file_pos_step, - model::timestamp initial_time); - - /// Add new tuple to the index. - void add( - model::offset rp_offset, - kafka::offset kaf_offset, - int64_t file_offset, - model::timestamp); - - struct find_result { - model::offset rp_offset; - kafka::offset kaf_offset; - int64_t file_pos; - }; - - /// Estimate memory usage by the index - size_t estimate_memory_use() const; - - /// Find index entry which is strictly lower than the redpanda offset - /// - /// The returned value has rp_offset less than upper_bound. - /// If all elements are larger than 'upper_bound' nullopt is returned. - /// If all elements are smaller than 'upper_bound' the last value is - /// returned. - std::optional find_rp_offset(model::offset upper_bound); - - /// Find index entry which is strictly lower than the kafka offset - /// - /// The returned value has kaf_offset less than upper_bound. - /// If all elements are larger than 'upper_bound' nullopt is returned. - /// If all elements are smaller than 'upper_bound' the last value is - /// returned. - std::optional find_kaf_offset(kafka::offset upper_bound); - - /// Find index entry which is strictly lower than the timestamp - /// - /// The returned value has timestamp less than upper_bound. - /// If all elements are larger than 'upper_bound' nullopt is returned. - /// If all elements are smaller than 'upper_bound' the last value is - /// returned. - std::optional find_timestamp(model::timestamp upper_bound); - - /// Builds a coarse index mapping kafka offsets to file positions. The step - /// size is the resolution of the index. So given a step size of 16MiB, the - /// result contains mappings of kafka offset to file position from the index - /// where entries are _roughly_ 16MiB apart in terms of file position. - using coarse_index_t = absl::btree_map; - coarse_index_t - build_coarse_index(uint64_t step_size, std::string_view index_path) const; - - /// Serialize offset_index - iobuf to_iobuf() const; - - /// Deserialize offset_index - void from_iobuf(iobuf in); - -private: - struct index_value { - size_t ix; - int64_t value; - }; - - /// Find index entry which is strictly lower than the provided value - /// - /// The encoder and the write buffer have to be provided via parameters. - /// The returned value is a variant which contains a monostate if no - /// value can be found; index_value if the value is found in the - /// encoder; find_result if the value is found in the write buffer (in - /// this case no further search is needed). - std::variant maybe_find_offset( - int64_t upper_bound, - deltafor_encoder& encoder, - const std::array& write_buffer); - - /// Find element inside the offset range stored in the decoder which is - /// less than offset. Return last element if no such element can be - /// found. Return nullopt if all emlements are larger or equal than - /// offset. - static std::optional - _find_under(deltafor_decoder decoder, int64_t offset); - - /// Return element by index. - template - static std::optional - _fetch_ix(DecoderT decoder, size_t target_ix) { - size_t ix = 0; - std::array buffer{}; - while (decoder.read(buffer)) { - for (auto o : buffer) { - if (ix == target_ix) { - return o; - } - ix++; - } - buffer = {}; - } - return std::nullopt; - } - -private: - std::array _rp_offsets; - std::array _kaf_offsets; - std::array _file_offsets; - std::array _time_offsets; - uint64_t _pos; - model::offset _initial_rp; - kafka::offset _initial_kaf; - int64_t _initial_file_pos; - model::timestamp _initial_time; - - using encoder_t = deltafor_encoder; - using decoder_t = deltafor_decoder; - using delta_delta_t = details::delta_delta; - using foffset_encoder_t = deltafor_encoder; - using foffset_decoder_t = deltafor_decoder; - - encoder_t _rp_index; - encoder_t _kaf_index; - foffset_encoder_t _file_index; - encoder_t _time_index; - int64_t _min_file_pos_step; - - friend class offset_index_accessor; -}; - struct segment_record_stats { // Offset of the first record in the segment model::offset base_rp_offset; From 3f11ea138f50d656831c5309f0aa9fb394384425 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Wed, 10 Jun 2026 14:33:42 -0700 Subject: [PATCH 2/9] cloud_topics/l1: carry tiered-storage segment metadata in L1 Add an optional descriptor that lets an L1 object/extent point at a segment that physically lives in the tiered-storage bucket instead of an L1-native object. Metadata/encoding only; no read, write, or delete path consumes it yet. - imported_ts_info (object_id.h): the per-segment descriptor -- ts_path, delta_offset/delta_offset_end (log<->kafka offset translation), segment_term, and last_kafka_offset. Stored split by ownership: ts_path on the object row (imported_ts_object_location), delta/term on the extent row (imported_ts_segment_info); absent means a native L1 object. - Thread it through the metastore read/interface types on both backends: an optional imported_ts_info on object_response / extent_object_info (recomposed from the storage split on read), plus rpc_types and the replicated/domain passthrough. - Encode it: serde-versioned on both backends (object_entry/extent and the LSM rows), the LSM debug-serde path, and the ImportedTsObjectLocation / ImportedTsSegmentInfo protobufs. Old-version decode defaults to absent. Testing: serde round-trip unit tests for the new fields, including old-version decode. Co-Authored-By: Claude Opus 4.8 --- .../internal/cloud_topics/v1/metastore.proto | 28 +++ src/v/cloud_topics/level_one/common/BUILD | 4 + .../cloud_topics/level_one/common/object_id.h | 99 +++++++++- .../level_one/domain/db_domain_manager.cc | 2 + .../level_one/domain/simple_domain_manager.cc | 1 + .../level_one/metastore/lsm/debug_reader.cc | 18 ++ .../level_one/metastore/lsm/debug_serde.cc | 17 ++ .../metastore/lsm/tests/debug_reader_test.cc | 54 ++++++ .../level_one/metastore/lsm/values.h | 6 +- .../level_one/metastore/metastore.h | 18 +- .../metastore/replicated_metastore.cc | 1 + .../level_one/metastore/rpc_types.h | 7 +- .../level_one/metastore/simple_metastore.cc | 6 + .../cloud_topics/level_one/metastore/state.h | 112 ++++++++++- .../level_one/metastore/state_update.cc | 1 + .../level_one/metastore/state_update.h | 27 ++- .../level_one/metastore/tests/BUILD | 17 ++ .../metastore/tests/imported_serde_test.cc | 177 ++++++++++++++++++ .../internal/cloud_topics/v1/metastore_pb2.py | 92 ++++----- .../cloud_topics/v1/metastore_pb2.pyi | 88 ++++++++- 20 files changed, 713 insertions(+), 62 deletions(-) create mode 100644 src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc diff --git a/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto b/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto index 103c0298a2aed..41f2905d19e1a 100644 --- a/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto +++ b/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto @@ -239,6 +239,32 @@ message MetadataValue { uint64 num_extents = 5; } +// Where the backing tiered-storage segment of an imported object lives. An +// object property: the imported analog of a native object's id->bucket path. +message ImportedTsObjectLocation { + string ts_path = 1; +} + +// Whether an imported tiered-storage segment has an aborted-transaction (.tx) +// manifest, resolved at import time so the read path need not probe object +// storage. UNKNOWN (v1/v2 non-compacted) => probe and tolerate a missing .tx. +enum ImportedTsTxManifestState { + IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN = 0; + IMPORTED_TS_TX_MANIFEST_STATE_ABSENT = 1; + IMPORTED_TS_TX_MANIFEST_STATE_PRESENT = 2; +} + +// Properties of the data in an imported tiered-storage extent. An extent +// property, alongside the extent's Kafka offset bounds (base/last_offset). +message ImportedTsSegmentInfo { + int64 segment_term = 1; + // Offset-translation delta at the segment's base (source segment_meta's + // delta_offset). + int64 delta_base = 2; + // Whether the segment's .tx manifest exists (resolved at import time). + ImportedTsTxManifestState tx_state = 3; +} + // Value for an extent row. message ExtentValue { int64 last_offset = 1; @@ -246,6 +272,7 @@ message ExtentValue { uint64 filepos = 3; uint64 len = 4; string object_id = 5; + optional ImportedTsSegmentInfo imported_ts_info = 6; } // Value for a term row. @@ -280,6 +307,7 @@ message ObjectValue { uint64 object_size = 4; int64 last_updated = 5; bool is_preregistration = 6; + optional ImportedTsObjectLocation imported_ts_location = 7; } // A typed value for a metastore row. diff --git a/src/v/cloud_topics/level_one/common/BUILD b/src/v/cloud_topics/level_one/common/BUILD index c1241d3773856..fa7e98c15ef7f 100644 --- a/src/v/cloud_topics/level_one/common/BUILD +++ b/src/v/cloud_topics/level_one/common/BUILD @@ -20,9 +20,13 @@ redpanda_cc_library( ], deps = [ "//src/v/base", + "//src/v/model", + "//src/v/serde", + "//src/v/serde:enum", "//src/v/utils:named_type", "//src/v/utils:uuid", "@fmt", + "@seastar", ], ) diff --git a/src/v/cloud_topics/level_one/common/object_id.h b/src/v/cloud_topics/level_one/common/object_id.h index 27cfcb50e2444..e5cd8f058750c 100644 --- a/src/v/cloud_topics/level_one/common/object_id.h +++ b/src/v/cloud_topics/level_one/common/object_id.h @@ -12,9 +12,17 @@ #pragma once #include "base/format_to.h" +#include "model/fundamental.h" +#include "serde/envelope.h" +#include "serde/rw/enum.h" #include "utils/named_type.h" #include "utils/uuid.h" +#include + +#include +#include + namespace cloud_topics::l1 { // An object ID is a unique identifier for a cloud topic L1 object. @@ -22,13 +30,102 @@ using object_id = named_type; inline object_id create_object_id() { return object_id{uuid_t::create()}; } -// An extent of a remote object, which is a pair of offset and size. +/// The cloud-storage object path of a tiered-storage segment imported into L1 +/// by reference (an opaque sname_format path). +using ts_segment_path = named_type; + +/// Whether an imported tiered-storage segment has an aborted-transaction (.tx) +/// manifest. Resolved once at import time from the source segment_meta so the +/// read path need not probe object storage to find out -- preserving a +/// capability native tiered storage already has (see remote_segment.cc): a v3 +/// segment records its .tx manifest size in metadata_size_hint (0 => none), and +/// a compacted segment has no aborted batches by construction. Only v1/v2 +/// non-compacted segments are genuinely unknowable without looking. +enum class tx_manifest_state : uint8_t { + /// Presence not encoded in the source metadata (v1/v2, non-compacted): the + /// read path probes object storage and tolerates a missing .tx. The + /// default, so a value that was never resolved degrades to the safe + /// (probe-and-tolerate) behavior rather than silently skipping a real .tx. + unknown = 0, + /// Known empty -- compacted (aborted batches removed by compaction) or v3 + /// with metadata_size_hint == 0. The read path skips the .tx download and + /// treats the segment as having no aborted transactions. + absent = 1, + /// Known present -- v3 with metadata_size_hint > 0. The read path downloads + /// the .tx. If it is unexpectedly missing (a lost/partial upload), the read + /// path warns but tolerates it (reads committed-only), matching tiered + /// storage, which treats a notfound .tx as empty. + present = 2, +}; + +/// The full descriptor of a tiered-storage segment imported into L1 by +/// reference, as the metastore *interface* and the IO read path see it: where +/// the segment lives plus how to interpret its data. +/// +/// The metastore *storage* layer decomposes this by ownership -- the path is an +/// object property and the delta/term are extent properties (see +/// `imported_ts_object_location` / `imported_ts_segment_info` in state.h) -- +/// but the API and the reader treat it as one unit. The extent's Kafka offset +/// bounds (carried separately by the API/IO types) give the segment's offset +/// range. +struct imported_ts_info + : public serde:: + envelope, serde::compat_version<0>> { + friend bool + operator==(const imported_ts_info&, const imported_ts_info&) = default; + auto serde_fields() { + return std::tie(ts_path, segment_term, delta_base, tx_state); + } + + /// Opaque tiered-storage segment object path (sname_format path). + ts_segment_path ts_path; + /// Raft term the segment was written in. Stamped onto every imported batch + /// as its partition leader epoch (a tiered-storage segment is single-term), + /// so a Kafka fetch of the imported region reports the original leader + /// epoch rather than -1. + model::term_id segment_term{}; + /// Offset-translation delta at the segment's base (the source + /// segment_meta's delta_offset: base log offset minus base Kafka offset). + /// Seeds the reader's running delta when a seek scans from the segment + /// start, so translation is correct even when compaction has removed the + /// leading records -- the first surviving batch may sit past the declared + /// base, and inferring the delta from it would renumber survivors down into + /// the hole. + model::offset_delta delta_base{}; + /// Whether the segment's aborted-transaction (.tx) manifest exists, + /// resolved at import time so the read path can skip the probe where the + /// answer is known. See tx_manifest_state. + tx_manifest_state tx_state{tx_manifest_state::unknown}; +}; + +/// An L1 object or imported TS segment, as seen by the IO read path. For an +/// imported extent `imported` is the segment descriptor (path + term + delta). +/// +/// `position` and `size` carry format-specific semantics: +/// native (imported == nullopt): +/// position = footer byte offset within the object +/// size = footer length in bytes (= object_size - footer_pos) +/// imported (imported.has_value()): +/// position = 0 (unused) +/// size = total segment size in bytes struct object_extent { object_id id; size_t position = 0; size_t size = 0; + /// Set for an imported extent (nullopt for native L1); carries the + /// segment's path, term, and offset-translation delta. + std::optional imported; fmt::iterator format_to(fmt::iterator it) const; }; +/// One object to delete via io::delete_objects. A native L1 object (ts_path +/// nullopt) is deleted by its id-derived path; an imported tiered-storage +/// segment (ts_path set) is deleted at that path (the segment plus its .tx and +/// .index). Both reside in the one configured object bucket. +struct object_location { + object_id id; + std::optional ts_path; +}; + } // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/domain/db_domain_manager.cc b/src/v/cloud_topics/level_one/domain/db_domain_manager.cc index 295effa9a9b28..eadcd2b7f1828 100644 --- a/src/v/cloud_topics/level_one/domain/db_domain_manager.cc +++ b/src/v/cloud_topics/level_one/domain/db_domain_manager.cc @@ -1559,6 +1559,8 @@ db_domain_manager::get_extent_metadata(rpc::get_extent_metadata_request req) { .oid = extent.val.oid, .footer_pos = object.footer_pos, .object_size = object.object_size, + .imported = to_imported_ts_info( + object.imported_ts_location, extent.val.imported_ts_info), }; } extents.push_back(std::move(em)); diff --git a/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc b/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc index 675ddebd0d543..636e0a52e7c06 100644 --- a/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc +++ b/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc @@ -59,6 +59,7 @@ meta_to_rpc_extent_metadata(metastore::extent_metadata_vec v) { .oid = e.object_info->oid, .footer_pos = e.object_info->footer_pos, .object_size = e.object_info->object_size, + .imported = e.object_info->imported, }; } res.push_back( diff --git a/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc b/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc index 111d3c979f567..ebe50523f1c68 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc @@ -168,6 +168,18 @@ debug_reader::decode_value(row_type type, iobuf value) { ev.set_filepos(rv.filepos); ev.set_len(rv.len); ev.set_object_id(uuid_to_string(rv.oid())); + if (rv.imported_ts_info.has_value()) { + proto::admin::metastore::imported_ts_segment_info isi; + isi.set_segment_term(rv.imported_ts_info->segment_term()); + isi.set_delta_base(rv.imported_ts_info->delta_base()); + // The proto enum values mirror tx_manifest_state 1:1 (see the + // .proto); a straight cast keeps them in sync. + isi.set_tx_state( + static_cast< + proto::admin::metastore::imported_ts_tx_manifest_state>( + static_cast(rv.imported_ts_info->tx_state))); + ev.set_imported_ts_info(std::move(isi)); + } pv.set_extent(std::move(ev)); return pv; } @@ -210,6 +222,12 @@ debug_reader::decode_value(row_type type, iobuf value) { ov.set_object_size(rv.object.object_size); ov.set_last_updated(rv.object.last_updated()); ov.set_is_preregistration(rv.object.is_preregistration); + if (rv.object.imported_ts_location.has_value()) { + proto::admin::metastore::imported_ts_object_location loc; + loc.set_ts_path( + ss::sstring{rv.object.imported_ts_location->ts_path()}); + ov.set_imported_ts_location(std::move(loc)); + } pv.set_object(std::move(ov)); return pv; } diff --git a/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc b/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc index 90c0d875d46aa..035c2d6204629 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc @@ -163,6 +163,17 @@ debug_encode_value(const proto::admin::metastore::row_value& val) { .len = static_cast(v.get_len()), .oid = object_id{*uuid}, }; + if (v.has_imported_ts_info()) { + const auto& isi = v.get_imported_ts_info(); + rv.imported_ts_info = imported_ts_segment_info{ + .segment_term = model::term_id{isi.get_segment_term()}, + .delta_base = model::offset_delta{isi.get_delta_base()}, + // The proto enum values mirror tx_manifest_state 1:1 (see the + // .proto); a straight cast keeps them in sync. + .tx_state = static_cast( + static_cast(isi.get_tx_state())), + }; + } return serde::to_iobuf(std::move(rv)); } if (val.has_term()) { @@ -204,6 +215,12 @@ debug_encode_value(const proto::admin::metastore::row_value& val) { .last_updated = model::timestamp{v.get_last_updated()}, .is_preregistration = v.get_is_preregistration(), }; + if (v.has_imported_ts_location()) { + const auto& loc = v.get_imported_ts_location(); + entry.imported_ts_location = imported_ts_object_location{ + .ts_path = ts_segment_path{ss::sstring{loc.get_ts_path()}}, + }; + } object_row_value rv{ .object = std::move(entry), }; diff --git a/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc b/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc index 01689cfa63eaa..0ce743e30b480 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc @@ -238,6 +238,32 @@ TEST(DebugReaderTest, RoundTripExtentValue) { EXPECT_EQ(decoded->get_extent().get_filepos(), 512); EXPECT_EQ(decoded->get_extent().get_len(), 1024); EXPECT_EQ(decoded->get_extent().get_object_id(), test_obj_uuid_str); + EXPECT_FALSE(decoded->get_extent().has_imported_ts_info()); +} + +TEST(DebugReaderTest, RoundTripExtentValueImported) { + pm::row_value val; + pm::extent_value ev; + ev.set_last_offset(199); + ev.set_max_timestamp(42000); + ev.set_filepos(512); + ev.set_len(1024); + ev.set_object_id(std::string(test_obj_uuid_str)); + pm::imported_ts_segment_info isi; + isi.set_segment_term(3); + ev.set_imported_ts_info(std::move(isi)); + val.set_extent(std::move(ev)); + + auto encoded = debug_encode_value(val); + ASSERT_TRUE(encoded.has_value()); + + auto decoded = debug_reader::decode_value( + row_type::extent, std::move(*encoded)); + ASSERT_TRUE(decoded.has_value()); + ASSERT_TRUE(decoded->has_extent()); + ASSERT_TRUE(decoded->get_extent().has_imported_ts_info()); + const auto& d = decoded->get_extent().get_imported_ts_info(); + EXPECT_EQ(d.get_segment_term(), 3); } TEST(DebugReaderTest, RoundTripTermValue) { @@ -295,6 +321,34 @@ TEST(DebugReaderTest, RoundTripObjectValue) { EXPECT_EQ(decoded->get_object().get_object_size(), 5100); EXPECT_EQ(decoded->get_object().get_last_updated(), 99999); EXPECT_TRUE(decoded->get_object().get_is_preregistration()); + EXPECT_FALSE(decoded->get_object().has_imported_ts_location()); +} + +TEST(DebugReaderTest, RoundTripObjectValueImported) { + pm::row_value val; + pm::object_value ov; + ov.set_total_data_size(5000); + ov.set_removed_data_size(100); + ov.set_footer_pos(4800); + ov.set_object_size(5100); + ov.set_last_updated(99999); + ov.set_is_preregistration(false); + pm::imported_ts_object_location loc; + loc.set_ts_path("a/b/c-segment.log.1"); + ov.set_imported_ts_location(std::move(loc)); + val.set_object(std::move(ov)); + + auto encoded = debug_encode_value(val); + ASSERT_TRUE(encoded.has_value()); + + auto decoded = debug_reader::decode_value( + row_type::object, std::move(*encoded)); + ASSERT_TRUE(decoded.has_value()); + ASSERT_TRUE(decoded->has_object()); + ASSERT_TRUE(decoded->get_object().has_imported_ts_location()); + EXPECT_EQ( + decoded->get_object().get_imported_ts_location().get_ts_path(), + "a/b/c-segment.log.1"); } TEST(DebugReaderTest, UnknownRowTypeReturnsError) { diff --git a/src/v/cloud_topics/level_one/metastore/lsm/values.h b/src/v/cloud_topics/level_one/metastore/lsm/values.h index 619f2e3f1f50c..7ed8aa060add0 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/values.h +++ b/src/v/cloud_topics/level_one/metastore/lsm/values.h @@ -38,15 +38,17 @@ struct metadata_row_value struct extent_row_value : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { auto serde_fields() { - return std::tie(last_offset, max_timestamp, filepos, len, oid); + return std::tie( + last_offset, max_timestamp, filepos, len, oid, imported_ts_info); } kafka::offset last_offset; model::timestamp max_timestamp; size_t filepos{0}; size_t len{0}; object_id oid{}; + std::optional imported_ts_info; }; struct term_row_value diff --git a/src/v/cloud_topics/level_one/metastore/metastore.h b/src/v/cloud_topics/level_one/metastore/metastore.h index 448f1a730b5e2..603f9b583b65e 100644 --- a/src/v/cloud_topics/level_one/metastore/metastore.h +++ b/src/v/cloud_topics/level_one/metastore/metastore.h @@ -18,6 +18,7 @@ #include "container/chunked_vector.h" #include "model/fundamental.h" #include "model/timestamp.h" +#include "serde/rw/optional.h" #include #include @@ -100,6 +101,12 @@ class metastore { // The last offset available in the object (inclusive). // This can be used to skip to the next offset. kafka::offset last_offset; + // Set for an imported tiered-storage segment; nullopt for native L1. + // Recomposed from the storage split for the read path: ts_path from the + // object row, segment_term from the extent row, and the Kafka offset + // bounds from the extent (see extent_object_info, which is the + // authoritative read-side carrier -- object_response relays it). + std::optional imported; }; // Interface to build object metadata for the L1 metastore. Meant to be @@ -485,6 +492,8 @@ class metastore { object_id oid; size_t footer_pos{0}; size_t object_size{0}; + // Set for an imported tiered-storage segment; nullopt for native L1. + std::optional imported; }; struct extent_metadata { @@ -496,16 +505,21 @@ class metastore { fmt::iterator format_to(fmt::iterator it) const { if (object_info.has_value()) { - return fmt::format_to( + it = fmt::format_to( it, "{{offsets:({}~{}), max_timestamp:{}, oid:{}, " - "footer_pos:{}, object_size:{}}}", + "footer_pos:{}, object_size:{}", base_offset, last_offset, max_timestamp, object_info->oid, object_info->footer_pos, object_info->object_size); + if (object_info->imported) { + it = fmt::format_to( + it, ", ts_path:{}", object_info->imported->ts_path); + } + return fmt::format_to(it, "}}"); } return fmt::format_to( it, diff --git a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc index 6303d59da3191..99bb96f9b1257 100644 --- a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc @@ -296,6 +296,7 @@ rpc_to_meta_extent_metadata(chunked_vector v) { .oid = e.object_info->oid, .footer_pos = e.object_info->footer_pos, .object_size = e.object_info->object_size, + .imported = e.object_info->imported, }; } res.push_back( diff --git a/src/v/cloud_topics/level_one/metastore/rpc_types.h b/src/v/cloud_topics/level_one/metastore/rpc_types.h index e347b75148828..406ded87a7839 100644 --- a/src/v/cloud_topics/level_one/metastore/rpc_types.h +++ b/src/v/cloud_topics/level_one/metastore/rpc_types.h @@ -242,13 +242,16 @@ struct get_size_request struct extent_object_info : serde::envelope< extent_object_info, - serde::version<0>, + serde::version<1>, serde::compat_version<0>> { - auto serde_fields() { return std::tie(oid, footer_pos, object_size); } + auto serde_fields() { + return std::tie(oid, footer_pos, object_size, imported); + } object_id oid; size_t footer_pos{0}; size_t object_size{0}; + std::optional imported; }; struct extent_metadata diff --git a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc index aa37027bc12be..7b3d0b48dc338 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc @@ -369,6 +369,8 @@ simple_metastore::get_first_ge( .object_size = object_size, .first_offset = it->base_offset, .last_offset = it->last_offset, + .imported = to_imported_ts_info( + object_it->second.imported_ts_location, it->imported_ts_info), }; } return std::unexpected(metastore::errc::out_of_range); @@ -409,6 +411,8 @@ simple_metastore::get_first_ge( .object_size = object_size, .first_offset = obj.base_offset, .last_offset = obj.last_offset, + .imported = to_imported_ts_info( + object_it->second.imported_ts_location, obj.imported_ts_info), }; } } @@ -902,6 +906,8 @@ simple_metastore::get_extent_metadata_forwards( .oid = ext.oid, .footer_pos = object_it->second.footer_pos, .object_size = object_it->second.object_size, + .imported = to_imported_ts_info( + object_it->second.imported_ts_location, ext.imported_ts_info), }; } diff --git a/src/v/cloud_topics/level_one/metastore/state.h b/src/v/cloud_topics/level_one/metastore/state.h index 5084f85684e30..9ffd3b466480b 100644 --- a/src/v/cloud_topics/level_one/metastore/state.h +++ b/src/v/cloud_topics/level_one/metastore/state.h @@ -22,6 +22,7 @@ #include "serde/rw/set.h" #include +#include #include @@ -30,17 +31,111 @@ namespace cloud_topics::l1 { // Represents the state managed by the replicated state machine that serves L1 // metadata. +// The metastore stores an imported tiered-storage segment's descriptor +// (l1::imported_ts_info) decomposed by row ownership: the path is an object +// property, the delta/term are extent properties. The interface and IO path +// recompose the two into an imported_ts_info; only the storage layer (and its +// debug-serde encoding) sees this split. + +/// Where a referenced tiered-storage segment lives. An object property: the +/// imported analog of a native object's oid -> bucket path, independent of +/// which Kafka offsets any extent over it covers. +struct imported_ts_object_location + : public serde::envelope< + imported_ts_object_location, + serde::version<0>, + serde::compat_version<0>> { + friend bool operator==( + const imported_ts_object_location&, + const imported_ts_object_location&) = default; + auto serde_fields() { return std::tie(ts_path); } + + /// Opaque tiered-storage segment object path (sname_format path). + ts_segment_path ts_path; +}; + +/// How to interpret the data within an extent imported from a tiered-storage +/// segment. An extent property, alongside the extent's Kafka offset bounds +/// (base_offset/last_offset). +struct imported_ts_segment_info + : public serde::envelope< + imported_ts_segment_info, + serde::version<0>, + serde::compat_version<0>> { + friend bool + operator==(const imported_ts_segment_info&, const imported_ts_segment_info&) + = default; + std::strong_ordering + operator<=>(const imported_ts_segment_info&) const = default; + auto serde_fields() { return std::tie(segment_term, delta_base, tx_state); } + + /// Raft term the segment was written in. Stamped onto every imported batch + /// as its partition leader epoch (a tiered-storage segment is single-term), + /// so a Kafka fetch of the imported region reports the original leader + /// epoch rather than -1. + model::term_id segment_term{}; + /// Offset-translation delta at the segment's base (source segment_meta's + /// delta_offset). Seeds the reader's running delta at the segment start so + /// translation survives a compacted front hole. See l1::imported_ts_info. + model::offset_delta delta_base{}; + /// Whether the segment's aborted-transaction (.tx) manifest exists, + /// resolved at import time so the read path can skip the probe. See + /// l1::tx_manifest_state. + tx_manifest_state tx_state{tx_manifest_state::unknown}; +}; + +// Translate between the storage split (object-owned location + extent-owned +// segment descriptor) and the unified imported_ts_info the interface and IO +// path use. +inline imported_ts_object_location +to_object_location(const imported_ts_info& i) { + return imported_ts_object_location{.ts_path = i.ts_path}; +} +inline imported_ts_segment_info to_segment_info(const imported_ts_info& i) { + return imported_ts_segment_info{ + .segment_term = i.segment_term, + .delta_base = i.delta_base, + .tx_state = i.tx_state}; +} +inline imported_ts_info to_imported_ts_info( + const imported_ts_object_location& loc, const imported_ts_segment_info& seg) { + return imported_ts_info{ + .ts_path = loc.ts_path, + .segment_term = seg.segment_term, + .delta_base = seg.delta_base, + .tx_state = seg.tx_state}; +} +// Recompose a read response's imported_ts_info from the object row's location +// and the extent row's segment descriptor (set together for an imported extent, +// both nullopt for native L1). The extent's Kafka offset bounds live on the +// extent itself (base_offset/last_offset); the read path does not need them +// here. +inline std::optional to_imported_ts_info( + const std::optional& loc, + const std::optional& seg) { + if (loc.has_value() && seg.has_value()) { + return to_imported_ts_info(*loc, *seg); + } + return std::nullopt; +} + // A description of a portion of an object that points at data for a contiguous // range of batches belonging to a single partition. The object itself may // contain ranges of data from many partitions. struct extent : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { friend bool operator==(const extent&, const extent&) = default; std::strong_ordering operator<=>(const extent&) const = default; auto serde_fields() { return std::tie( - base_offset, last_offset, max_timestamp, filepos, len, oid); + base_offset, + last_offset, + max_timestamp, + filepos, + len, + oid, + imported_ts_info); } kafka::offset base_offset; @@ -50,6 +145,10 @@ struct extent size_t len; // TODO: avoid duplicating the UUIDs with some indirection. object_id oid; + // Properties of the data in this extent when it is imported from a + // tiered-storage segment (nullopt for native L1). The segment's Kafka + // bounds are base_offset/last_offset above; ts_path lives on the object. + std::optional imported_ts_info; fmt::iterator format_to(fmt::iterator it) const { return fmt::format_to( @@ -273,7 +372,7 @@ struct topic_state // Metadata about a given object that is not specific to any partition. struct object_entry : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { friend bool operator==(const object_entry&, const object_entry&) = default; auto serde_fields() { return std::tie( @@ -282,7 +381,8 @@ struct object_entry footer_pos, object_size, last_updated, - is_preregistration); + is_preregistration, + imported_ts_location); } size_t total_data_size{0}; size_t removed_data_size{0}; @@ -290,6 +390,10 @@ struct object_entry size_t object_size{0}; model::timestamp last_updated; bool is_preregistration{false}; + // Location of the backing tiered-storage segment when this object is + // imported by reference (nullopt for native L1). Used by GC to delete the + // segment + .tx + .index; the per-data fields live on the extent. + std::optional imported_ts_location; }; // Tracks the state of each topic revision. diff --git a/src/v/cloud_topics/level_one/metastore/state_update.cc b/src/v/cloud_topics/level_one/metastore/state_update.cc index 758781664e45f..d2fdf6594b8f7 100644 --- a/src/v/cloud_topics/level_one/metastore/state_update.cc +++ b/src/v/cloud_topics/level_one/metastore/state_update.cc @@ -267,6 +267,7 @@ new_object::collect_extents_by_tidp(sorted_extents_by_tidp_t* ret) const { .filepos = extent_meta.filepos, .len = extent_meta.len, .oid = oid, + .imported_ts_info = extent_meta.imported_ts_info, }); } } diff --git a/src/v/cloud_topics/level_one/metastore/state_update.h b/src/v/cloud_topics/level_one/metastore/state_update.h index dc724f2c7a425..7ce7c8519a618 100644 --- a/src/v/cloud_topics/level_one/metastore/state_update.h +++ b/src/v/cloud_topics/level_one/metastore/state_update.h @@ -36,14 +36,19 @@ using stm_update_error = named_type; struct new_object : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { struct metadata : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { friend bool operator==(const metadata&, const metadata&) = default; auto serde_fields() { return std::tie( - base_offset, last_offset, max_timestamp, filepos, len); + base_offset, + last_offset, + max_timestamp, + filepos, + len, + imported_ts_info); } kafka::offset base_offset; @@ -51,11 +56,18 @@ struct new_object model::timestamp max_timestamp; size_t filepos; size_t len; + // The imported segment's descriptor (delta/term), for an extent + // imported from a tiered-storage segment; nullopt for native L1. An + // extent property -- the segment's location (ts_path) is an object + // property carried on new_object directly. apply() copies this onto the + // extent row verbatim. + std::optional imported_ts_info; }; friend bool operator==(const new_object&, const new_object&) = default; auto serde_fields() { - return std::tie(oid, footer_pos, object_size, extent_metas); + return std::tie( + oid, footer_pos, object_size, extent_metas, imported_ts_location); } object_id oid; @@ -66,6 +78,13 @@ struct new_object chunked_hash_map> extent_metas; + // The imported segment's location (ts_path), for an object imported from a + // tiered-storage segment; nullopt for natively written L1 objects. An + // object property -- the per-extent segment descriptor (delta/term) lives + // on each `metadata` entry above. apply() copies this onto the object row + // verbatim. + std::optional imported_ts_location; + // Returns the sum of lengths of the extents collected. size_t collect_extents_by_tidp(sorted_extents_by_tidp_t*) const; }; diff --git a/src/v/cloud_topics/level_one/metastore/tests/BUILD b/src/v/cloud_topics/level_one/metastore/tests/BUILD index 0fd8fa45006e1..d7da7ee4ef8c9 100644 --- a/src/v/cloud_topics/level_one/metastore/tests/BUILD +++ b/src/v/cloud_topics/level_one/metastore/tests/BUILD @@ -239,6 +239,23 @@ redpanda_cc_gtest( ], ) +redpanda_cc_gtest( + name = "imported_serde_test", + timeout = "short", + srcs = [ + "imported_serde_test.cc", + ], + deps = [ + "//src/v/bytes:iobuf", + "//src/v/cloud_topics/level_one/common:object_id", + "//src/v/cloud_topics/level_one/metastore:state", + "//src/v/cloud_topics/level_one/metastore/lsm:values", + "//src/v/serde", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + ], +) + redpanda_cc_gtest( name = "extent_metadata_reader_test", timeout = "short", diff --git a/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc b/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc new file mode 100644 index 0000000000000..5c9a9310ebc90 --- /dev/null +++ b/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc @@ -0,0 +1,177 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ + +#include "bytes/iobuf.h" +#include "cloud_topics/level_one/common/object_id.h" +#include "cloud_topics/level_one/metastore/lsm/values.h" +#include "cloud_topics/level_one/metastore/state.h" +#include "serde/rw/envelope.h" + +#include + +using namespace cloud_topics::l1; + +namespace { + +/// Mirrors extent_row_value at version<0>, before the `imported` field was +/// added in version<1>. +struct extent_row_value_v0 + : public serde::envelope< + extent_row_value_v0, + serde::version<0>, + serde::compat_version<0>> { + auto serde_fields() { + return std::tie(last_offset, max_timestamp, filepos, len, oid); + } + kafka::offset last_offset{}; + model::timestamp max_timestamp{}; + size_t filepos{0}; + size_t len{0}; + object_id oid{}; +}; + +/// Mirrors object_entry at version<1>, before the `imported` field was added +/// in version<2>. +struct object_entry_v1 + : public serde:: + envelope, serde::compat_version<0>> { + auto serde_fields() { + return std::tie( + total_data_size, + removed_data_size, + footer_pos, + object_size, + last_updated, + is_preregistration); + } + size_t total_data_size{0}; + size_t removed_data_size{0}; + size_t footer_pos{0}; + size_t object_size{0}; + model::timestamp last_updated{}; + bool is_preregistration{false}; +}; + +} // namespace + +// Encode a v0 extent_row_value (no `imported` field) and verify that decoding +// it as the current v1 struct leaves `imported` as nullopt. +TEST(ImportedSerdeTest, ExtentRowValueV0CompatV1) { + extent_row_value_v0 v0; + v0.last_offset = kafka::offset{42}; + v0.max_timestamp = model::timestamp{1000}; + v0.filepos = 100; + v0.len = 200; + v0.oid = create_object_id(); + + iobuf buf = serde::to_iobuf(v0); + auto v1 = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(v1.last_offset, v0.last_offset); + ASSERT_EQ(v1.max_timestamp, v0.max_timestamp); + ASSERT_EQ(v1.filepos, v0.filepos); + ASSERT_EQ(v1.len, v0.len); + ASSERT_EQ(v1.oid, v0.oid); + ASSERT_EQ(v1.imported_ts_info, std::nullopt); +} + +// Verify that a v1 extent_row_value with `imported` populated round-trips +// through serde correctly. +TEST(ImportedSerdeTest, ExtentRowValueV1RoundTrip) { + extent_row_value v1; + v1.last_offset = kafka::offset{100}; + v1.max_timestamp = model::timestamp{2000}; + v1.filepos = 300; + v1.len = 400; + v1.oid = create_object_id(); + v1.imported_ts_info = imported_ts_segment_info{ + .segment_term = model::term_id{7}, + .delta_base = model::offset_delta{5}, + .tx_state = tx_manifest_state::present, + }; + + iobuf buf = serde::to_iobuf(v1); + auto decoded = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(decoded.last_offset, v1.last_offset); + ASSERT_EQ(decoded.max_timestamp, v1.max_timestamp); + ASSERT_EQ(decoded.filepos, v1.filepos); + ASSERT_EQ(decoded.len, v1.len); + ASSERT_EQ(decoded.oid, v1.oid); + ASSERT_TRUE(decoded.imported_ts_info.has_value()); + ASSERT_EQ(decoded.imported_ts_info, v1.imported_ts_info); +} + +// Encode a v1 object_entry (no `imported` field) and verify that decoding it +// as the current v2 struct leaves `imported` as nullopt. +TEST(ImportedSerdeTest, ObjectEntryV1CompatV2) { + object_entry_v1 v1; + v1.total_data_size = 1024; + v1.removed_data_size = 512; + v1.footer_pos = 900; + v1.object_size = 1000; + v1.last_updated = model::timestamp{3000}; + v1.is_preregistration = false; + + iobuf buf = serde::to_iobuf(v1); + auto v2 = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(v2.total_data_size, v1.total_data_size); + ASSERT_EQ(v2.removed_data_size, v1.removed_data_size); + ASSERT_EQ(v2.footer_pos, v1.footer_pos); + ASSERT_EQ(v2.object_size, v1.object_size); + ASSERT_EQ(v2.last_updated, v1.last_updated); + ASSERT_EQ(v2.is_preregistration, v1.is_preregistration); + ASSERT_EQ(v2.imported_ts_location, std::nullopt); +} + +// Verify that a v2 object_entry with `imported` populated round-trips through +// serde correctly. +TEST(ImportedSerdeTest, ObjectEntryV2RoundTrip) { + object_entry v2; + v2.total_data_size = 2048; + v2.removed_data_size = 0; + v2.footer_pos = 1900; + v2.object_size = 2000; + v2.last_updated = model::timestamp{4000}; + v2.is_preregistration = false; + v2.imported_ts_location = imported_ts_object_location{ + .ts_path = ts_segment_path{"00000000000000000000-1-v1.log"}, + }; + + iobuf buf = serde::to_iobuf(v2); + auto decoded = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(decoded, v2); +} + +// Verify that imported_ts_segment_info round-trips through serde correctly. +TEST(ImportedSerdeTest, ImportedTsSegmentInfoRoundTrip) { + imported_ts_segment_info info{ + .segment_term = model::term_id{3}, + .delta_base = model::offset_delta{2}, + .tx_state = tx_manifest_state::absent, + }; + + iobuf buf = serde::to_iobuf(info); + auto decoded = serde::from_iobuf(std::move(buf)); + ASSERT_EQ(decoded, info); +} +// Verify that imported_ts_object_location round-trips through serde correctly. +TEST(ImportedSerdeTest, ImportedTsObjectLocationRoundTrip) { + imported_ts_object_location loc{ + .ts_path = ts_segment_path{"00000000000000000000-1-v1.log"}, + }; + + iobuf buf = serde::to_iobuf(loc); + auto decoded = serde::from_iobuf( + std::move(buf)); + ASSERT_EQ(decoded, loc); +} diff --git a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py index ed2b198478d31..a6a00619046ee 100644 --- a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py +++ b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py @@ -9,7 +9,7 @@ from ........proto.redpanda.core.pbgen import options_pb2 as proto_dot_redpanda_dot_core_dot_pbgen_dot_options__pb2 from ........proto.redpanda.core.pbgen import rpc_pb2 as proto_dot_redpanda_dot_core_dot_pbgen_dot_rpc__pb2 from ........proto.redpanda.core.common.v1 import ntp_pb2 as proto_dot_redpanda_dot_core_dot_common_dot_v1_dot_ntp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\nBproto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto\x12,redpanda.core.admin.internal.cloud_topics.v1\x1a\'proto/redpanda/core/pbgen/options.proto\x1a#proto/redpanda/core/pbgen/rpc.proto\x1a\'proto/redpanda/core/common/v1/ntp.proto"O\n\x11GetOffsetsRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"\\\n\x12GetOffsetsResponse\x12F\n\x07offsets\x18\x02 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.Offsets"4\n\x07Offsets\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03"L\n\x0eGetSizeRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"%\n\x0fGetSizeResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x04"6\n\x17GetDatabaseStatsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r"\xbd\x01\n\x18GetDatabaseStatsResponse\x12\x1d\n\x15active_memtable_bytes\x18\x01 \x01(\x04\x12 \n\x18immutable_memtable_bytes\x18\x02 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x01(\x04\x12F\n\x06levels\x18\x04 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.LsmLevel"f\n\x08LsmLevel\x12\x14\n\x0clevel_number\x18\x01 \x01(\x05\x12D\n\x05files\x18\x02 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.LsmFile"m\n\x07LsmFile\x12\r\n\x05epoch\x18\x01 \x01(\x04\x12\n\n\x02id\x18\x02 \x01(\x04\x12\x12\n\nsize_bytes\x18\x03 \x01(\x04\x12\x19\n\x11smallest_key_info\x18\x04 \x01(\t\x12\x18\n\x10largest_key_info\x18\x05 \x01(\t"5\n\x0bMetadataKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"H\n\tExtentKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x13\n\x0bbase_offset\x18\x03 \x01(\x03"B\n\x07TermKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x0f\n\x07term_id\x18\x03 \x01(\x03"7\n\rCompactionKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"\x1e\n\tObjectKey\x12\x11\n\tobject_id\x18\x01 \x01(\t"\x8e\x03\n\x06RowKey\x12M\n\x08metadata\x18\x01 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.MetadataKeyH\x00\x12I\n\x06extent\x18\x02 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ExtentKeyH\x00\x12E\n\x04term\x18\x03 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.TermKeyH\x00\x12Q\n\ncompaction\x18\x04 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.CompactionKeyH\x00\x12I\n\x06object\x18\x05 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ObjectKeyH\x00B\x05\n\x03key"w\n\rMetadataValue\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x18\n\x10compaction_epoch\x18\x03 \x01(\x03\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x13\n\x0bnum_extents\x18\x05 \x01(\x04"j\n\x0bExtentValue\x12\x13\n\x0blast_offset\x18\x01 \x01(\x03\x12\x15\n\rmax_timestamp\x18\x02 \x01(\x03\x12\x0f\n\x07filepos\x18\x03 \x01(\x04\x12\x0b\n\x03len\x18\x04 \x01(\x04\x12\x11\n\tobject_id\x18\x05 \x01(\t"&\n\tTermValue\x12\x19\n\x11term_start_offset\x18\x01 \x01(\x03"7\n\x0bOffsetRange\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"j\n\x1aCleanedRangeWithTombstones\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12"\n\x1acleaned_with_tombstones_at\x18\x03 \x01(\x03"\xd6\x01\n\x0fCompactionValue\x12Q\n\x0ecleaned_ranges\x18\x01 \x03(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.OffsetRange\x12p\n\x1ecleaned_ranges_with_tombstones\x18\x02 \x03(\x0b2H.redpanda.core.admin.internal.cloud_topics.v1.CleanedRangeWithTombstones"\x9c\x01\n\x0bObjectValue\x12\x17\n\x0ftotal_data_size\x18\x01 \x01(\x04\x12\x19\n\x11removed_data_size\x18\x02 \x01(\x04\x12\x12\n\nfooter_pos\x18\x03 \x01(\x04\x12\x13\n\x0bobject_size\x18\x04 \x01(\x04\x12\x14\n\x0clast_updated\x18\x05 \x01(\x03\x12\x1a\n\x12is_preregistration\x18\x06 \x01(\x08"\x9c\x03\n\x08RowValue\x12O\n\x08metadata\x18\x01 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.MetadataValueH\x00\x12K\n\x06extent\x18\x02 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ExtentValueH\x00\x12G\n\x04term\x18\x03 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.TermValueH\x00\x12S\n\ncompaction\x18\x04 \x01(\x0b2=.redpanda.core.admin.internal.cloud_topics.v1.CompactionValueH\x00\x12K\n\x06object\x18\x05 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ObjectValueH\x00B\x07\n\x05value"\x94\x01\n\x08WriteRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\xbe\x01\n\x10WriteRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12F\n\x06writes\x18\x02 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.WriteRow\x12E\n\x07deletes\x18\x03 \x03(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey")\n\x11WriteRowsResponse\x12\x14\n\x0crows_written\x18\x01 \x01(\r"\x93\x01\n\x07ReadRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\x94\x02\n\x0fReadRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12H\n\x08seek_key\x18\x02 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x00\x12\x16\n\x0craw_seek_key\x18\x03 \x01(\tH\x00\x12H\n\x08last_key\x18\x04 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x01\x12\x16\n\x0craw_last_key\x18\x05 \x01(\tH\x01\x12\x10\n\x08max_rows\x18\x06 \x01(\rB\x06\n\x04seekB\x06\n\x04last"i\n\x10ReadRowsResponse\x12C\n\x04rows\x18\x01 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.ReadRow\x12\x10\n\x08next_key\x18\x02 \x01(\t"\xc8\x01\n\x18ValidatePartitionRequest\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x1d\n\x15check_object_metadata\x18\x03 \x01(\x08\x12\x1c\n\x14check_object_storage\x18\x04 \x01(\x08\x12\x1d\n\x10resume_at_offset\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x13\n\x0bmax_extents\x18\x06 \x01(\rB\x13\n\x11_resume_at_offset"\xbd\x01\n\x19ValidatePartitionResponse\x12Q\n\tanomalies\x18\x01 \x03(\x0b2>.redpanda.core.admin.internal.cloud_topics.v1.MetastoreAnomaly\x12\x1d\n\x10resume_at_offset\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11extents_validated\x18\x03 \x01(\rB\x13\n\x11_resume_at_offset"x\n\x10MetastoreAnomaly\x12O\n\x0canomaly_type\x18\x01 \x01(\x0e29.redpanda.core.admin.internal.cloud_topics.v1.AnomalyType\x12\x13\n\x0bdescription\x18\x02 \x01(\t"F\n\x16ListCloudTopicsRequest\x12\x18\n\x10after_topic_name\x18\x01 \x01(\t\x12\x12\n\nmax_topics\x18\x02 \x01(\r"y\n\x17ListCloudTopicsResponse\x12L\n\x06topics\x18\x01 \x03(\x0b2<.redpanda.core.admin.internal.cloud_topics.v1.CloudTopicInfo\x12\x10\n\x08has_more\x18\x02 \x01(\x08"O\n\x0eCloudTopicInfo\x12\x12\n\ntopic_name\x18\x01 \x01(\t\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x17\n\x0fpartition_count\x18\x03 \x01(\x05*\xbb\x04\n\x0bAnomalyType\x12\x1c\n\x18ANOMALY_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bANOMALY_TYPE_EXTENT_OVERLAP\x10\x01\x12\x1b\n\x17ANOMALY_TYPE_EXTENT_GAP\x10\x02\x12%\n!ANOMALY_TYPE_NEXT_OFFSET_MISMATCH\x10\x03\x12&\n"ANOMALY_TYPE_START_OFFSET_MISMATCH\x10\x04\x12!\n\x1dANOMALY_TYPE_OBJECT_NOT_FOUND\x10\x05\x12%\n!ANOMALY_TYPE_OBJECT_PREREGISTERED\x10\x06\x12&\n"ANOMALY_TYPE_OBJECT_NOT_IN_STORAGE\x10\x07\x12-\n)ANOMALY_TYPE_COMPACTION_RANGE_BELOW_START\x10\x08\x12,\n(ANOMALY_TYPE_COMPACTION_RANGE_ABOVE_NEXT\x10\t\x12-\n)ANOMALY_TYPE_COMPACTION_TOMBSTONE_OVERLAP\x10\n\x125\n1ANOMALY_TYPE_COMPACTION_TOMBSTONE_OUTSIDE_CLEANED\x10\x0b\x12\x1e\n\x1aANOMALY_TYPE_TERM_ORDERING\x10\x0c\x12,\n(ANOMALY_TYPE_COMPACTION_STATE_UNEXPECTED\x10\r2\xec\x08\n\x10MetastoreService\x12\x97\x01\n\nGetOffsets\x12?.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsRequest\x1a@.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x8e\x01\n\x07GetSize\x12<.redpanda.core.admin.internal.cloud_topics.v1.GetSizeRequest\x1a=.redpanda.core.admin.internal.cloud_topics.v1.GetSizeResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa9\x01\n\x10GetDatabaseStats\x12E.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsRequest\x1aF.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x94\x01\n\tWriteRows\x12>.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsRequest\x1a?.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x91\x01\n\x08ReadRows\x12=.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsRequest\x1a>.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa6\x01\n\x0fListCloudTopics\x12D.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsRequest\x1aE.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xac\x01\n\x11ValidatePartition\x12F.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionRequest\x1aG.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionResponse"\x06\xea\x92\x19\x02\x10\x03B\x1b\xea\x92\x19\x17proto::admin::metastoreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\nBproto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto\x12,redpanda.core.admin.internal.cloud_topics.v1\x1a\'proto/redpanda/core/pbgen/options.proto\x1a#proto/redpanda/core/pbgen/rpc.proto\x1a\'proto/redpanda/core/common/v1/ntp.proto"O\n\x11GetOffsetsRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"\\\n\x12GetOffsetsResponse\x12F\n\x07offsets\x18\x02 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.Offsets"4\n\x07Offsets\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03"L\n\x0eGetSizeRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"%\n\x0fGetSizeResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x04"6\n\x17GetDatabaseStatsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r"\xbd\x01\n\x18GetDatabaseStatsResponse\x12\x1d\n\x15active_memtable_bytes\x18\x01 \x01(\x04\x12 \n\x18immutable_memtable_bytes\x18\x02 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x01(\x04\x12F\n\x06levels\x18\x04 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.LsmLevel"f\n\x08LsmLevel\x12\x14\n\x0clevel_number\x18\x01 \x01(\x05\x12D\n\x05files\x18\x02 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.LsmFile"m\n\x07LsmFile\x12\r\n\x05epoch\x18\x01 \x01(\x04\x12\n\n\x02id\x18\x02 \x01(\x04\x12\x12\n\nsize_bytes\x18\x03 \x01(\x04\x12\x19\n\x11smallest_key_info\x18\x04 \x01(\t\x12\x18\n\x10largest_key_info\x18\x05 \x01(\t"5\n\x0bMetadataKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"H\n\tExtentKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x13\n\x0bbase_offset\x18\x03 \x01(\x03"B\n\x07TermKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x0f\n\x07term_id\x18\x03 \x01(\x03"7\n\rCompactionKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"\x1e\n\tObjectKey\x12\x11\n\tobject_id\x18\x01 \x01(\t"\x8e\x03\n\x06RowKey\x12M\n\x08metadata\x18\x01 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.MetadataKeyH\x00\x12I\n\x06extent\x18\x02 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ExtentKeyH\x00\x12E\n\x04term\x18\x03 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.TermKeyH\x00\x12Q\n\ncompaction\x18\x04 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.CompactionKeyH\x00\x12I\n\x06object\x18\x05 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ObjectKeyH\x00B\x05\n\x03key"w\n\rMetadataValue\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x18\n\x10compaction_epoch\x18\x03 \x01(\x03\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x13\n\x0bnum_extents\x18\x05 \x01(\x04"+\n\x18ImportedTsObjectLocation\x12\x0f\n\x07ts_path\x18\x01 \x01(\t"\x9c\x01\n\x15ImportedTsSegmentInfo\x12\x14\n\x0csegment_term\x18\x01 \x01(\x03\x12\x12\n\ndelta_base\x18\x02 \x01(\x03\x12Y\n\x08tx_state\x18\x03 \x01(\x0e2G.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsTxManifestState"\xe3\x01\n\x0bExtentValue\x12\x13\n\x0blast_offset\x18\x01 \x01(\x03\x12\x15\n\rmax_timestamp\x18\x02 \x01(\x03\x12\x0f\n\x07filepos\x18\x03 \x01(\x04\x12\x0b\n\x03len\x18\x04 \x01(\x04\x12\x11\n\tobject_id\x18\x05 \x01(\t\x12b\n\x10imported_ts_info\x18\x06 \x01(\x0b2C.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsSegmentInfoH\x00\x88\x01\x01B\x13\n\x11_imported_ts_info"&\n\tTermValue\x12\x19\n\x11term_start_offset\x18\x01 \x01(\x03"7\n\x0bOffsetRange\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"j\n\x1aCleanedRangeWithTombstones\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12"\n\x1acleaned_with_tombstones_at\x18\x03 \x01(\x03"\xd6\x01\n\x0fCompactionValue\x12Q\n\x0ecleaned_ranges\x18\x01 \x03(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.OffsetRange\x12p\n\x1ecleaned_ranges_with_tombstones\x18\x02 \x03(\x0b2H.redpanda.core.admin.internal.cloud_topics.v1.CleanedRangeWithTombstones"\xa0\x02\n\x0bObjectValue\x12\x17\n\x0ftotal_data_size\x18\x01 \x01(\x04\x12\x19\n\x11removed_data_size\x18\x02 \x01(\x04\x12\x12\n\nfooter_pos\x18\x03 \x01(\x04\x12\x13\n\x0bobject_size\x18\x04 \x01(\x04\x12\x14\n\x0clast_updated\x18\x05 \x01(\x03\x12\x1a\n\x12is_preregistration\x18\x06 \x01(\x08\x12i\n\x14imported_ts_location\x18\x07 \x01(\x0b2F.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsObjectLocationH\x00\x88\x01\x01B\x17\n\x15_imported_ts_location"\x9c\x03\n\x08RowValue\x12O\n\x08metadata\x18\x01 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.MetadataValueH\x00\x12K\n\x06extent\x18\x02 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ExtentValueH\x00\x12G\n\x04term\x18\x03 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.TermValueH\x00\x12S\n\ncompaction\x18\x04 \x01(\x0b2=.redpanda.core.admin.internal.cloud_topics.v1.CompactionValueH\x00\x12K\n\x06object\x18\x05 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ObjectValueH\x00B\x07\n\x05value"\x94\x01\n\x08WriteRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\xbe\x01\n\x10WriteRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12F\n\x06writes\x18\x02 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.WriteRow\x12E\n\x07deletes\x18\x03 \x03(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey")\n\x11WriteRowsResponse\x12\x14\n\x0crows_written\x18\x01 \x01(\r"\x93\x01\n\x07ReadRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\x94\x02\n\x0fReadRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12H\n\x08seek_key\x18\x02 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x00\x12\x16\n\x0craw_seek_key\x18\x03 \x01(\tH\x00\x12H\n\x08last_key\x18\x04 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x01\x12\x16\n\x0craw_last_key\x18\x05 \x01(\tH\x01\x12\x10\n\x08max_rows\x18\x06 \x01(\rB\x06\n\x04seekB\x06\n\x04last"i\n\x10ReadRowsResponse\x12C\n\x04rows\x18\x01 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.ReadRow\x12\x10\n\x08next_key\x18\x02 \x01(\t"\xc8\x01\n\x18ValidatePartitionRequest\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x1d\n\x15check_object_metadata\x18\x03 \x01(\x08\x12\x1c\n\x14check_object_storage\x18\x04 \x01(\x08\x12\x1d\n\x10resume_at_offset\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x13\n\x0bmax_extents\x18\x06 \x01(\rB\x13\n\x11_resume_at_offset"\xbd\x01\n\x19ValidatePartitionResponse\x12Q\n\tanomalies\x18\x01 \x03(\x0b2>.redpanda.core.admin.internal.cloud_topics.v1.MetastoreAnomaly\x12\x1d\n\x10resume_at_offset\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11extents_validated\x18\x03 \x01(\rB\x13\n\x11_resume_at_offset"x\n\x10MetastoreAnomaly\x12O\n\x0canomaly_type\x18\x01 \x01(\x0e29.redpanda.core.admin.internal.cloud_topics.v1.AnomalyType\x12\x13\n\x0bdescription\x18\x02 \x01(\t"F\n\x16ListCloudTopicsRequest\x12\x18\n\x10after_topic_name\x18\x01 \x01(\t\x12\x12\n\nmax_topics\x18\x02 \x01(\r"y\n\x17ListCloudTopicsResponse\x12L\n\x06topics\x18\x01 \x03(\x0b2<.redpanda.core.admin.internal.cloud_topics.v1.CloudTopicInfo\x12\x10\n\x08has_more\x18\x02 \x01(\x08"O\n\x0eCloudTopicInfo\x12\x12\n\ntopic_name\x18\x01 \x01(\t\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x17\n\x0fpartition_count\x18\x03 \x01(\x05*\x9b\x01\n\x19ImportedTsTxManifestState\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN\x10\x00\x12(\n$IMPORTED_TS_TX_MANIFEST_STATE_ABSENT\x10\x01\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_PRESENT\x10\x02*\xbb\x04\n\x0bAnomalyType\x12\x1c\n\x18ANOMALY_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bANOMALY_TYPE_EXTENT_OVERLAP\x10\x01\x12\x1b\n\x17ANOMALY_TYPE_EXTENT_GAP\x10\x02\x12%\n!ANOMALY_TYPE_NEXT_OFFSET_MISMATCH\x10\x03\x12&\n"ANOMALY_TYPE_START_OFFSET_MISMATCH\x10\x04\x12!\n\x1dANOMALY_TYPE_OBJECT_NOT_FOUND\x10\x05\x12%\n!ANOMALY_TYPE_OBJECT_PREREGISTERED\x10\x06\x12&\n"ANOMALY_TYPE_OBJECT_NOT_IN_STORAGE\x10\x07\x12-\n)ANOMALY_TYPE_COMPACTION_RANGE_BELOW_START\x10\x08\x12,\n(ANOMALY_TYPE_COMPACTION_RANGE_ABOVE_NEXT\x10\t\x12-\n)ANOMALY_TYPE_COMPACTION_TOMBSTONE_OVERLAP\x10\n\x125\n1ANOMALY_TYPE_COMPACTION_TOMBSTONE_OUTSIDE_CLEANED\x10\x0b\x12\x1e\n\x1aANOMALY_TYPE_TERM_ORDERING\x10\x0c\x12,\n(ANOMALY_TYPE_COMPACTION_STATE_UNEXPECTED\x10\r2\xec\x08\n\x10MetastoreService\x12\x97\x01\n\nGetOffsets\x12?.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsRequest\x1a@.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x8e\x01\n\x07GetSize\x12<.redpanda.core.admin.internal.cloud_topics.v1.GetSizeRequest\x1a=.redpanda.core.admin.internal.cloud_topics.v1.GetSizeResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa9\x01\n\x10GetDatabaseStats\x12E.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsRequest\x1aF.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x94\x01\n\tWriteRows\x12>.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsRequest\x1a?.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x91\x01\n\x08ReadRows\x12=.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsRequest\x1a>.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa6\x01\n\x0fListCloudTopics\x12D.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsRequest\x1aE.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xac\x01\n\x11ValidatePartition\x12F.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionRequest\x1aG.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionResponse"\x06\xea\x92\x19\x02\x10\x03B\x1b\xea\x92\x19\x17proto::admin::metastoreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'proto.redpanda.core.admin.internal.cloud_topics.v1.metastore_pb2', _globals) @@ -30,8 +30,10 @@ _globals['_METASTORESERVICE'].methods_by_name['ListCloudTopics']._serialized_options = b'\xea\x92\x19\x02\x10\x03' _globals['_METASTORESERVICE'].methods_by_name['ValidatePartition']._loaded_options = None _globals['_METASTORESERVICE'].methods_by_name['ValidatePartition']._serialized_options = b'\xea\x92\x19\x02\x10\x03' - _globals['_ANOMALYTYPE']._serialized_start = 4673 - _globals['_ANOMALYTYPE']._serialized_end = 5244 + _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_start = 5131 + _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_end = 5286 + _globals['_ANOMALYTYPE']._serialized_start = 5289 + _globals['_ANOMALYTYPE']._serialized_end = 5860 _globals['_GETOFFSETSREQUEST']._serialized_start = 235 _globals['_GETOFFSETSREQUEST']._serialized_end = 314 _globals['_GETOFFSETSRESPONSE']._serialized_start = 316 @@ -64,43 +66,47 @@ _globals['_ROWKEY']._serialized_end = 1729 _globals['_METADATAVALUE']._serialized_start = 1731 _globals['_METADATAVALUE']._serialized_end = 1850 - _globals['_EXTENTVALUE']._serialized_start = 1852 - _globals['_EXTENTVALUE']._serialized_end = 1958 - _globals['_TERMVALUE']._serialized_start = 1960 - _globals['_TERMVALUE']._serialized_end = 1998 - _globals['_OFFSETRANGE']._serialized_start = 2000 - _globals['_OFFSETRANGE']._serialized_end = 2055 - _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_start = 2057 - _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_end = 2163 - _globals['_COMPACTIONVALUE']._serialized_start = 2166 - _globals['_COMPACTIONVALUE']._serialized_end = 2380 - _globals['_OBJECTVALUE']._serialized_start = 2383 - _globals['_OBJECTVALUE']._serialized_end = 2539 - _globals['_ROWVALUE']._serialized_start = 2542 - _globals['_ROWVALUE']._serialized_end = 2954 - _globals['_WRITEROW']._serialized_start = 2957 - _globals['_WRITEROW']._serialized_end = 3105 - _globals['_WRITEROWSREQUEST']._serialized_start = 3108 - _globals['_WRITEROWSREQUEST']._serialized_end = 3298 - _globals['_WRITEROWSRESPONSE']._serialized_start = 3300 - _globals['_WRITEROWSRESPONSE']._serialized_end = 3341 - _globals['_READROW']._serialized_start = 3344 - _globals['_READROW']._serialized_end = 3491 - _globals['_READROWSREQUEST']._serialized_start = 3494 - _globals['_READROWSREQUEST']._serialized_end = 3770 - _globals['_READROWSRESPONSE']._serialized_start = 3772 - _globals['_READROWSRESPONSE']._serialized_end = 3877 - _globals['_VALIDATEPARTITIONREQUEST']._serialized_start = 3880 - _globals['_VALIDATEPARTITIONREQUEST']._serialized_end = 4080 - _globals['_VALIDATEPARTITIONRESPONSE']._serialized_start = 4083 - _globals['_VALIDATEPARTITIONRESPONSE']._serialized_end = 4272 - _globals['_METASTOREANOMALY']._serialized_start = 4274 - _globals['_METASTOREANOMALY']._serialized_end = 4394 - _globals['_LISTCLOUDTOPICSREQUEST']._serialized_start = 4396 - _globals['_LISTCLOUDTOPICSREQUEST']._serialized_end = 4466 - _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_start = 4468 - _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_end = 4589 - _globals['_CLOUDTOPICINFO']._serialized_start = 4591 - _globals['_CLOUDTOPICINFO']._serialized_end = 4670 - _globals['_METASTORESERVICE']._serialized_start = 5247 - _globals['_METASTORESERVICE']._serialized_end = 6379 \ No newline at end of file + _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_start = 1852 + _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_end = 1895 + _globals['_IMPORTEDTSSEGMENTINFO']._serialized_start = 1898 + _globals['_IMPORTEDTSSEGMENTINFO']._serialized_end = 2054 + _globals['_EXTENTVALUE']._serialized_start = 2057 + _globals['_EXTENTVALUE']._serialized_end = 2284 + _globals['_TERMVALUE']._serialized_start = 2286 + _globals['_TERMVALUE']._serialized_end = 2324 + _globals['_OFFSETRANGE']._serialized_start = 2326 + _globals['_OFFSETRANGE']._serialized_end = 2381 + _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_start = 2383 + _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_end = 2489 + _globals['_COMPACTIONVALUE']._serialized_start = 2492 + _globals['_COMPACTIONVALUE']._serialized_end = 2706 + _globals['_OBJECTVALUE']._serialized_start = 2709 + _globals['_OBJECTVALUE']._serialized_end = 2997 + _globals['_ROWVALUE']._serialized_start = 3000 + _globals['_ROWVALUE']._serialized_end = 3412 + _globals['_WRITEROW']._serialized_start = 3415 + _globals['_WRITEROW']._serialized_end = 3563 + _globals['_WRITEROWSREQUEST']._serialized_start = 3566 + _globals['_WRITEROWSREQUEST']._serialized_end = 3756 + _globals['_WRITEROWSRESPONSE']._serialized_start = 3758 + _globals['_WRITEROWSRESPONSE']._serialized_end = 3799 + _globals['_READROW']._serialized_start = 3802 + _globals['_READROW']._serialized_end = 3949 + _globals['_READROWSREQUEST']._serialized_start = 3952 + _globals['_READROWSREQUEST']._serialized_end = 4228 + _globals['_READROWSRESPONSE']._serialized_start = 4230 + _globals['_READROWSRESPONSE']._serialized_end = 4335 + _globals['_VALIDATEPARTITIONREQUEST']._serialized_start = 4338 + _globals['_VALIDATEPARTITIONREQUEST']._serialized_end = 4538 + _globals['_VALIDATEPARTITIONRESPONSE']._serialized_start = 4541 + _globals['_VALIDATEPARTITIONRESPONSE']._serialized_end = 4730 + _globals['_METASTOREANOMALY']._serialized_start = 4732 + _globals['_METASTOREANOMALY']._serialized_end = 4852 + _globals['_LISTCLOUDTOPICSREQUEST']._serialized_start = 4854 + _globals['_LISTCLOUDTOPICSREQUEST']._serialized_end = 4924 + _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_start = 4926 + _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_end = 5047 + _globals['_CLOUDTOPICINFO']._serialized_start = 5049 + _globals['_CLOUDTOPICINFO']._serialized_end = 5128 + _globals['_METASTORESERVICE']._serialized_start = 5863 + _globals['_METASTORESERVICE']._serialized_end = 6995 \ No newline at end of file diff --git a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi index 4a263371e1583..4dfead507708e 100644 --- a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi +++ b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi @@ -30,6 +30,26 @@ else: import typing_extensions DESCRIPTOR: google.protobuf.descriptor.FileDescriptor +class _ImportedTsTxManifestState: + ValueType = typing.NewType('ValueType', builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _ImportedTsTxManifestStateEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[_ImportedTsTxManifestState.ValueType], builtins.type): + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN: _ImportedTsTxManifestState.ValueType + IMPORTED_TS_TX_MANIFEST_STATE_ABSENT: _ImportedTsTxManifestState.ValueType + IMPORTED_TS_TX_MANIFEST_STATE_PRESENT: _ImportedTsTxManifestState.ValueType + +class ImportedTsTxManifestState(_ImportedTsTxManifestState, metaclass=_ImportedTsTxManifestStateEnumTypeWrapper): + """Whether an imported tiered-storage segment has an aborted-transaction (.tx) + manifest, resolved at import time so the read path need not probe object + storage. UNKNOWN (v1/v2 non-compacted) => probe and tolerate a missing .tx. + """ +IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN: ImportedTsTxManifestState.ValueType +IMPORTED_TS_TX_MANIFEST_STATE_ABSENT: ImportedTsTxManifestState.ValueType +IMPORTED_TS_TX_MANIFEST_STATE_PRESENT: ImportedTsTxManifestState.ValueType +Global___ImportedTsTxManifestState: typing_extensions.TypeAlias = ImportedTsTxManifestState + class _AnomalyType: ValueType = typing.NewType('ValueType', builtins.int) V: typing_extensions.TypeAlias = ValueType @@ -403,6 +423,44 @@ class MetadataValue(google.protobuf.message.Message): ... Global___MetadataValue: typing_extensions.TypeAlias = MetadataValue +@typing.final +class ImportedTsObjectLocation(google.protobuf.message.Message): + """Where the backing tiered-storage segment of an imported object lives. An + object property: the imported analog of a native object's id->bucket path. + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + TS_PATH_FIELD_NUMBER: builtins.int + ts_path: builtins.str + + def __init__(self, *, ts_path: builtins.str=...) -> None: + ... + + def ClearField(self, field_name: typing.Literal['ts_path', b'ts_path']) -> None: + ... +Global___ImportedTsObjectLocation: typing_extensions.TypeAlias = ImportedTsObjectLocation + +@typing.final +class ImportedTsSegmentInfo(google.protobuf.message.Message): + """Properties of the data in an imported tiered-storage extent. An extent + property, alongside the extent's Kafka offset bounds (base/last_offset). + """ + DESCRIPTOR: google.protobuf.descriptor.Descriptor + SEGMENT_TERM_FIELD_NUMBER: builtins.int + DELTA_BASE_FIELD_NUMBER: builtins.int + TX_STATE_FIELD_NUMBER: builtins.int + segment_term: builtins.int + delta_base: builtins.int + "Offset-translation delta at the segment's base (source segment_meta's\n delta_offset).\n " + tx_state: Global___ImportedTsTxManifestState.ValueType + "Whether the segment's .tx manifest exists (resolved at import time)." + + def __init__(self, *, segment_term: builtins.int=..., delta_base: builtins.int=..., tx_state: Global___ImportedTsTxManifestState.ValueType=...) -> None: + ... + + def ClearField(self, field_name: typing.Literal['delta_base', b'delta_base', 'segment_term', b'segment_term', 'tx_state', b'tx_state']) -> None: + ... +Global___ImportedTsSegmentInfo: typing_extensions.TypeAlias = ImportedTsSegmentInfo + @typing.final class ExtentValue(google.protobuf.message.Message): """Value for an extent row.""" @@ -412,16 +470,27 @@ class ExtentValue(google.protobuf.message.Message): FILEPOS_FIELD_NUMBER: builtins.int LEN_FIELD_NUMBER: builtins.int OBJECT_ID_FIELD_NUMBER: builtins.int + IMPORTED_TS_INFO_FIELD_NUMBER: builtins.int last_offset: builtins.int max_timestamp: builtins.int filepos: builtins.int len: builtins.int object_id: builtins.str - def __init__(self, *, last_offset: builtins.int=..., max_timestamp: builtins.int=..., filepos: builtins.int=..., len: builtins.int=..., object_id: builtins.str=...) -> None: + @property + def imported_ts_info(self) -> Global___ImportedTsSegmentInfo: + ... + + def __init__(self, *, last_offset: builtins.int=..., max_timestamp: builtins.int=..., filepos: builtins.int=..., len: builtins.int=..., object_id: builtins.str=..., imported_ts_info: Global___ImportedTsSegmentInfo | None=...) -> None: + ... + + def HasField(self, field_name: typing.Literal['_imported_ts_info', b'_imported_ts_info', 'imported_ts_info', b'imported_ts_info']) -> builtins.bool: ... - def ClearField(self, field_name: typing.Literal['filepos', b'filepos', 'last_offset', b'last_offset', 'len', b'len', 'max_timestamp', b'max_timestamp', 'object_id', b'object_id']) -> None: + def ClearField(self, field_name: typing.Literal['_imported_ts_info', b'_imported_ts_info', 'filepos', b'filepos', 'imported_ts_info', b'imported_ts_info', 'last_offset', b'last_offset', 'len', b'len', 'max_timestamp', b'max_timestamp', 'object_id', b'object_id']) -> None: + ... + + def WhichOneof(self, oneof_group: typing.Literal['_imported_ts_info', b'_imported_ts_info']) -> typing.Literal['imported_ts_info'] | None: ... Global___ExtentValue: typing_extensions.TypeAlias = ExtentValue @@ -505,6 +574,7 @@ class ObjectValue(google.protobuf.message.Message): OBJECT_SIZE_FIELD_NUMBER: builtins.int LAST_UPDATED_FIELD_NUMBER: builtins.int IS_PREREGISTRATION_FIELD_NUMBER: builtins.int + IMPORTED_TS_LOCATION_FIELD_NUMBER: builtins.int total_data_size: builtins.int removed_data_size: builtins.int footer_pos: builtins.int @@ -512,10 +582,20 @@ class ObjectValue(google.protobuf.message.Message): last_updated: builtins.int is_preregistration: builtins.bool - def __init__(self, *, total_data_size: builtins.int=..., removed_data_size: builtins.int=..., footer_pos: builtins.int=..., object_size: builtins.int=..., last_updated: builtins.int=..., is_preregistration: builtins.bool=...) -> None: + @property + def imported_ts_location(self) -> Global___ImportedTsObjectLocation: + ... + + def __init__(self, *, total_data_size: builtins.int=..., removed_data_size: builtins.int=..., footer_pos: builtins.int=..., object_size: builtins.int=..., last_updated: builtins.int=..., is_preregistration: builtins.bool=..., imported_ts_location: Global___ImportedTsObjectLocation | None=...) -> None: + ... + + def HasField(self, field_name: typing.Literal['_imported_ts_location', b'_imported_ts_location', 'imported_ts_location', b'imported_ts_location']) -> builtins.bool: + ... + + def ClearField(self, field_name: typing.Literal['_imported_ts_location', b'_imported_ts_location', 'footer_pos', b'footer_pos', 'imported_ts_location', b'imported_ts_location', 'is_preregistration', b'is_preregistration', 'last_updated', b'last_updated', 'object_size', b'object_size', 'removed_data_size', b'removed_data_size', 'total_data_size', b'total_data_size']) -> None: ... - def ClearField(self, field_name: typing.Literal['footer_pos', b'footer_pos', 'is_preregistration', b'is_preregistration', 'last_updated', b'last_updated', 'object_size', b'object_size', 'removed_data_size', b'removed_data_size', 'total_data_size', b'total_data_size']) -> None: + def WhichOneof(self, oneof_group: typing.Literal['_imported_ts_location', b'_imported_ts_location']) -> typing.Literal['imported_ts_location'] | None: ... Global___ObjectValue: typing_extensions.TypeAlias = ObjectValue From aee815e8c7c1bac48cfec773a28b5698a7204d3c Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Fri, 5 Jun 2026 15:47:05 -0700 Subject: [PATCH 3/9] cloud_topics/l1: add a per-partition migrating flag to the metastore Add a durable, per-partition boolean migrating to each backend's partition record, set true while a partition is mid tiered->cloud migration. - state: add the field to partition_state (serde v0->v1) and the LSM metadata_row_value (v1->v2), defaulting to false so snapshots predating the field decode as native cloud topics; carry it through the MetadataValue debug proto. - op: add set_migrating(partition, bool) across the metastore interface, the RPC service/client, the leader router (+probe), both domain managers, and both backends. It is idempotent (setting the current value is a no-op), applies in either direction, and creates the partition record when set on an absent partition; metadata-row rewrites (add_objects, set_start_offset) preserve the flag. - read: surface the flag on get_offsets. Tests (both backends): create-on-absent, set-then-clear, idempotent set, and preservation across add_objects; serde back-compat for both records. Co-Authored-By: Claude Opus 4.8 --- .../internal/cloud_topics/v1/metastore.proto | 2 + .../level_one/domain/db_domain_manager.cc | 39 ++++++ .../level_one/domain/db_domain_manager.h | 3 + .../level_one/domain/domain_manager.h | 3 + .../level_one/domain/simple_domain_manager.cc | 43 +++++++ .../level_one/domain/simple_domain_manager.h | 3 + src/v/cloud_topics/level_one/metastore/BUILD | 2 + .../level_one/metastore/leader_router.cc | 39 ++++++ .../level_one/metastore/leader_router.h | 8 ++ .../metastore/leader_router_probe.cc | 4 + .../level_one/metastore/leader_router_probe.h | 5 + .../level_one/metastore/lsm/debug_reader.cc | 1 + .../level_one/metastore/lsm/debug_serde.cc | 1 + .../level_one/metastore/lsm/state_update.cc | 35 ++++++ .../level_one/metastore/lsm/state_update.h | 13 ++ .../metastore/lsm/tests/debug_reader_test.cc | 22 ++++ .../level_one/metastore/lsm/values.h | 12 +- .../level_one/metastore/metastore.h | 10 ++ .../metastore/replicated_metastore.cc | 22 ++++ .../metastore/replicated_metastore.h | 3 + .../cloud_topics/level_one/metastore/rpc.json | 5 + .../level_one/metastore/rpc_types.h | 28 ++++- .../level_one/metastore/service.cc | 6 + .../level_one/metastore/service.h | 3 + .../level_one/metastore/simple_metastore.cc | 17 +++ .../level_one/metastore/simple_metastore.h | 3 + .../level_one/metastore/simple_stm.cc | 6 + .../cloud_topics/level_one/metastore/state.cc | 1 + .../cloud_topics/level_one/metastore/state.h | 12 +- .../level_one/metastore/state_update.cc | 45 +++++++ .../level_one/metastore/state_update.h | 30 +++++ .../metastore/tests/imported_serde_test.cc | 113 ++++++++++++++++++ .../metastore/tests/state_update_test.cc | 81 +++++++++++++ .../read_replica/snapshot_metastore.cc | 6 + .../read_replica/snapshot_metastore.h | 3 + .../internal/cloud_topics/v1/metastore_pb2.py | 102 ++++++++-------- .../cloud_topics/v1/metastore_pb2.pyi | 7 +- 37 files changed, 679 insertions(+), 59 deletions(-) diff --git a/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto b/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto index 41f2905d19e1a..b964e63a76e05 100644 --- a/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto +++ b/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto @@ -237,6 +237,8 @@ message MetadataValue { int64 compaction_epoch = 3; uint64 size = 4; uint64 num_extents = 5; + // True while the partition is mid tiered->cloud migration. + bool migrating = 6; } // Where the backing tiered-storage segment of an imported object lives. An diff --git a/src/v/cloud_topics/level_one/domain/db_domain_manager.cc b/src/v/cloud_topics/level_one/domain/db_domain_manager.cc index eadcd2b7f1828..c6f004c71b692 100644 --- a/src/v/cloud_topics/level_one/domain/db_domain_manager.cc +++ b/src/v/cloud_topics/level_one/domain/db_domain_manager.cc @@ -743,6 +743,7 @@ db_domain_manager::get_offsets(rpc::get_offsets_request req) { .ec = rpc::errc::ok, .start_offset = metadata.start_offset, .next_offset = metadata.next_offset, + .migrating = metadata.migrating, }; } @@ -1141,6 +1142,44 @@ db_domain_manager::set_start_offset(rpc::set_start_offset_request req) { co_return co_await do_set_start_offset(locks_res.value(), req); } +ss::future +db_domain_manager::set_migrating(rpc::set_migrating_request req) { + auto locks_res = co_await gate_and_open_writes({ + .topic_read_locks = {req.tp.topic_id}, + .partition_locks = {req.tp}, + }); + if (!locks_res.has_value()) { + co_return rpc::set_migrating_reply{ + .ec = locks_res.error(), + }; + } + + auto reader = state_reader(db_->db().create_snapshot()); + auto update = set_migrating_db_update{ + .tp = req.tp, + .migrating = req.migrating, + }; + chunked_vector rows; + bool is_no_op = false; + auto build_res = co_await update.build_rows(reader, rows, &is_no_op); + if (!build_res.has_value()) { + co_return rpc::set_migrating_reply{ + .ec = log_and_convert( + build_res.error(), "Rejecting request to set migration phase: "), + }; + } + if (is_no_op) { + co_return rpc::set_migrating_reply{.ec = rpc::errc::ok}; + } + auto apply_res = co_await write_rows(locks_res.value(), std::move(rows)); + if (!apply_res.has_value()) { + co_return rpc::set_migrating_reply{ + .ec = apply_res.error(), + }; + } + co_return rpc::set_migrating_reply{.ec = rpc::errc::ok}; +} + ss::future< std::expected> db_domain_manager::set_partitions_empty( diff --git a/src/v/cloud_topics/level_one/domain/db_domain_manager.h b/src/v/cloud_topics/level_one/domain/db_domain_manager.h index eb73b76a30775..b7d853040caaf 100644 --- a/src/v/cloud_topics/level_one/domain/db_domain_manager.h +++ b/src/v/cloud_topics/level_one/domain/db_domain_manager.h @@ -83,6 +83,9 @@ class db_domain_manager final : public domain_manager { ss::future set_start_offset(rpc::set_start_offset_request) override; + ss::future + set_migrating(rpc::set_migrating_request) override; + ss::future remove_topics(rpc::remove_topics_request) override; diff --git a/src/v/cloud_topics/level_one/domain/domain_manager.h b/src/v/cloud_topics/level_one/domain/domain_manager.h index 0578965347e21..75ef9a5ae714d 100644 --- a/src/v/cloud_topics/level_one/domain/domain_manager.h +++ b/src/v/cloud_topics/level_one/domain/domain_manager.h @@ -91,6 +91,9 @@ class domain_manager { virtual ss::future set_start_offset(rpc::set_start_offset_request) = 0; + virtual ss::future + set_migrating(rpc::set_migrating_request) = 0; + virtual ss::future remove_topics(rpc::remove_topics_request) = 0; diff --git a/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc b/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc index 636e0a52e7c06..89d7dcf6431b6 100644 --- a/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc +++ b/src/v/cloud_topics/level_one/domain/simple_domain_manager.cc @@ -421,6 +421,7 @@ simple_domain_manager::get_offsets(rpc::get_offsets_request req) { .ec = rpc::errc::ok, .start_offset = get_res->start_offset, .next_offset = get_res->next_offset, + .migrating = get_res->migrating, }; } @@ -606,6 +607,48 @@ simple_domain_manager::set_start_offset(rpc::set_start_offset_request req) { co_return rpc::set_start_offset_reply{.ec = rpc::errc::ok}; } +ss::future +simple_domain_manager::set_migrating(rpc::set_migrating_request req) { + auto gate = maybe_gate(); + if (!gate.has_value()) { + co_return rpc::set_migrating_reply{.ec = rpc::errc::not_leader}; + } + + auto sync_res = co_await stm_->sync(10s); + if (!sync_res.has_value()) { + co_return rpc::set_migrating_reply{ + .ec = convert_stm_errc(sync_res.error())}; + } + bool is_no_op = false; + auto update_res = set_migrating_update::build( + stm_->state(), req.tp, req.migrating, &is_no_op); + if (!update_res.has_value()) { + vlog( + cd_log.debug, + "Rejecting request to set migration phase: {}", + update_res.error()); + co_return rpc::set_migrating_reply{ + .ec = rpc::errc::concurrent_requests, + }; + } + if (is_no_op) { + co_return rpc::set_migrating_reply{.ec = rpc::errc::ok}; + } + storage::record_batch_builder builder( + model::record_batch_type::l1_stm, model::offset{0}); + builder.add_raw_kv( + serde::to_iobuf(set_migrating_update::key), + serde::to_iobuf(std::move(update_res.value()))); + auto repl_res = co_await stm_->replicate_and_wait( + sync_res.value(), std::move(builder).build(), as_); + if (!repl_res.has_value()) { + co_return rpc::set_migrating_reply{ + .ec = convert_stm_errc(repl_res.error()), + }; + } + co_return rpc::set_migrating_reply{.ec = rpc::errc::ok}; +} + ss::future simple_domain_manager::remove_topics(rpc::remove_topics_request req) { auto gate = maybe_gate(); diff --git a/src/v/cloud_topics/level_one/domain/simple_domain_manager.h b/src/v/cloud_topics/level_one/domain/simple_domain_manager.h index 84b4b05b86357..9816aa5ceae32 100644 --- a/src/v/cloud_topics/level_one/domain/simple_domain_manager.h +++ b/src/v/cloud_topics/level_one/domain/simple_domain_manager.h @@ -66,6 +66,9 @@ class simple_domain_manager final : public domain_manager { ss::future set_start_offset(rpc::set_start_offset_request) override; + ss::future + set_migrating(rpc::set_migrating_request) override; + ss::future remove_topics(rpc::remove_topics_request) override; diff --git a/src/v/cloud_topics/level_one/metastore/BUILD b/src/v/cloud_topics/level_one/metastore/BUILD index cfbd29b09cadc..382cc8a85b374 100644 --- a/src/v/cloud_topics/level_one/metastore/BUILD +++ b/src/v/cloud_topics/level_one/metastore/BUILD @@ -165,6 +165,7 @@ redpanda_cc_library( ":leveling_range_builder", ":metastore_manifest", ":offset_interval_set", + ":state", "//src/v/base", "//src/v/cloud_topics/level_one/common:object_id", "//src/v/container:chunked_hash_map", @@ -301,6 +302,7 @@ redpanda_cc_library( "//src/v/container:chunked_hash_map", "//src/v/model", "//src/v/serde", + "//src/v/serde:enum", "//src/v/serde:map", "//src/v/serde:named_type", "//src/v/serde:optional", diff --git a/src/v/cloud_topics/level_one/metastore/leader_router.cc b/src/v/cloud_topics/level_one/metastore/leader_router.cc index e635308453c3b..e077995fd26dc 100644 --- a/src/v/cloud_topics/level_one/metastore/leader_router.cc +++ b/src/v/cloud_topics/level_one/metastore/leader_router.cc @@ -162,6 +162,17 @@ ss::future do_set_start_offset( co_return co_await domain_mgr->set_start_offset(std::move(req)); } +ss::future do_set_migrating( + domain_supervisor& domain_supervisor, + const model::ntp& ntp, + rpc::set_migrating_request req) { + auto domain_mgr = domain_supervisor.get(ntp); + if (!domain_mgr) { + co_return rpc::set_migrating_reply{.ec = rpc::errc::not_leader}; + } + co_return co_await domain_mgr->set_migrating(std::move(req)); +} + ss::future do_remove_topics( domain_supervisor& domain_supervisor, const model::ntp& ntp, @@ -438,6 +449,13 @@ template ss::future leader_router::process< &leader_router::client::set_start_offset>( rpc::set_start_offset_request, bool); +template ss::future + leader_router::remote_dispatch<&leader_router::client::set_migrating>( + rpc::set_migrating_request, model::node_id); +template ss::future leader_router::process< + &leader_router::set_migrating_locally, + &leader_router::client::set_migrating>(rpc::set_migrating_request, bool); + template ss::future leader_router::remote_dispatch<&leader_router::client::flush_domain>( rpc::flush_domain_request, model::node_id); @@ -777,6 +795,27 @@ ss::future leader_router::set_start_offset( &client::set_start_offset>(std::move(request), bool(local_only_exec)); } +ss::future leader_router::set_migrating_locally( + rpc::set_migrating_request request, + const model::ntp& metastore_ntp, + ss::shard_id shard) { + auto m = _probe.auto_measure_set_migrating(); + co_return co_await container().invoke_on( + shard, + [&metastore_ntp, req = std::move(request)](leader_router& fe) mutable { + return do_set_migrating( + *(fe._domain_supervisor), metastore_ntp, std::move(req)); + }); +} + +ss::future leader_router::set_migrating( + rpc::set_migrating_request request, local_only local_only_exec) { + auto holder = _gate.hold(); + co_return co_await process< + &leader_router::set_migrating_locally, + &client::set_migrating>(std::move(request), bool(local_only_exec)); +} + ss::future leader_router::remove_topics_locally( rpc::remove_topics_request request, const model::ntp& metastore_ntp, diff --git a/src/v/cloud_topics/level_one/metastore/leader_router.h b/src/v/cloud_topics/level_one/metastore/leader_router.h index 095ae3e3a2554..e865f21af977c 100644 --- a/src/v/cloud_topics/level_one/metastore/leader_router.h +++ b/src/v/cloud_topics/level_one/metastore/leader_router.h @@ -94,6 +94,9 @@ class leader_router : public ss::peering_sharded_service { ss::future set_start_offset( rpc::set_start_offset_request, local_only = local_only::no); + ss::future + set_migrating(rpc::set_migrating_request, local_only = local_only::no); + ss::future remove_topics(rpc::remove_topics_request, local_only = local_only::no); @@ -211,6 +214,11 @@ class leader_router : public ss::peering_sharded_service { const model::ntp& metastore_ntp, ss::shard_id); + ss::future set_migrating_locally( + rpc::set_migrating_request, + const model::ntp& metastore_ntp, + ss::shard_id); + ss::future remove_topics_locally( rpc::remove_topics_request, const model::ntp& metastore_ntp, diff --git a/src/v/cloud_topics/level_one/metastore/leader_router_probe.cc b/src/v/cloud_topics/level_one/metastore/leader_router_probe.cc index 5aee0af7fa238..bd4123ee3a271 100644 --- a/src/v/cloud_topics/level_one/metastore/leader_router_probe.cc +++ b/src/v/cloud_topics/level_one/metastore/leader_router_probe.cc @@ -80,6 +80,10 @@ void leader_router_probe::setup_metrics() { "set_start_offset_duration_microseconds", [this] { return _set_start_offset.internal_histogram_logform(); }, sm::description("Latency of local set_start_offset requests")), + sm::make_histogram( + "set_migrating_duration_microseconds", + [this] { return _set_migrating.internal_histogram_logform(); }, + sm::description("Latency of local set_migrating requests")), sm::make_histogram( "remove_topics_duration_microseconds", [this] { return _remove_topics.internal_histogram_logform(); }, diff --git a/src/v/cloud_topics/level_one/metastore/leader_router_probe.h b/src/v/cloud_topics/level_one/metastore/leader_router_probe.h index 9ffdae028f63b..d742be347e149 100644 --- a/src/v/cloud_topics/level_one/metastore/leader_router_probe.h +++ b/src/v/cloud_topics/level_one/metastore/leader_router_probe.h @@ -82,6 +82,10 @@ class leader_router_probe { return _set_start_offset.auto_measure(); } + std::unique_ptr auto_measure_set_migrating() { + return _set_migrating.auto_measure(); + } + std::unique_ptr auto_measure_remove_topics() { return _remove_topics.auto_measure(); } @@ -119,6 +123,7 @@ class leader_router_probe { hist_t _get_term_for_offset; hist_t _get_end_offset_for_term; hist_t _set_start_offset; + hist_t _set_migrating; hist_t _remove_topics; hist_t _get_compaction_infos; hist_t _get_extent_metadata; diff --git a/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc b/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc index ebe50523f1c68..38aa31bd0910a 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/debug_reader.cc @@ -157,6 +157,7 @@ debug_reader::decode_value(row_type type, iobuf value) { mv.set_compaction_epoch(rv.compaction_epoch()); mv.set_size(rv.size); mv.set_num_extents(rv.num_extents); + mv.set_migrating(rv.migrating); pv.set_metadata(std::move(mv)); return pv; } diff --git a/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc b/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc index 035c2d6204629..d5473d16c2b7c 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/debug_serde.cc @@ -144,6 +144,7 @@ debug_encode_value(const proto::admin::metastore::row_value& val) { = partition_state::compaction_epoch_t{v.get_compaction_epoch()}, .size = static_cast(v.get_size()), .num_extents = static_cast(v.get_num_extents()), + .migrating = v.get_migrating(), }; return serde::to_iobuf(std::move(rv)); } diff --git a/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc b/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc index a771fbd04d21d..fca4879a75453 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc @@ -695,6 +695,7 @@ add_objects_db_update::build_rows( : partition_state::compaction_epoch_t{0}, .size = (opt ? opt->size : 0) + extent_size_sum, .num_extents = (opt ? opt->num_extents : 0) + extents.size(), + .migrating = opt ? opt->migrating : false, }; } // Now that we've validated the offsets of our extents, validate the terms @@ -1316,6 +1317,7 @@ set_start_offset_db_update::build_rows( .num_extents = metadata.num_extents - std::min(metadata.num_extents, extent_keys_to_delete.size()), + .migrating = metadata.migrating, }), }); @@ -1375,6 +1377,39 @@ set_start_offset_db_update::discover_truncated_object_ids( co_return discovered_oids; } +ss::future> +set_migrating_db_update::build_rows( + state_reader& reader, + chunked_vector& out, + bool* is_no_op) const { + auto meta_res = co_await reader.get_metadata(tp); + if (!meta_res.has_value()) { + co_return std::unexpected(wrap_read_err( + std::move(meta_res.error()), "Error reading metadata for {}", tp)); + } + // An absent partition defaults to not-migrating; setting it migrating + // creates its metadata row (the migrating marker may be the partition's + // first write). + metadata_row_value updated = meta_res->has_value() ? **meta_res + : metadata_row_value{}; + const auto current = updated.migrating; + + if (migrating == current) { + if (is_no_op) { + *is_no_op = true; + } + co_return std::expected{}; + } + + updated.migrating = migrating; + out.emplace_back( + write_batch_row{ + .key = metadata_row_key::encode(tp), + .value = serde::to_iobuf(updated), + }); + co_return std::expected{}; +} + ss::future> remove_topics_db_update::build_rows( state_reader& reader, chunked_vector& out) const { diff --git a/src/v/cloud_topics/level_one/metastore/lsm/state_update.h b/src/v/cloud_topics/level_one/metastore/lsm/state_update.h index faa0ba28ae412..c1d3a47c20192 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/state_update.h +++ b/src/v/cloud_topics/level_one/metastore/lsm/state_update.h @@ -121,6 +121,19 @@ struct set_start_offset_db_update { kafka::offset new_start_offset; }; +// Sets the partition's `migrating` flag on its metadata row. Setting the +// current value is a no-op. Creates the metadata row (with default offsets) if +// the partition has none yet. +struct set_migrating_db_update { + ss::future> build_rows( + state_reader&, + chunked_vector&, + bool* is_no_op = nullptr) const; + + model::topic_id_partition tp; + bool migrating{}; +}; + struct remove_topics_db_update { ss::future> build_rows(state_reader&, chunked_vector&) const; diff --git a/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc b/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc index 0ce743e30b480..c5595ee25d373 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/tests/debug_reader_test.cc @@ -222,6 +222,28 @@ TEST(DebugReaderTest, RoundTripMetadataValue) { EXPECT_EQ(decoded->get_metadata().get_compaction_epoch(), 1); EXPECT_EQ(decoded->get_metadata().get_size(), 1000); EXPECT_EQ(decoded->get_metadata().get_num_extents(), 42); + EXPECT_FALSE(decoded->get_metadata().get_migrating()); +} + +TEST(DebugReaderTest, RoundTripMetadataValueMigrating) { + pm::row_value val; + pm::metadata_value mv; + mv.set_start_offset(10); + mv.set_next_offset(20); + mv.set_compaction_epoch(1); + mv.set_size(1000); + mv.set_num_extents(42); + mv.set_migrating(true); + val.set_metadata(std::move(mv)); + + auto encoded = debug_encode_value(val); + ASSERT_TRUE(encoded.has_value()); + + auto decoded = debug_reader::decode_value( + row_type::metadata, std::move(*encoded)); + ASSERT_TRUE(decoded.has_value()); + ASSERT_TRUE(decoded->has_metadata()); + EXPECT_TRUE(decoded->get_metadata().get_migrating()); } TEST(DebugReaderTest, RoundTripExtentValue) { diff --git a/src/v/cloud_topics/level_one/metastore/lsm/values.h b/src/v/cloud_topics/level_one/metastore/lsm/values.h index 7ed8aa060add0..f7f02aaacf5ee 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/values.h +++ b/src/v/cloud_topics/level_one/metastore/lsm/values.h @@ -20,11 +20,16 @@ namespace cloud_topics::l1 { struct metadata_row_value : public serde::envelope< metadata_row_value, - serde::version<1>, + serde::version<2>, serde::compat_version<0>> { auto serde_fields() { return std::tie( - start_offset, next_offset, compaction_epoch, size, num_extents); + start_offset, + next_offset, + compaction_epoch, + size, + num_extents, + migrating); } kafka::offset start_offset{}; kafka::offset next_offset{}; @@ -34,6 +39,9 @@ struct metadata_row_value size_t size{0}; // Number of extents in the partition, updated incrementally. size_t num_extents{0}; + // True while mid tiered->cloud migration; for offline/remote consumers. + // Defaults to false for rows predating this field. + bool migrating{}; }; struct extent_row_value diff --git a/src/v/cloud_topics/level_one/metastore/metastore.h b/src/v/cloud_topics/level_one/metastore/metastore.h index 603f9b583b65e..60fb366f77d9e 100644 --- a/src/v/cloud_topics/level_one/metastore/metastore.h +++ b/src/v/cloud_topics/level_one/metastore/metastore.h @@ -14,6 +14,7 @@ #include "cloud_topics/level_one/metastore/leveling_range_builder.h" #include "cloud_topics/level_one/metastore/metastore_manifest.h" #include "cloud_topics/level_one/metastore/offset_interval_set.h" +#include "cloud_topics/level_one/metastore/state.h" #include "container/chunked_hash_map.h" #include "container/chunked_vector.h" #include "model/fundamental.h" @@ -161,6 +162,9 @@ class metastore { struct offsets_response { kafka::offset start_offset; kafka::offset next_offset; + // True while mid tiered->cloud migration; for offline/remote consumers. + // false for a native cloud topic. + bool migrating{}; }; virtual ss::future< std::expected, errc>> @@ -219,6 +223,12 @@ class metastore { virtual ss::future> set_start_offset(const model::topic_id_partition&, kafka::offset) = 0; + // Sets the partition's migration phase. Monotonic (none -> migrating -> + // complete) and idempotent; a backward transition is rejected. Creates the + // partition's metastore entry if absent. + virtual ss::future> + set_migrating(const model::topic_id_partition&, bool) = 0; + struct topic_removal_response { // Topic IDs that were not removed from the metastore and still have // state to be removed in a subsequent attempt. diff --git a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc index 99bb96f9b1257..5d7634f7f9587 100644 --- a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc @@ -358,6 +358,7 @@ replicated_metastore::get_offsets(const model::topic_id_partition& tidp) { metastore::offsets_response resp; resp.start_offset = reply.start_offset; resp.next_offset = reply.next_offset; + resp.migrating = reply.migrating; co_return resp; } @@ -548,6 +549,27 @@ replicated_metastore::set_start_offset( } } +ss::future> +replicated_metastore::set_migrating( + const model::topic_id_partition& tidp, bool migrating) { + rpc::set_migrating_request req; + req.tp = tidp; + req.migrating = migrating; + + auto reply_fut = co_await ss::coroutine::as_future( + fe_.set_migrating(std::move(req))); + if (reply_fut.failed()) { + auto ex = reply_fut.get_exception(); + vlog(cd_log.warn, "Error while sending request: {}", ex); + co_return std::unexpected(metastore::errc::transport_error); + } + auto reply = reply_fut.get(); + if (reply.ec != rpc::errc::ok) { + co_return std::unexpected(rpc_to_meta_errc(reply.ec)); + } + co_return std::expected{}; +} + ss::future> replicated_metastore::remove_topics( const chunked_vector& topics) { diff --git a/src/v/cloud_topics/level_one/metastore/replicated_metastore.h b/src/v/cloud_topics/level_one/metastore/replicated_metastore.h index a93ad6b5737cb..574976f4c2a08 100644 --- a/src/v/cloud_topics/level_one/metastore/replicated_metastore.h +++ b/src/v/cloud_topics/level_one/metastore/replicated_metastore.h @@ -50,6 +50,9 @@ class replicated_metastore : public metastore { ss::future> set_start_offset(const model::topic_id_partition&, kafka::offset) override; + ss::future> + set_migrating(const model::topic_id_partition&, bool) override; + ss::future> remove_topics(const chunked_vector&) override; diff --git a/src/v/cloud_topics/level_one/metastore/rpc.json b/src/v/cloud_topics/level_one/metastore/rpc.json index 1587a1aaa9666..fc80098d5bb37 100644 --- a/src/v/cloud_topics/level_one/metastore/rpc.json +++ b/src/v/cloud_topics/level_one/metastore/rpc.json @@ -71,6 +71,11 @@ "input_type": "set_start_offset_request", "output_type": "set_start_offset_reply" }, + { + "name": "set_migrating", + "input_type": "set_migrating_request", + "output_type": "set_migrating_reply" + }, { "name": "remove_topics", "input_type": "remove_topics_request", diff --git a/src/v/cloud_topics/level_one/metastore/rpc_types.h b/src/v/cloud_topics/level_one/metastore/rpc_types.h index 406ded87a7839..d66234d58d4eb 100644 --- a/src/v/cloud_topics/level_one/metastore/rpc_types.h +++ b/src/v/cloud_topics/level_one/metastore/rpc_types.h @@ -203,12 +203,15 @@ struct get_first_offset_for_bytes_request struct get_offsets_reply : serde:: - envelope, serde::compat_version<0>> { - auto serde_fields() { return std::tie(ec, start_offset, next_offset); } + envelope, serde::compat_version<0>> { + auto serde_fields() { + return std::tie(ec, start_offset, next_offset, migrating); + } errc ec; kafka::offset start_offset; kafka::offset next_offset; + bool migrating{}; }; struct get_offsets_request : serde::envelope< @@ -376,6 +379,27 @@ struct set_start_offset_request kafka::offset start_offset; }; +struct set_migrating_reply + : serde::envelope< + set_migrating_reply, + serde::version<0>, + serde::compat_version<0>> { + auto serde_fields() { return std::tie(ec); } + + errc ec; +}; +struct set_migrating_request + : serde::envelope< + set_migrating_request, + serde::version<0>, + serde::compat_version<0>> { + using resp_t = set_migrating_reply; + auto serde_fields() { return std::tie(tp, migrating); } + + model::topic_id_partition tp; + bool migrating{}; +}; + struct remove_topics_reply : serde::envelope< remove_topics_reply, diff --git a/src/v/cloud_topics/level_one/metastore/service.cc b/src/v/cloud_topics/level_one/metastore/service.cc index c3befabda91a1..a91e7f37a6dde 100644 --- a/src/v/cloud_topics/level_one/metastore/service.cc +++ b/src/v/cloud_topics/level_one/metastore/service.cc @@ -45,6 +45,12 @@ ss::future service::set_start_offset( std::move(request), leader_router::local_only::yes); } +ss::future service::set_migrating( + set_migrating_request request, ::rpc::streaming_context&) { + return _leader_router->local().set_migrating( + std::move(request), leader_router::local_only::yes); +} + ss::future service::remove_topics( remove_topics_request request, ::rpc::streaming_context&) { return _leader_router->local().remove_topics( diff --git a/src/v/cloud_topics/level_one/metastore/service.h b/src/v/cloud_topics/level_one/metastore/service.h index 306d48e1f8e5e..f1b03a89f8b11 100644 --- a/src/v/cloud_topics/level_one/metastore/service.h +++ b/src/v/cloud_topics/level_one/metastore/service.h @@ -38,6 +38,9 @@ class service final : public impl::l1_rpc_service { ss::future set_start_offset( set_start_offset_request, ::rpc::streaming_context&) override; + ss::future + set_migrating(set_migrating_request, ::rpc::streaming_context&) override; + ss::future remove_topics(remove_topics_request, ::rpc::streaming_context&) override; diff --git a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc index 7b3d0b48dc338..481ec35daecff 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc @@ -189,6 +189,7 @@ simple_metastore::get_offsets( return offsets_response{ .start_offset = prt.start_offset, .next_offset = prt.next_offset, + .migrating = prt.migrating, }; } @@ -312,6 +313,22 @@ simple_metastore::set_start_offset( co_return std::expected{}; } +ss::future> +simple_metastore::set_migrating( + const model::topic_id_partition& tp, bool migrating) { + auto update_res = set_migrating_update::build(state_, tp, migrating); + if (!update_res.has_value()) { + vlog(cd_log.debug, "set_migrating failed: {}", update_res.error()); + co_return std::unexpected(metastore::errc::invalid_request); + } + auto apply_res = update_res->apply(state_); + vassert( + apply_res.has_value(), + "Apply must succeed if can_apply() is true: {}", + apply_res.error()); + co_return std::expected{}; +} + ss::future> simple_metastore::remove_topics(const chunked_vector& topics) { auto update_res = remove_topics_update::build(state_, topics.copy()); diff --git a/src/v/cloud_topics/level_one/metastore/simple_metastore.h b/src/v/cloud_topics/level_one/metastore/simple_metastore.h index ed7d02086a339..8335a80052eab 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_metastore.h +++ b/src/v/cloud_topics/level_one/metastore/simple_metastore.h @@ -78,6 +78,9 @@ class simple_metastore : public metastore { ss::future> set_start_offset(const model::topic_id_partition&, kafka::offset) override; + ss::future> + set_migrating(const model::topic_id_partition&, bool) override; + ss::future> remove_topics(const chunked_vector&) override; diff --git a/src/v/cloud_topics/level_one/metastore/simple_stm.cc b/src/v/cloud_topics/level_one/metastore/simple_stm.cc index 61aebfc502782..612e6aa384188 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_stm.cc +++ b/src/v/cloud_topics/level_one/metastore/simple_stm.cc @@ -154,6 +154,12 @@ ss::future<> simple_stm::do_apply(const model::record_batch& batch) { maybe_log_update_error(_log, key, o, result); break; } + case update_key::set_migrating: { + auto update = serde::read(value_parser); + auto result = update.apply(state_); + maybe_log_update_error(_log, key, o, result); + break; + } case update_key::remove_objects: { auto update = serde::read(value_parser); auto result = update.apply(state_); diff --git a/src/v/cloud_topics/level_one/metastore/state.cc b/src/v/cloud_topics/level_one/metastore/state.cc index 145e1ccf2f305..c209e0a79d68f 100644 --- a/src/v/cloud_topics/level_one/metastore/state.cc +++ b/src/v/cloud_topics/level_one/metastore/state.cc @@ -201,6 +201,7 @@ partition_state partition_state::copy() const { : std::nullopt, .compaction_epoch = compaction_epoch, .term_starts = term_starts, + .migrating = migrating, }; return res; } diff --git a/src/v/cloud_topics/level_one/metastore/state.h b/src/v/cloud_topics/level_one/metastore/state.h index 9ffd3b466480b..7c9a260db79ba 100644 --- a/src/v/cloud_topics/level_one/metastore/state.h +++ b/src/v/cloud_topics/level_one/metastore/state.h @@ -17,6 +17,7 @@ #include "model/fundamental.h" #include "model/timestamp.h" #include "serde/envelope.h" +#include "serde/rw/enum.h" #include "serde/rw/envelope.h" #include "serde/rw/optional.h" #include "serde/rw/set.h" @@ -286,7 +287,7 @@ struct compaction_state // no overlaps and no gaps in order to ensure there is no data loss. struct partition_state : public serde:: - envelope, serde::compat_version<0>> { + envelope, serde::compat_version<0>> { friend bool operator==(const partition_state&, const partition_state&) = default; auto serde_fields() { @@ -296,7 +297,8 @@ struct partition_state next_offset, compaction_state, compaction_epoch, - term_starts); + term_starts, + migrating); } partition_state copy() const; @@ -356,6 +358,12 @@ struct partition_state // has been prefix truncated to be empty. I.e. this list should never be // empty once there has been data in the log. absl::btree_set term_starts; + + // True while the partition is mid tiered->cloud migration. For + // offline/remote consumers (cluster recovery, read-replica) that cannot + // observe the live archival-STM routing signal. Defaults to false, so a + // snapshot predating this field restores as a native cloud topic. + bool migrating{}; }; // Tracks the state managed for each partition of a Kafka topic. diff --git a/src/v/cloud_topics/level_one/metastore/state_update.cc b/src/v/cloud_topics/level_one/metastore/state_update.cc index d2fdf6594b8f7..33c59bfe451f1 100644 --- a/src/v/cloud_topics/level_one/metastore/state_update.cc +++ b/src/v/cloud_topics/level_one/metastore/state_update.cc @@ -938,6 +938,51 @@ set_start_offset_update::apply(state& state) { return std::monostate{}; } +std::expected +set_migrating_update::build( + const state& state, + const model::topic_id_partition& tp, + bool migrating, + bool* is_no_op) { + set_migrating_update update{ + .tp = tp, + .migrating = migrating, + }; + auto allowed = update.can_apply(state, is_no_op); + if (!allowed.has_value()) { + return std::unexpected(allowed.error()); + } + return update; +} + +std::expected +set_migrating_update::can_apply(const state& state, bool* is_no_op) { + // An absent partition defaults to not-migrating; setting it migrating + // creates the partition (the migrating marker may be its first write). + auto prt_ref = state.partition_state(tp); + auto current = prt_ref.has_value() && prt_ref->get().migrating; + if (is_no_op) { + *is_no_op = migrating == current; + } + return std::monostate{}; +} + +std::expected +set_migrating_update::apply(state& state) { + bool is_no_op = false; + auto allowed = can_apply(state, &is_no_op); + if (!allowed.has_value()) { + return std::unexpected(allowed.error()); + } + if (is_no_op) { + return std::monostate{}; + } + auto& p_state + = state.topic_to_state[tp.topic_id].pid_to_state[tp.partition]; + p_state.migrating = migrating; + return std::monostate{}; +} + std::expected remove_objects_update::build( const state& state, chunked_vector objects) { diff --git a/src/v/cloud_topics/level_one/metastore/state_update.h b/src/v/cloud_topics/level_one/metastore/state_update.h index 7ce7c8519a618..7b6e8497af417 100644 --- a/src/v/cloud_topics/level_one/metastore/state_update.h +++ b/src/v/cloud_topics/level_one/metastore/state_update.h @@ -30,6 +30,7 @@ enum class update_key : uint8_t { preregister_objects = 5, expire_preregistered_objects = 6, replace_objects = 7, + set_migrating = 8, }; using stm_update_error = named_type; @@ -251,6 +252,33 @@ struct set_start_offset_update kafka::offset new_start_offset; }; +// Sets the partition's `migrating` flag. Setting the current value is a no-op. +// Creates the partition's state if absent (the migrating marker may be the +// first write for a partition being migrated). +struct set_migrating_update + : public serde::envelope< + set_migrating_update, + serde::version<0>, + serde::compat_version<0>> { + friend bool operator==( + const set_migrating_update&, const set_migrating_update&) = default; + auto serde_fields() { return std::tie(tp, migrating); } + + static constexpr auto key{update_key::set_migrating}; + static std::expected build( + const state&, + const model::topic_id_partition&, + bool migrating, + bool* is_no_op = nullptr); + + std::expected + can_apply(const state&, bool* is_no_op = nullptr); + std::expected apply(state&); + + model::topic_id_partition tp; + bool migrating{}; +}; + struct remove_objects_update : public serde::envelope< remove_objects_update, @@ -355,6 +383,8 @@ struct fmt::formatter final "expire_preregistered_objects", ctx); case cloud_topics::l1::update_key::replace_objects: return formatter::format("replace_objects", ctx); + case cloud_topics::l1::update_key::set_migrating: + return formatter::format("set_migrating", ctx); } } }; diff --git a/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc b/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc index 5c9a9310ebc90..d6a7b8a3001b2 100644 --- a/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc +++ b/src/v/cloud_topics/level_one/metastore/tests/imported_serde_test.cc @@ -164,6 +164,7 @@ TEST(ImportedSerdeTest, ImportedTsSegmentInfoRoundTrip) { auto decoded = serde::from_iobuf(std::move(buf)); ASSERT_EQ(decoded, info); } + // Verify that imported_ts_object_location round-trips through serde correctly. TEST(ImportedSerdeTest, ImportedTsObjectLocationRoundTrip) { imported_ts_object_location loc{ @@ -175,3 +176,115 @@ TEST(ImportedSerdeTest, ImportedTsObjectLocationRoundTrip) { std::move(buf)); ASSERT_EQ(decoded, loc); } + +namespace { + +/// Mirrors metadata_row_value at version<1>, before the `migrating` field +/// field was added in version<2>. +struct metadata_row_value_v1 + : public serde::envelope< + metadata_row_value_v1, + serde::version<1>, + serde::compat_version<0>> { + auto serde_fields() { + return std::tie( + start_offset, next_offset, compaction_epoch, size, num_extents); + } + kafka::offset start_offset{}; + kafka::offset next_offset{}; + partition_state::compaction_epoch_t compaction_epoch{0}; + size_t size{0}; + size_t num_extents{0}; +}; + +/// Mirrors partition_state at version<0>, before the `migrating` field +/// was added in version<1>. +struct partition_state_v0 + : public serde::envelope< + partition_state_v0, + serde::version<0>, + serde::compat_version<0>> { + auto serde_fields() { + return std::tie( + extents, + start_offset, + next_offset, + compaction_state, + compaction_epoch, + term_starts); + } + partition_state::extent_set_t extents; + kafka::offset start_offset{0}; + kafka::offset next_offset{0}; + std::optional compaction_state; + partition_state::compaction_epoch_t compaction_epoch{0}; + absl::btree_set term_starts; +}; + +} // namespace + +// A v1 metadata_row_value (no `migrating` field) decodes as the current v2 +// struct with migrating defaulting to false. +TEST(ImportedSerdeTest, MetadataRowValueV1CompatV2) { + metadata_row_value_v1 v1; + v1.start_offset = kafka::offset{5}; + v1.next_offset = kafka::offset{100}; + v1.compaction_epoch = partition_state::compaction_epoch_t{3}; + v1.size = 4096; + v1.num_extents = 7; + + iobuf buf = serde::to_iobuf(v1); + auto v2 = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(v2.start_offset, v1.start_offset); + ASSERT_EQ(v2.next_offset, v1.next_offset); + ASSERT_EQ(v2.compaction_epoch, v1.compaction_epoch); + ASSERT_EQ(v2.size, v1.size); + ASSERT_EQ(v2.num_extents, v1.num_extents); + ASSERT_EQ(v2.migrating, false); +} + +// A v2 metadata_row_value with migrating set round-trips through serde. +TEST(ImportedSerdeTest, MetadataRowValueV2RoundTrip) { + metadata_row_value v2; + v2.start_offset = kafka::offset{5}; + v2.next_offset = kafka::offset{100}; + v2.migrating = true; + + iobuf buf = serde::to_iobuf(v2); + auto decoded = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(decoded.start_offset, v2.start_offset); + ASSERT_EQ(decoded.next_offset, v2.next_offset); + ASSERT_EQ(decoded.migrating, true); +} + +// A v0 partition_state (no `migrating` field) decodes as the current v1 struct +// with migrating defaulting to false. +TEST(ImportedSerdeTest, PartitionStateV0CompatV1) { + partition_state_v0 v0; + v0.start_offset = kafka::offset{10}; + v0.next_offset = kafka::offset{42}; + + iobuf buf = serde::to_iobuf(v0); + auto v1 = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(v1.start_offset, v0.start_offset); + ASSERT_EQ(v1.next_offset, v0.next_offset); + ASSERT_EQ(v1.migrating, false); +} + +// A v1 partition_state with migrating set round-trips through serde. +TEST(ImportedSerdeTest, PartitionStateV1RoundTrip) { + partition_state ps; + ps.start_offset = kafka::offset{10}; + ps.next_offset = kafka::offset{42}; + ps.migrating = true; + + iobuf buf = serde::to_iobuf(ps); + auto decoded = serde::from_iobuf(std::move(buf)); + + ASSERT_EQ(decoded.start_offset, ps.start_offset); + ASSERT_EQ(decoded.next_offset, ps.next_offset); + ASSERT_EQ(decoded.migrating, true); +} diff --git a/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc b/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc index 9eb8bbd831301..32bee675bb23f 100644 --- a/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc +++ b/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc @@ -322,6 +322,46 @@ class StateUpdateParamTest : public ::testing::TestWithParam { return std::monostate{}; } + std::expected + apply_set_migrating(const model::topic_id_partition& tp, bool migrating) { + if (GetParam() == state_backend::simple) { + auto update = set_migrating_update::build(state_, tp, migrating); + if (!update.has_value()) { + return std::unexpected( + stm_update_error{fmt::format("{}", update.error())}); + } + return update->apply(state_); + } + set_migrating_db_update db_update{ + .tp = tp, + .migrating = migrating, + }; + auto reader = state_reader(db_->create_snapshot()); + chunked_vector rows; + auto result = db_update.build_rows(reader, rows).get(); + if (!result.has_value()) { + return std::unexpected( + stm_update_error{fmt::format("{}", result.error())}); + } + apply_rows_to_db(rows); + return std::monostate{}; + } + + // Reads the partition's `migrating` flag from whichever backend is active, + // defaulting to false when the partition has no state yet. + bool read_migrating(const model::topic_id_partition& tp) { + if (GetParam() == state_backend::simple) { + auto prt = state_.partition_state(tp); + return prt.has_value() && prt->get().migrating; + } + auto reader = state_reader(db_->create_snapshot()); + auto meta = reader.get_metadata(tp).get(); + if (!meta.has_value() || !meta->has_value()) { + return false; + } + return (*meta)->migrating; + } + std::expected apply_remove_topics(std::initializer_list topics_list) { chunked_vector topics; @@ -542,6 +582,47 @@ TEST_P(StateUpdateParamTest, TestAddBasic) { } } +TEST_P(StateUpdateParamTest, TestSetMigratingCreatesOnAbsentPartition) { + auto tp = model::topic_id_partition::from(tidp_a); + // Setting `migrating` on a partition with no prior state creates it. + EXPECT_FALSE(read_migrating(tp)); + auto res = apply_set_migrating(tp, true); + EXPECT_TRUE(res.has_value()); + EXPECT_TRUE(read_migrating(tp)); +} + +TEST_P(StateUpdateParamTest, TestSetMigratingThenClear) { + auto tp = model::topic_id_partition::from(tidp_a); + // The mirror sets migrating; cutover clears it. Both directions apply. + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + EXPECT_TRUE(read_migrating(tp)); + ASSERT_TRUE(apply_set_migrating(tp, false).has_value()); + EXPECT_FALSE(read_migrating(tp)); +} + +TEST_P(StateUpdateParamTest, TestSetMigratingIdempotent) { + auto tp = model::topic_id_partition::from(tidp_a); + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + // Re-setting the current value is a no-op success. + auto res = apply_set_migrating(tp, true); + EXPECT_TRUE(res.has_value()); + EXPECT_TRUE(read_migrating(tp)); +} + +TEST_P(StateUpdateParamTest, TestSetMigratingPreservedAcrossAddObjects) { + auto tp = model::topic_id_partition::from(tidp_a); + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + auto update = add_objects_builder() + .add(new_obj_builder(oid1, 100, 1100) + .add(tidp_a, 0_o, 10_o, 1999_t, 0, 99) + .build()) + .add_term_start(tidp_a, 0_tm, 0_o) + .build(); + ASSERT_TRUE(apply_add_objects(std::move(update)).has_value()); + // Adding objects must not clobber the migration phase. + EXPECT_TRUE(read_migrating(tp)); +} + TEST_P(StateUpdateParamTest, TestDuplicateAddSingleUpdate) { ASSERT_TRUE(apply_preregister_objects({oid1, oid2}).has_value()); auto update = add_objects_builder() diff --git a/src/v/cloud_topics/read_replica/snapshot_metastore.cc b/src/v/cloud_topics/read_replica/snapshot_metastore.cc index e9fe7ca25215a..3269e1820d94b 100644 --- a/src/v/cloud_topics/read_replica/snapshot_metastore.cc +++ b/src/v/cloud_topics/read_replica/snapshot_metastore.cc @@ -171,6 +171,7 @@ snapshot_metastore::get_offsets(const model::topic_id_partition& tidp) { co_return offsets_response{ .start_offset = metadata.start_offset, .next_offset = metadata.next_offset, + .migrating = metadata.migrating, }; } @@ -212,6 +213,11 @@ snapshot_metastore::set_start_offset( co_return std::unexpected(errc::invalid_request); } +ss::future> +snapshot_metastore::set_migrating(const model::topic_id_partition&, bool) { + co_return std::unexpected(errc::invalid_request); +} + ss::future< std::expected> snapshot_metastore::remove_topics(const chunked_vector&) { diff --git a/src/v/cloud_topics/read_replica/snapshot_metastore.h b/src/v/cloud_topics/read_replica/snapshot_metastore.h index 82c1c61508455..216664a8380c7 100644 --- a/src/v/cloud_topics/read_replica/snapshot_metastore.h +++ b/src/v/cloud_topics/read_replica/snapshot_metastore.h @@ -52,6 +52,9 @@ class snapshot_metastore : public l1::metastore { ss::future> set_start_offset(const model::topic_id_partition&, kafka::offset) override; + ss::future> + set_migrating(const model::topic_id_partition&, bool) override; + ss::future> remove_topics(const chunked_vector&) override; diff --git a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py index a6a00619046ee..fe75d5d53d1af 100644 --- a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py +++ b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.py @@ -9,7 +9,7 @@ from ........proto.redpanda.core.pbgen import options_pb2 as proto_dot_redpanda_dot_core_dot_pbgen_dot_options__pb2 from ........proto.redpanda.core.pbgen import rpc_pb2 as proto_dot_redpanda_dot_core_dot_pbgen_dot_rpc__pb2 from ........proto.redpanda.core.common.v1 import ntp_pb2 as proto_dot_redpanda_dot_core_dot_common_dot_v1_dot_ntp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\nBproto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto\x12,redpanda.core.admin.internal.cloud_topics.v1\x1a\'proto/redpanda/core/pbgen/options.proto\x1a#proto/redpanda/core/pbgen/rpc.proto\x1a\'proto/redpanda/core/common/v1/ntp.proto"O\n\x11GetOffsetsRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"\\\n\x12GetOffsetsResponse\x12F\n\x07offsets\x18\x02 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.Offsets"4\n\x07Offsets\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03"L\n\x0eGetSizeRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"%\n\x0fGetSizeResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x04"6\n\x17GetDatabaseStatsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r"\xbd\x01\n\x18GetDatabaseStatsResponse\x12\x1d\n\x15active_memtable_bytes\x18\x01 \x01(\x04\x12 \n\x18immutable_memtable_bytes\x18\x02 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x01(\x04\x12F\n\x06levels\x18\x04 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.LsmLevel"f\n\x08LsmLevel\x12\x14\n\x0clevel_number\x18\x01 \x01(\x05\x12D\n\x05files\x18\x02 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.LsmFile"m\n\x07LsmFile\x12\r\n\x05epoch\x18\x01 \x01(\x04\x12\n\n\x02id\x18\x02 \x01(\x04\x12\x12\n\nsize_bytes\x18\x03 \x01(\x04\x12\x19\n\x11smallest_key_info\x18\x04 \x01(\t\x12\x18\n\x10largest_key_info\x18\x05 \x01(\t"5\n\x0bMetadataKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"H\n\tExtentKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x13\n\x0bbase_offset\x18\x03 \x01(\x03"B\n\x07TermKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x0f\n\x07term_id\x18\x03 \x01(\x03"7\n\rCompactionKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"\x1e\n\tObjectKey\x12\x11\n\tobject_id\x18\x01 \x01(\t"\x8e\x03\n\x06RowKey\x12M\n\x08metadata\x18\x01 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.MetadataKeyH\x00\x12I\n\x06extent\x18\x02 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ExtentKeyH\x00\x12E\n\x04term\x18\x03 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.TermKeyH\x00\x12Q\n\ncompaction\x18\x04 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.CompactionKeyH\x00\x12I\n\x06object\x18\x05 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ObjectKeyH\x00B\x05\n\x03key"w\n\rMetadataValue\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x18\n\x10compaction_epoch\x18\x03 \x01(\x03\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x13\n\x0bnum_extents\x18\x05 \x01(\x04"+\n\x18ImportedTsObjectLocation\x12\x0f\n\x07ts_path\x18\x01 \x01(\t"\x9c\x01\n\x15ImportedTsSegmentInfo\x12\x14\n\x0csegment_term\x18\x01 \x01(\x03\x12\x12\n\ndelta_base\x18\x02 \x01(\x03\x12Y\n\x08tx_state\x18\x03 \x01(\x0e2G.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsTxManifestState"\xe3\x01\n\x0bExtentValue\x12\x13\n\x0blast_offset\x18\x01 \x01(\x03\x12\x15\n\rmax_timestamp\x18\x02 \x01(\x03\x12\x0f\n\x07filepos\x18\x03 \x01(\x04\x12\x0b\n\x03len\x18\x04 \x01(\x04\x12\x11\n\tobject_id\x18\x05 \x01(\t\x12b\n\x10imported_ts_info\x18\x06 \x01(\x0b2C.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsSegmentInfoH\x00\x88\x01\x01B\x13\n\x11_imported_ts_info"&\n\tTermValue\x12\x19\n\x11term_start_offset\x18\x01 \x01(\x03"7\n\x0bOffsetRange\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"j\n\x1aCleanedRangeWithTombstones\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12"\n\x1acleaned_with_tombstones_at\x18\x03 \x01(\x03"\xd6\x01\n\x0fCompactionValue\x12Q\n\x0ecleaned_ranges\x18\x01 \x03(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.OffsetRange\x12p\n\x1ecleaned_ranges_with_tombstones\x18\x02 \x03(\x0b2H.redpanda.core.admin.internal.cloud_topics.v1.CleanedRangeWithTombstones"\xa0\x02\n\x0bObjectValue\x12\x17\n\x0ftotal_data_size\x18\x01 \x01(\x04\x12\x19\n\x11removed_data_size\x18\x02 \x01(\x04\x12\x12\n\nfooter_pos\x18\x03 \x01(\x04\x12\x13\n\x0bobject_size\x18\x04 \x01(\x04\x12\x14\n\x0clast_updated\x18\x05 \x01(\x03\x12\x1a\n\x12is_preregistration\x18\x06 \x01(\x08\x12i\n\x14imported_ts_location\x18\x07 \x01(\x0b2F.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsObjectLocationH\x00\x88\x01\x01B\x17\n\x15_imported_ts_location"\x9c\x03\n\x08RowValue\x12O\n\x08metadata\x18\x01 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.MetadataValueH\x00\x12K\n\x06extent\x18\x02 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ExtentValueH\x00\x12G\n\x04term\x18\x03 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.TermValueH\x00\x12S\n\ncompaction\x18\x04 \x01(\x0b2=.redpanda.core.admin.internal.cloud_topics.v1.CompactionValueH\x00\x12K\n\x06object\x18\x05 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ObjectValueH\x00B\x07\n\x05value"\x94\x01\n\x08WriteRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\xbe\x01\n\x10WriteRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12F\n\x06writes\x18\x02 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.WriteRow\x12E\n\x07deletes\x18\x03 \x03(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey")\n\x11WriteRowsResponse\x12\x14\n\x0crows_written\x18\x01 \x01(\r"\x93\x01\n\x07ReadRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\x94\x02\n\x0fReadRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12H\n\x08seek_key\x18\x02 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x00\x12\x16\n\x0craw_seek_key\x18\x03 \x01(\tH\x00\x12H\n\x08last_key\x18\x04 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x01\x12\x16\n\x0craw_last_key\x18\x05 \x01(\tH\x01\x12\x10\n\x08max_rows\x18\x06 \x01(\rB\x06\n\x04seekB\x06\n\x04last"i\n\x10ReadRowsResponse\x12C\n\x04rows\x18\x01 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.ReadRow\x12\x10\n\x08next_key\x18\x02 \x01(\t"\xc8\x01\n\x18ValidatePartitionRequest\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x1d\n\x15check_object_metadata\x18\x03 \x01(\x08\x12\x1c\n\x14check_object_storage\x18\x04 \x01(\x08\x12\x1d\n\x10resume_at_offset\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x13\n\x0bmax_extents\x18\x06 \x01(\rB\x13\n\x11_resume_at_offset"\xbd\x01\n\x19ValidatePartitionResponse\x12Q\n\tanomalies\x18\x01 \x03(\x0b2>.redpanda.core.admin.internal.cloud_topics.v1.MetastoreAnomaly\x12\x1d\n\x10resume_at_offset\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11extents_validated\x18\x03 \x01(\rB\x13\n\x11_resume_at_offset"x\n\x10MetastoreAnomaly\x12O\n\x0canomaly_type\x18\x01 \x01(\x0e29.redpanda.core.admin.internal.cloud_topics.v1.AnomalyType\x12\x13\n\x0bdescription\x18\x02 \x01(\t"F\n\x16ListCloudTopicsRequest\x12\x18\n\x10after_topic_name\x18\x01 \x01(\t\x12\x12\n\nmax_topics\x18\x02 \x01(\r"y\n\x17ListCloudTopicsResponse\x12L\n\x06topics\x18\x01 \x03(\x0b2<.redpanda.core.admin.internal.cloud_topics.v1.CloudTopicInfo\x12\x10\n\x08has_more\x18\x02 \x01(\x08"O\n\x0eCloudTopicInfo\x12\x12\n\ntopic_name\x18\x01 \x01(\t\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x17\n\x0fpartition_count\x18\x03 \x01(\x05*\x9b\x01\n\x19ImportedTsTxManifestState\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN\x10\x00\x12(\n$IMPORTED_TS_TX_MANIFEST_STATE_ABSENT\x10\x01\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_PRESENT\x10\x02*\xbb\x04\n\x0bAnomalyType\x12\x1c\n\x18ANOMALY_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bANOMALY_TYPE_EXTENT_OVERLAP\x10\x01\x12\x1b\n\x17ANOMALY_TYPE_EXTENT_GAP\x10\x02\x12%\n!ANOMALY_TYPE_NEXT_OFFSET_MISMATCH\x10\x03\x12&\n"ANOMALY_TYPE_START_OFFSET_MISMATCH\x10\x04\x12!\n\x1dANOMALY_TYPE_OBJECT_NOT_FOUND\x10\x05\x12%\n!ANOMALY_TYPE_OBJECT_PREREGISTERED\x10\x06\x12&\n"ANOMALY_TYPE_OBJECT_NOT_IN_STORAGE\x10\x07\x12-\n)ANOMALY_TYPE_COMPACTION_RANGE_BELOW_START\x10\x08\x12,\n(ANOMALY_TYPE_COMPACTION_RANGE_ABOVE_NEXT\x10\t\x12-\n)ANOMALY_TYPE_COMPACTION_TOMBSTONE_OVERLAP\x10\n\x125\n1ANOMALY_TYPE_COMPACTION_TOMBSTONE_OUTSIDE_CLEANED\x10\x0b\x12\x1e\n\x1aANOMALY_TYPE_TERM_ORDERING\x10\x0c\x12,\n(ANOMALY_TYPE_COMPACTION_STATE_UNEXPECTED\x10\r2\xec\x08\n\x10MetastoreService\x12\x97\x01\n\nGetOffsets\x12?.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsRequest\x1a@.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x8e\x01\n\x07GetSize\x12<.redpanda.core.admin.internal.cloud_topics.v1.GetSizeRequest\x1a=.redpanda.core.admin.internal.cloud_topics.v1.GetSizeResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa9\x01\n\x10GetDatabaseStats\x12E.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsRequest\x1aF.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x94\x01\n\tWriteRows\x12>.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsRequest\x1a?.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x91\x01\n\x08ReadRows\x12=.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsRequest\x1a>.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa6\x01\n\x0fListCloudTopics\x12D.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsRequest\x1aE.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xac\x01\n\x11ValidatePartition\x12F.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionRequest\x1aG.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionResponse"\x06\xea\x92\x19\x02\x10\x03B\x1b\xea\x92\x19\x17proto::admin::metastoreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\nBproto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto\x12,redpanda.core.admin.internal.cloud_topics.v1\x1a\'proto/redpanda/core/pbgen/options.proto\x1a#proto/redpanda/core/pbgen/rpc.proto\x1a\'proto/redpanda/core/common/v1/ntp.proto"O\n\x11GetOffsetsRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"\\\n\x12GetOffsetsResponse\x12F\n\x07offsets\x18\x02 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.Offsets"4\n\x07Offsets\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03"L\n\x0eGetSizeRequest\x12:\n\tpartition\x18\x01 \x01(\x0b2\'.redpanda.core.common.v1.TopicPartition"%\n\x0fGetSizeResponse\x12\x12\n\nsize_bytes\x18\x01 \x01(\x04"6\n\x17GetDatabaseStatsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r"\xbd\x01\n\x18GetDatabaseStatsResponse\x12\x1d\n\x15active_memtable_bytes\x18\x01 \x01(\x04\x12 \n\x18immutable_memtable_bytes\x18\x02 \x01(\x04\x12\x18\n\x10total_size_bytes\x18\x03 \x01(\x04\x12F\n\x06levels\x18\x04 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.LsmLevel"f\n\x08LsmLevel\x12\x14\n\x0clevel_number\x18\x01 \x01(\x05\x12D\n\x05files\x18\x02 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.LsmFile"m\n\x07LsmFile\x12\r\n\x05epoch\x18\x01 \x01(\x04\x12\n\n\x02id\x18\x02 \x01(\x04\x12\x12\n\nsize_bytes\x18\x03 \x01(\x04\x12\x19\n\x11smallest_key_info\x18\x04 \x01(\t\x12\x18\n\x10largest_key_info\x18\x05 \x01(\t"5\n\x0bMetadataKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"H\n\tExtentKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x13\n\x0bbase_offset\x18\x03 \x01(\x03"B\n\x07TermKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x0f\n\x07term_id\x18\x03 \x01(\x03"7\n\rCompactionKey\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05"\x1e\n\tObjectKey\x12\x11\n\tobject_id\x18\x01 \x01(\t"\x8e\x03\n\x06RowKey\x12M\n\x08metadata\x18\x01 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.MetadataKeyH\x00\x12I\n\x06extent\x18\x02 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ExtentKeyH\x00\x12E\n\x04term\x18\x03 \x01(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.TermKeyH\x00\x12Q\n\ncompaction\x18\x04 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.CompactionKeyH\x00\x12I\n\x06object\x18\x05 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.ObjectKeyH\x00B\x05\n\x03key"\x8a\x01\n\rMetadataValue\x12\x14\n\x0cstart_offset\x18\x01 \x01(\x03\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x18\n\x10compaction_epoch\x18\x03 \x01(\x03\x12\x0c\n\x04size\x18\x04 \x01(\x04\x12\x13\n\x0bnum_extents\x18\x05 \x01(\x04\x12\x11\n\tmigrating\x18\x06 \x01(\x08"+\n\x18ImportedTsObjectLocation\x12\x0f\n\x07ts_path\x18\x01 \x01(\t"\x9c\x01\n\x15ImportedTsSegmentInfo\x12\x14\n\x0csegment_term\x18\x01 \x01(\x03\x12\x12\n\ndelta_base\x18\x02 \x01(\x03\x12Y\n\x08tx_state\x18\x03 \x01(\x0e2G.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsTxManifestState"\xe3\x01\n\x0bExtentValue\x12\x13\n\x0blast_offset\x18\x01 \x01(\x03\x12\x15\n\rmax_timestamp\x18\x02 \x01(\x03\x12\x0f\n\x07filepos\x18\x03 \x01(\x04\x12\x0b\n\x03len\x18\x04 \x01(\x04\x12\x11\n\tobject_id\x18\x05 \x01(\t\x12b\n\x10imported_ts_info\x18\x06 \x01(\x0b2C.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsSegmentInfoH\x00\x88\x01\x01B\x13\n\x11_imported_ts_info"&\n\tTermValue\x12\x19\n\x11term_start_offset\x18\x01 \x01(\x03"7\n\x0bOffsetRange\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"j\n\x1aCleanedRangeWithTombstones\x12\x13\n\x0bbase_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12"\n\x1acleaned_with_tombstones_at\x18\x03 \x01(\x03"\xd6\x01\n\x0fCompactionValue\x12Q\n\x0ecleaned_ranges\x18\x01 \x03(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.OffsetRange\x12p\n\x1ecleaned_ranges_with_tombstones\x18\x02 \x03(\x0b2H.redpanda.core.admin.internal.cloud_topics.v1.CleanedRangeWithTombstones"\xa0\x02\n\x0bObjectValue\x12\x17\n\x0ftotal_data_size\x18\x01 \x01(\x04\x12\x19\n\x11removed_data_size\x18\x02 \x01(\x04\x12\x12\n\nfooter_pos\x18\x03 \x01(\x04\x12\x13\n\x0bobject_size\x18\x04 \x01(\x04\x12\x14\n\x0clast_updated\x18\x05 \x01(\x03\x12\x1a\n\x12is_preregistration\x18\x06 \x01(\x08\x12i\n\x14imported_ts_location\x18\x07 \x01(\x0b2F.redpanda.core.admin.internal.cloud_topics.v1.ImportedTsObjectLocationH\x00\x88\x01\x01B\x17\n\x15_imported_ts_location"\x9c\x03\n\x08RowValue\x12O\n\x08metadata\x18\x01 \x01(\x0b2;.redpanda.core.admin.internal.cloud_topics.v1.MetadataValueH\x00\x12K\n\x06extent\x18\x02 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ExtentValueH\x00\x12G\n\x04term\x18\x03 \x01(\x0b27.redpanda.core.admin.internal.cloud_topics.v1.TermValueH\x00\x12S\n\ncompaction\x18\x04 \x01(\x0b2=.redpanda.core.admin.internal.cloud_topics.v1.CompactionValueH\x00\x12K\n\x06object\x18\x05 \x01(\x0b29.redpanda.core.admin.internal.cloud_topics.v1.ObjectValueH\x00B\x07\n\x05value"\x94\x01\n\x08WriteRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\xbe\x01\n\x10WriteRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12F\n\x06writes\x18\x02 \x03(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.WriteRow\x12E\n\x07deletes\x18\x03 \x03(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey")\n\x11WriteRowsResponse\x12\x14\n\x0crows_written\x18\x01 \x01(\r"\x93\x01\n\x07ReadRow\x12A\n\x03key\x18\x01 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKey\x12E\n\x05value\x18\x02 \x01(\x0b26.redpanda.core.admin.internal.cloud_topics.v1.RowValue"\x94\x02\n\x0fReadRowsRequest\x12\x1b\n\x13metastore_partition\x18\x01 \x01(\r\x12H\n\x08seek_key\x18\x02 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x00\x12\x16\n\x0craw_seek_key\x18\x03 \x01(\tH\x00\x12H\n\x08last_key\x18\x04 \x01(\x0b24.redpanda.core.admin.internal.cloud_topics.v1.RowKeyH\x01\x12\x16\n\x0craw_last_key\x18\x05 \x01(\tH\x01\x12\x10\n\x08max_rows\x18\x06 \x01(\rB\x06\n\x04seekB\x06\n\x04last"i\n\x10ReadRowsResponse\x12C\n\x04rows\x18\x01 \x03(\x0b25.redpanda.core.admin.internal.cloud_topics.v1.ReadRow\x12\x10\n\x08next_key\x18\x02 \x01(\t"\xc8\x01\n\x18ValidatePartitionRequest\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x14\n\x0cpartition_id\x18\x02 \x01(\x05\x12\x1d\n\x15check_object_metadata\x18\x03 \x01(\x08\x12\x1c\n\x14check_object_storage\x18\x04 \x01(\x08\x12\x1d\n\x10resume_at_offset\x18\x05 \x01(\x03H\x00\x88\x01\x01\x12\x13\n\x0bmax_extents\x18\x06 \x01(\rB\x13\n\x11_resume_at_offset"\xbd\x01\n\x19ValidatePartitionResponse\x12Q\n\tanomalies\x18\x01 \x03(\x0b2>.redpanda.core.admin.internal.cloud_topics.v1.MetastoreAnomaly\x12\x1d\n\x10resume_at_offset\x18\x02 \x01(\x03H\x00\x88\x01\x01\x12\x19\n\x11extents_validated\x18\x03 \x01(\rB\x13\n\x11_resume_at_offset"x\n\x10MetastoreAnomaly\x12O\n\x0canomaly_type\x18\x01 \x01(\x0e29.redpanda.core.admin.internal.cloud_topics.v1.AnomalyType\x12\x13\n\x0bdescription\x18\x02 \x01(\t"F\n\x16ListCloudTopicsRequest\x12\x18\n\x10after_topic_name\x18\x01 \x01(\t\x12\x12\n\nmax_topics\x18\x02 \x01(\r"y\n\x17ListCloudTopicsResponse\x12L\n\x06topics\x18\x01 \x03(\x0b2<.redpanda.core.admin.internal.cloud_topics.v1.CloudTopicInfo\x12\x10\n\x08has_more\x18\x02 \x01(\x08"O\n\x0eCloudTopicInfo\x12\x12\n\ntopic_name\x18\x01 \x01(\t\x12\x10\n\x08topic_id\x18\x02 \x01(\t\x12\x17\n\x0fpartition_count\x18\x03 \x01(\x05*\x9b\x01\n\x19ImportedTsTxManifestState\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN\x10\x00\x12(\n$IMPORTED_TS_TX_MANIFEST_STATE_ABSENT\x10\x01\x12)\n%IMPORTED_TS_TX_MANIFEST_STATE_PRESENT\x10\x02*\xbb\x04\n\x0bAnomalyType\x12\x1c\n\x18ANOMALY_TYPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bANOMALY_TYPE_EXTENT_OVERLAP\x10\x01\x12\x1b\n\x17ANOMALY_TYPE_EXTENT_GAP\x10\x02\x12%\n!ANOMALY_TYPE_NEXT_OFFSET_MISMATCH\x10\x03\x12&\n"ANOMALY_TYPE_START_OFFSET_MISMATCH\x10\x04\x12!\n\x1dANOMALY_TYPE_OBJECT_NOT_FOUND\x10\x05\x12%\n!ANOMALY_TYPE_OBJECT_PREREGISTERED\x10\x06\x12&\n"ANOMALY_TYPE_OBJECT_NOT_IN_STORAGE\x10\x07\x12-\n)ANOMALY_TYPE_COMPACTION_RANGE_BELOW_START\x10\x08\x12,\n(ANOMALY_TYPE_COMPACTION_RANGE_ABOVE_NEXT\x10\t\x12-\n)ANOMALY_TYPE_COMPACTION_TOMBSTONE_OVERLAP\x10\n\x125\n1ANOMALY_TYPE_COMPACTION_TOMBSTONE_OUTSIDE_CLEANED\x10\x0b\x12\x1e\n\x1aANOMALY_TYPE_TERM_ORDERING\x10\x0c\x12,\n(ANOMALY_TYPE_COMPACTION_STATE_UNEXPECTED\x10\r2\xec\x08\n\x10MetastoreService\x12\x97\x01\n\nGetOffsets\x12?.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsRequest\x1a@.redpanda.core.admin.internal.cloud_topics.v1.GetOffsetsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x8e\x01\n\x07GetSize\x12<.redpanda.core.admin.internal.cloud_topics.v1.GetSizeRequest\x1a=.redpanda.core.admin.internal.cloud_topics.v1.GetSizeResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa9\x01\n\x10GetDatabaseStats\x12E.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsRequest\x1aF.redpanda.core.admin.internal.cloud_topics.v1.GetDatabaseStatsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x94\x01\n\tWriteRows\x12>.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsRequest\x1a?.redpanda.core.admin.internal.cloud_topics.v1.WriteRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\x91\x01\n\x08ReadRows\x12=.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsRequest\x1a>.redpanda.core.admin.internal.cloud_topics.v1.ReadRowsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xa6\x01\n\x0fListCloudTopics\x12D.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsRequest\x1aE.redpanda.core.admin.internal.cloud_topics.v1.ListCloudTopicsResponse"\x06\xea\x92\x19\x02\x10\x03\x12\xac\x01\n\x11ValidatePartition\x12F.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionRequest\x1aG.redpanda.core.admin.internal.cloud_topics.v1.ValidatePartitionResponse"\x06\xea\x92\x19\x02\x10\x03B\x1b\xea\x92\x19\x17proto::admin::metastoreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'proto.redpanda.core.admin.internal.cloud_topics.v1.metastore_pb2', _globals) @@ -30,10 +30,10 @@ _globals['_METASTORESERVICE'].methods_by_name['ListCloudTopics']._serialized_options = b'\xea\x92\x19\x02\x10\x03' _globals['_METASTORESERVICE'].methods_by_name['ValidatePartition']._loaded_options = None _globals['_METASTORESERVICE'].methods_by_name['ValidatePartition']._serialized_options = b'\xea\x92\x19\x02\x10\x03' - _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_start = 5131 - _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_end = 5286 - _globals['_ANOMALYTYPE']._serialized_start = 5289 - _globals['_ANOMALYTYPE']._serialized_end = 5860 + _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_start = 5151 + _globals['_IMPORTEDTSTXMANIFESTSTATE']._serialized_end = 5306 + _globals['_ANOMALYTYPE']._serialized_start = 5309 + _globals['_ANOMALYTYPE']._serialized_end = 5880 _globals['_GETOFFSETSREQUEST']._serialized_start = 235 _globals['_GETOFFSETSREQUEST']._serialized_end = 314 _globals['_GETOFFSETSRESPONSE']._serialized_start = 316 @@ -64,49 +64,49 @@ _globals['_OBJECTKEY']._serialized_end = 1328 _globals['_ROWKEY']._serialized_start = 1331 _globals['_ROWKEY']._serialized_end = 1729 - _globals['_METADATAVALUE']._serialized_start = 1731 - _globals['_METADATAVALUE']._serialized_end = 1850 - _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_start = 1852 - _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_end = 1895 - _globals['_IMPORTEDTSSEGMENTINFO']._serialized_start = 1898 - _globals['_IMPORTEDTSSEGMENTINFO']._serialized_end = 2054 - _globals['_EXTENTVALUE']._serialized_start = 2057 - _globals['_EXTENTVALUE']._serialized_end = 2284 - _globals['_TERMVALUE']._serialized_start = 2286 - _globals['_TERMVALUE']._serialized_end = 2324 - _globals['_OFFSETRANGE']._serialized_start = 2326 - _globals['_OFFSETRANGE']._serialized_end = 2381 - _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_start = 2383 - _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_end = 2489 - _globals['_COMPACTIONVALUE']._serialized_start = 2492 - _globals['_COMPACTIONVALUE']._serialized_end = 2706 - _globals['_OBJECTVALUE']._serialized_start = 2709 - _globals['_OBJECTVALUE']._serialized_end = 2997 - _globals['_ROWVALUE']._serialized_start = 3000 - _globals['_ROWVALUE']._serialized_end = 3412 - _globals['_WRITEROW']._serialized_start = 3415 - _globals['_WRITEROW']._serialized_end = 3563 - _globals['_WRITEROWSREQUEST']._serialized_start = 3566 - _globals['_WRITEROWSREQUEST']._serialized_end = 3756 - _globals['_WRITEROWSRESPONSE']._serialized_start = 3758 - _globals['_WRITEROWSRESPONSE']._serialized_end = 3799 - _globals['_READROW']._serialized_start = 3802 - _globals['_READROW']._serialized_end = 3949 - _globals['_READROWSREQUEST']._serialized_start = 3952 - _globals['_READROWSREQUEST']._serialized_end = 4228 - _globals['_READROWSRESPONSE']._serialized_start = 4230 - _globals['_READROWSRESPONSE']._serialized_end = 4335 - _globals['_VALIDATEPARTITIONREQUEST']._serialized_start = 4338 - _globals['_VALIDATEPARTITIONREQUEST']._serialized_end = 4538 - _globals['_VALIDATEPARTITIONRESPONSE']._serialized_start = 4541 - _globals['_VALIDATEPARTITIONRESPONSE']._serialized_end = 4730 - _globals['_METASTOREANOMALY']._serialized_start = 4732 - _globals['_METASTOREANOMALY']._serialized_end = 4852 - _globals['_LISTCLOUDTOPICSREQUEST']._serialized_start = 4854 - _globals['_LISTCLOUDTOPICSREQUEST']._serialized_end = 4924 - _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_start = 4926 - _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_end = 5047 - _globals['_CLOUDTOPICINFO']._serialized_start = 5049 - _globals['_CLOUDTOPICINFO']._serialized_end = 5128 - _globals['_METASTORESERVICE']._serialized_start = 5863 - _globals['_METASTORESERVICE']._serialized_end = 6995 \ No newline at end of file + _globals['_METADATAVALUE']._serialized_start = 1732 + _globals['_METADATAVALUE']._serialized_end = 1870 + _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_start = 1872 + _globals['_IMPORTEDTSOBJECTLOCATION']._serialized_end = 1915 + _globals['_IMPORTEDTSSEGMENTINFO']._serialized_start = 1918 + _globals['_IMPORTEDTSSEGMENTINFO']._serialized_end = 2074 + _globals['_EXTENTVALUE']._serialized_start = 2077 + _globals['_EXTENTVALUE']._serialized_end = 2304 + _globals['_TERMVALUE']._serialized_start = 2306 + _globals['_TERMVALUE']._serialized_end = 2344 + _globals['_OFFSETRANGE']._serialized_start = 2346 + _globals['_OFFSETRANGE']._serialized_end = 2401 + _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_start = 2403 + _globals['_CLEANEDRANGEWITHTOMBSTONES']._serialized_end = 2509 + _globals['_COMPACTIONVALUE']._serialized_start = 2512 + _globals['_COMPACTIONVALUE']._serialized_end = 2726 + _globals['_OBJECTVALUE']._serialized_start = 2729 + _globals['_OBJECTVALUE']._serialized_end = 3017 + _globals['_ROWVALUE']._serialized_start = 3020 + _globals['_ROWVALUE']._serialized_end = 3432 + _globals['_WRITEROW']._serialized_start = 3435 + _globals['_WRITEROW']._serialized_end = 3583 + _globals['_WRITEROWSREQUEST']._serialized_start = 3586 + _globals['_WRITEROWSREQUEST']._serialized_end = 3776 + _globals['_WRITEROWSRESPONSE']._serialized_start = 3778 + _globals['_WRITEROWSRESPONSE']._serialized_end = 3819 + _globals['_READROW']._serialized_start = 3822 + _globals['_READROW']._serialized_end = 3969 + _globals['_READROWSREQUEST']._serialized_start = 3972 + _globals['_READROWSREQUEST']._serialized_end = 4248 + _globals['_READROWSRESPONSE']._serialized_start = 4250 + _globals['_READROWSRESPONSE']._serialized_end = 4355 + _globals['_VALIDATEPARTITIONREQUEST']._serialized_start = 4358 + _globals['_VALIDATEPARTITIONREQUEST']._serialized_end = 4558 + _globals['_VALIDATEPARTITIONRESPONSE']._serialized_start = 4561 + _globals['_VALIDATEPARTITIONRESPONSE']._serialized_end = 4750 + _globals['_METASTOREANOMALY']._serialized_start = 4752 + _globals['_METASTOREANOMALY']._serialized_end = 4872 + _globals['_LISTCLOUDTOPICSREQUEST']._serialized_start = 4874 + _globals['_LISTCLOUDTOPICSREQUEST']._serialized_end = 4944 + _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_start = 4946 + _globals['_LISTCLOUDTOPICSRESPONSE']._serialized_end = 5067 + _globals['_CLOUDTOPICINFO']._serialized_start = 5069 + _globals['_CLOUDTOPICINFO']._serialized_end = 5148 + _globals['_METASTORESERVICE']._serialized_start = 5883 + _globals['_METASTORESERVICE']._serialized_end = 7015 \ No newline at end of file diff --git a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi index 4dfead507708e..af4637faaa3f6 100644 --- a/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi +++ b/tests/rptest/clients/admin/proto/redpanda/core/admin/internal/cloud_topics/v1/metastore_pb2.pyi @@ -410,16 +410,19 @@ class MetadataValue(google.protobuf.message.Message): COMPACTION_EPOCH_FIELD_NUMBER: builtins.int SIZE_FIELD_NUMBER: builtins.int NUM_EXTENTS_FIELD_NUMBER: builtins.int + MIGRATING_FIELD_NUMBER: builtins.int start_offset: builtins.int next_offset: builtins.int compaction_epoch: builtins.int size: builtins.int num_extents: builtins.int + migrating: builtins.bool + 'True while the partition is mid tiered->cloud migration.' - def __init__(self, *, start_offset: builtins.int=..., next_offset: builtins.int=..., compaction_epoch: builtins.int=..., size: builtins.int=..., num_extents: builtins.int=...) -> None: + def __init__(self, *, start_offset: builtins.int=..., next_offset: builtins.int=..., compaction_epoch: builtins.int=..., size: builtins.int=..., num_extents: builtins.int=..., migrating: builtins.bool=...) -> None: ... - def ClearField(self, field_name: typing.Literal['compaction_epoch', b'compaction_epoch', 'next_offset', b'next_offset', 'num_extents', b'num_extents', 'size', b'size', 'start_offset', b'start_offset']) -> None: + def ClearField(self, field_name: typing.Literal['compaction_epoch', b'compaction_epoch', 'migrating', b'migrating', 'next_offset', b'next_offset', 'num_extents', b'num_extents', 'size', b'size', 'start_offset', b'start_offset']) -> None: ... Global___MetadataValue: typing_extensions.TypeAlias = MetadataValue From 67e51da0e71249e5851ee189714169b31067c0d2 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Fri, 5 Jun 2026 16:40:31 -0700 Subject: [PATCH 4/9] cloud_topics/l1: register imported segments through add_objects Let the metastore register tiered-storage segments as imported L1 extents by reference, reusing the existing add_objects extent-addition path rather than a dedicated command. - add_objects now handles imported objects: - skips the pre-registration check for imported objects (they reference an existing tiered-storage segment; nothing is written to L1) - copies imported_ts_location / imported_ts_delta onto the object and extent rows - sets start_offset from the first extent for an empty migrating partition (a non-zero start -- the TS->CT mount), rather than requiring offset 0 - the object builder's add_imported(ntp_metadata) registers one imported segment as a finished, single-extent object: with no L1 write to reserve it creates the object id and records it as finished without pre-registration, to be committed by a subsequent add_objects. meta.imported must be set; the read-replica metastore rejects it. Tests (both backends): a migrating partition adopts a non-zero start at the first imported extent; successive batches forward-append; a non-contiguous batch is rejected. Co-Authored-By: Claude Opus 4.8 --- src/v/cloud_topics/level_one/metastore/BUILD | 1 + .../level_one/metastore/lsm/state_update.cc | 57 +++++-- .../level_one/metastore/metastore.h | 16 ++ .../metastore/partition_validator.cc | 122 ++++++++++---- .../level_one/metastore/partition_validator.h | 8 + .../metastore/replicated_metastore.cc | 47 ++++++ .../level_one/metastore/simple_metastore.cc | 42 ++++- .../level_one/metastore/simple_metastore.h | 2 + .../level_one/metastore/state_update.cc | 69 +++++--- .../metastore/tests/state_update_test.cc | 149 +++++++++++++++++- .../read_replica/snapshot_metastore.cc | 6 + 11 files changed, 454 insertions(+), 65 deletions(-) diff --git a/src/v/cloud_topics/level_one/metastore/BUILD b/src/v/cloud_topics/level_one/metastore/BUILD index 382cc8a85b374..d24f73dcee95a 100644 --- a/src/v/cloud_topics/level_one/metastore/BUILD +++ b/src/v/cloud_topics/level_one/metastore/BUILD @@ -201,6 +201,7 @@ redpanda_cc_library( ":rpc_types", ":state_update", "//src/v/cloud_topics:logger", + "//src/v/cloud_topics/level_one/common:object_id", ], visibility = ["//visibility:public"], deps = [ diff --git a/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc b/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc index fca4879a75453..396c5629a2d30 100644 --- a/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc +++ b/src/v/cloud_topics/level_one/metastore/lsm/state_update.cc @@ -38,9 +38,10 @@ db_update_error wrap_read_err( } } -// Checks that each new object is pre-registered (exists with -// is_preregistration=true), then collects extents and committed object -// entries into the output maps. +// Checks that each new native object is pre-registered (exists with +// is_preregistration=true) -- imported objects reference an existing +// tiered-storage segment and skip that check -- then collects extents and +// committed object entries into the output maps. ss::future> validate_preregistered_and_collect( const chunked_vector& new_objects, @@ -48,16 +49,22 @@ validate_preregistered_and_collect( sorted_extents_by_tidp_t& out_extents, chunked_hash_map& out_objects) { for (const auto& o : new_objects) { - auto object_res = co_await state.get_object(o.oid); - if (!object_res.has_value()) { - co_return std::unexpected(db_update_error( - invalid_update, fmt::format("Error getting object {}", o.oid))); - } - if ( - !object_res->has_value() || !object_res->value().is_preregistration) { - co_return std::unexpected(db_update_error( - invalid_update, - fmt::format("object {} not pre-registered", o.oid))); + if (!o.imported_ts_location.has_value()) { + // Native objects must be pre-registered, but we allow imported + // objects to skip that step as it isn't useful. + auto object_res = co_await state.get_object(o.oid); + if (!object_res.has_value()) { + co_return std::unexpected(db_update_error( + invalid_update, + fmt::format("Error getting object {}", o.oid))); + } + if ( + !object_res->has_value() + || !object_res->value().is_preregistration) { + co_return std::unexpected(db_update_error( + invalid_update, + fmt::format("object {} not pre-registered", o.oid))); + } } auto data_size = o.collect_extents_by_tidp(&out_extents); out_objects.emplace( @@ -69,6 +76,7 @@ validate_preregistered_and_collect( .object_size = o.object_size, .last_updated = model::timestamp::now(), .is_preregistration = false, + .imported_ts_location = o.imported_ts_location, }); } co_return std::expected{}; @@ -660,8 +668,6 @@ add_objects_db_update::build_rows( chunked_hash_map verified_meta_vals; for (const auto& [tidp, extents] : new_extents_by_tp) { - // TODO: maybe we need some mount operation that adopts a partition log - // and allows it to start a specific offset. auto meta_res = co_await state.get_metadata(tidp); if (!meta_res.has_value()) { co_return std::unexpected(wrap_read_err( @@ -670,7 +676,21 @@ add_objects_db_update::build_rows( tidp)); } auto opt = meta_res.value(); - auto expected_next = opt ? opt->next_offset : kafka::offset{0}; + // An empty partition marked migrating (via set_migrating) adopts its + // log at the first incoming extent's base, allowing a non-zero start. + // Note, this may happen more than once if, during the migration + // process, segments happen to be gc'd past the latest point so far + // mirrored once more emptying the partition. In such a case, we want + // to still set expected_next based on the first new extent as there + // might be a gap. + // + // A brand-new (untracked) partition starts at 0; an existing non-empty + // one continues from its next_offset. + const bool fresh_migrating_mount = opt && opt->migrating + && opt->num_extents == 0; + auto expected_next = fresh_migrating_mount + ? extents.begin()->base_offset + : (opt ? opt->next_offset : kafka::offset{0}); if (extents.begin()->base_offset != expected_next) { // If the start of the new extents for this partition aren't @@ -689,7 +709,9 @@ add_objects_db_update::build_rows( extent_size_sum += extent.len; } verified_meta_vals[tidp] = metadata_row_value{ - .start_offset = opt ? opt->start_offset : kafka::offset{0}, + .start_offset = fresh_migrating_mount + ? extents.begin()->base_offset + : (opt ? opt->start_offset : kafka::offset{0}), .next_offset = kafka::next_offset(extents.rbegin()->last_offset), .compaction_epoch = opt ? opt->compaction_epoch : partition_state::compaction_epoch_t{0}, @@ -786,6 +808,7 @@ add_objects_db_update::build_rows( .filepos = entry.filepos, .len = entry.len, .oid = entry.oid, + .imported_ts_info = entry.imported_ts_info, }), }); } diff --git a/src/v/cloud_topics/level_one/metastore/metastore.h b/src/v/cloud_topics/level_one/metastore/metastore.h index 60fb366f77d9e..2f7ce6904cdf0 100644 --- a/src/v/cloud_topics/level_one/metastore/metastore.h +++ b/src/v/cloud_topics/level_one/metastore/metastore.h @@ -85,6 +85,11 @@ class metastore { model::timestamp max_timestamp; size_t pos; size_t size; + // Set when this extent is an imported tiered-storage segment (the + // TS->CT migration mirror): the segment descriptor (path + delta + + // term). The extent's Kafka offset bounds come from + // base_offset/last_offset above. nullopt for a native L1 extent. + std::optional imported; }; using ntp_metas_list_t = chunked_vector; @@ -147,6 +152,17 @@ class metastore { virtual std::expected add(object_id, object_metadata::ntp_metadata) = 0; + // Registers a single imported tiered-storage segment as a finished, + // single-extent object (the TS->CT migration mirror). Unlike the + // native path there is no L1 write to reserve, so the object is not + // pre-registered: this creates the object id, records it as finished, + // and returns it. `meta.imported` must be set. The object is added to + // the metastore by a subsequent add_objects() (which adopts a migrating + // partition's log at a non-zero start and skips the pre-registration + // check for imported objects). + virtual std::expected + add_imported(object_metadata::ntp_metadata) = 0; + // Tracks the given object as finished. Further calls to // get_or_create_object_for() will not return the finished object ID. virtual std::expected diff --git a/src/v/cloud_topics/level_one/metastore/partition_validator.cc b/src/v/cloud_topics/level_one/metastore/partition_validator.cc index c54de8d22a9ed..a9354367e99c7 100644 --- a/src/v/cloud_topics/level_one/metastore/partition_validator.cc +++ b/src/v/cloud_topics/level_one/metastore/partition_validator.cc @@ -513,13 +513,23 @@ partition_validator::validate_objects( co_return std::monostate{}; } for (const auto& oid : seen_objects) { + // The object entry is read from the metastore at most once per oid and + // only when needed: that read is the cost check_object_metadata exists + // to avoid. A native L1 object's storage path is derivable from the id + // alone, so on the common path we never touch the metastore. + std::optional obj; + bool obj_loaded = false; + if (opts.check_object_metadata) { auto obj_res = co_await reader_.get_object(oid); if (!obj_res.has_value()) { co_return std::unexpected(wrap_read_err( std::move(obj_res.error()), "reading object metadata")); } - if (!obj_res.value().has_value()) { + obj = std::move(obj_res.value()); + obj_loaded = true; + + if (!obj.has_value()) { result.record( anomaly_type::object_not_found, "extent references object {} which does not exist " @@ -527,7 +537,7 @@ partition_validator::validate_objects( oid); continue; } - if (obj_res.value().value().is_preregistration) { + if (obj.value().is_preregistration) { result.record( anomaly_type::object_preregistered, "extent references object {} which is still a " @@ -542,42 +552,98 @@ partition_validator::validate_objects( if (!opts.remote || !opts.bucket || !opts.as) { continue; } - auto path = object_path_factory::level_one_path(oid); - retry_chain_node rtc(*opts.as, 30s, 1s); - auto exists_fut = co_await ss::coroutine::as_future( - opts.remote->object_exists( - *opts.bucket, path, rtc, "partition_validator")); - if (exists_fut.failed()) { - auto ex = exists_fut.get_exception(); - auto ec = ssx::is_shutdown_exception(ex) ? errc::shutting_down - : errc::io_error; - co_return std::unexpected(error( - ec, - fmt::format( - "object_exists failed for {}: {}", - oid, - exists_fut.get_exception()))); + + // When the metadata has not been read (check_object_metadata is + // false), probe the native L1 path first and skip the metastore read + // entirely on a hit. Only on a miss do we read the entry: the extent + // may be an imported tiered-storage segment stored under its own path, + // which we then re-probe below. + if (!obj_loaded) { + auto l1_path = object_path_factory::level_one_path(oid); + auto present = co_await object_present_in_storage( + opts, oid, l1_path); + if (!present.has_value()) { + co_return std::unexpected(std::move(present.error())); + } + if (present.value()) { + continue; + } + auto obj_res = co_await reader_.get_object(oid); + if (!obj_res.has_value()) { + co_return std::unexpected(wrap_read_err( + std::move(obj_res.error()), "reading object metadata")); + } + obj = std::move(obj_res.value()); + obj_loaded = true; + if ( + !obj.has_value() + || !obj.value().imported_ts_location.has_value()) { + // Not an imported segment, so the L1 path was the right one + // and the object is genuinely missing from storage. + result.record( + anomaly_type::object_not_in_storage, + "object {} not found in cloud storage at {}", + oid, + l1_path()); + continue; + } + // Fall through to re-probe at the imported segment's path. + } + + // An imported extent points at the tiered-storage segment object by + // path (no native L1 object is written for it during migration), so + // verify it at that path instead of the L1 object path. + auto path + = obj.value().imported_ts_location.has_value() + ? cloud_storage_clients::object_key{obj.value() + .imported_ts_location + ->ts_path()} + : object_path_factory::level_one_path(oid); + auto present = co_await object_present_in_storage(opts, oid, path); + if (!present.has_value()) { + co_return std::unexpected(std::move(present.error())); } - auto dl_res = exists_fut.get(); - switch (dl_res) { - case cloud_io::download_result::success: - continue; - case cloud_io::download_result::notfound: + if (!present.value()) { result.record( anomaly_type::object_not_in_storage, "object {} not found in cloud storage at {}", oid, path()); - continue; - case cloud_io::download_result::timedout: - case cloud_io::download_result::failed: - co_return std::unexpected(error( - errc::io_error, - fmt::format("object_exists failed for {}: {}", oid, dl_res))); } } co_return std::monostate{}; } +ss::future> +partition_validator::object_present_in_storage( + const validate_partition_options& opts, + object_id oid, + const cloud_storage_clients::object_key& path) { + retry_chain_node rtc(*opts.as, 30s, 1s); + auto exists_fut = co_await ss::coroutine::as_future( + opts.remote->object_exists( + *opts.bucket, path, rtc, "partition_validator")); + if (exists_fut.failed()) { + auto ex = exists_fut.get_exception(); + auto ec = ssx::is_shutdown_exception(ex) ? errc::shutting_down + : errc::io_error; + co_return std::unexpected( + error(ec, fmt::format("object_exists failed for {}: {}", oid, ex))); + } + auto dl_res = exists_fut.get(); + switch (dl_res) { + case cloud_io::download_result::success: + co_return true; + case cloud_io::download_result::notfound: + co_return false; + case cloud_io::download_result::timedout: + case cloud_io::download_result::failed: + co_return std::unexpected(error( + errc::io_error, + fmt::format("object_exists failed for {}: {}", oid, dl_res))); + } + std::unreachable(); +} + } // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/metastore/partition_validator.h b/src/v/cloud_topics/level_one/metastore/partition_validator.h index 8c9b45207a1f5..51d6dfddc9bf7 100644 --- a/src/v/cloud_topics/level_one/metastore/partition_validator.h +++ b/src/v/cloud_topics/level_one/metastore/partition_validator.h @@ -156,6 +156,14 @@ class partition_validator { chunked_hash_set seen_objects, partition_validation_result& result); + /// HEAD-probe cloud storage for an object at `path`. Returns true if it + /// exists and false if it is absent; a timed-out or failed probe becomes + /// an error. `oid` is used only for diagnostics. + ss::future> object_present_in_storage( + const validate_partition_options& opts, + object_id oid, + const cloud_storage_clients::object_key& path); + state_reader& reader_; }; diff --git a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc index 5d7634f7f9587..5c09c74bb6029 100644 --- a/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/replicated_metastore.cc @@ -9,6 +9,7 @@ */ #include "cloud_topics/level_one/metastore/replicated_metastore.h" +#include "cloud_topics/level_one/common/object_id.h" #include "cloud_topics/level_one/metastore/leader_router.h" #include "cloud_topics/level_one/metastore/manifest_io.h" #include "cloud_topics/level_one/metastore/rpc_types.h" @@ -16,6 +17,7 @@ #include "cloud_topics/logger.h" #include +#include namespace cloud_topics::l1 { @@ -55,6 +57,14 @@ new_object meta_to_rpc_obj(const metastore::object_metadata& obj) { meta.max_timestamp = ntp_meta.max_timestamp; meta.filepos = ntp_meta.pos; meta.len = ntp_meta.size; + if (ntp_meta.imported.has_value()) { + // Split the imported descriptor onto the storage rows: the segment + // delta/term on the extent, the ts_path on the object (an imported + // object is single-extent, so its location is this extent's). + meta.imported_ts_info = to_segment_info(*ntp_meta.imported); + rpc_obj.imported_ts_location = to_object_location( + *ntp_meta.imported); + } topic_map[ntp_meta.tidp.partition] = std::move(meta); } @@ -107,6 +117,8 @@ class replicated_object_builder : public metastore::object_metadata_builder { std::expected remove_pending_object(object_id) override; std::expected add(object_id, metastore::object_metadata::ntp_metadata) override; + std::expected + add_imported(metastore::object_metadata::ntp_metadata) override; std::expected finish(object_id, size_t footer_pos, size_t object_size) override; bool is_empty() const override; @@ -246,6 +258,41 @@ replicated_object_builder::add( return {}; } +std::expected +replicated_object_builder::add_imported( + metastore::object_metadata::ntp_metadata ntp_meta) { + if (ntp_meta.base_offset > ntp_meta.last_offset) { + return std::unexpected( + error{fmt::format( + "Imported metadata has inverted offsets for partition {}: " + "base_offset {} > last_offset {}", + ntp_meta.tidp, + ntp_meta.base_offset, + ntp_meta.last_offset)}); + } + auto metastore_pid = fe_.metastore_partition(ntp_meta.tidp); + if (!metastore_pid) { + return std::unexpected( + error{"could not determine metastore partition for add_imported()"}); + } + // Imported objects reference an existing tiered-storage segment -- there is + // no L1 write to reserve -- so the id is created locally without + // pre-registration and recorded as a finished single-extent object for its + // metastore partition. + auto oid = create_object_id(); + auto object_size = ntp_meta.size; + metastore::object_metadata::ntp_metas_list_t metas; + metas.emplace_back(std::move(ntp_meta)); + partitions_[*metastore_pid].finished_objects_.emplace_back( + metastore::object_metadata{ + .oid = oid, + .footer_pos = 0, + .object_size = object_size, + .ntp_metas = std::move(metas), + }); + return oid; +} + std::expected replicated_object_builder::finish( object_id oid, size_t footer_pos, size_t object_size) { diff --git a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc index 481ec35daecff..304156fd3016b 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_metastore.cc +++ b/src/v/cloud_topics/level_one/metastore/simple_metastore.cc @@ -18,6 +18,8 @@ #include "container/chunked_vector.h" #include "model/fundamental.h" +#include + namespace cloud_topics::l1 { namespace { @@ -33,6 +35,7 @@ term_state_update_t make_terms_update(const metastore::term_offset_map_t& m) { } return ret; } + } // namespace ss::future> @@ -93,6 +96,35 @@ simple_object_builder::add( return {}; } +std::expected +simple_object_builder::add_imported( + metastore::object_metadata::ntp_metadata ntp_meta) { + if (ntp_meta.base_offset > ntp_meta.last_offset) { + return std::unexpected( + error{fmt::format( + "Imported metadata has inverted offsets for partition {}: " + "base_offset {} > last_offset {}", + ntp_meta.tidp, + ntp_meta.base_offset, + ntp_meta.last_offset)}); + } + // Imported objects reference an existing tiered-storage segment -- there is + // no L1 write to reserve -- so the id is created directly (no + // pre-registration) and recorded as a finished single-extent object. + auto oid = create_object_id(); + auto object_size = ntp_meta.size; + metastore::object_metadata::ntp_metas_list_t metas; + metas.emplace_back(std::move(ntp_meta)); + finished_objects_.emplace_back( + metastore::object_metadata{ + .oid = oid, + .footer_pos = 0, + .object_size = object_size, + .ntp_metas = std::move(metas), + }); + return oid; +} + std::expected simple_object_builder::finish( object_id oid, size_t footer_pos, size_t object_size) { @@ -137,13 +169,21 @@ new_object make_new_object(const metastore::object_metadata& o) { }; for (const auto& c : o.ntp_metas) { auto& extents = new_o.extent_metas[c.tidp.topic_id]; - extents[c.tidp.partition] = new_object::metadata{ + auto meta = new_object::metadata{ .base_offset = c.base_offset, .last_offset = c.last_offset, .max_timestamp = c.max_timestamp, .filepos = c.pos, .len = c.size, }; + if (c.imported.has_value()) { + // Split the imported descriptor onto the storage rows: the segment + // delta/term on the extent, the ts_path on the object (an imported + // object is single-extent, so its location is this extent's). + meta.imported_ts_info = to_segment_info(*c.imported); + new_o.imported_ts_location = to_object_location(*c.imported); + } + extents[c.tidp.partition] = std::move(meta); } return new_o; } diff --git a/src/v/cloud_topics/level_one/metastore/simple_metastore.h b/src/v/cloud_topics/level_one/metastore/simple_metastore.h index 8335a80052eab..be3a41a0d7d9c 100644 --- a/src/v/cloud_topics/level_one/metastore/simple_metastore.h +++ b/src/v/cloud_topics/level_one/metastore/simple_metastore.h @@ -37,6 +37,8 @@ class simple_object_builder : public metastore::object_metadata_builder { std::expected remove_pending_object(object_id) override; std::expected add(object_id, metastore::object_metadata::ntp_metadata) override; + std::expected + add_imported(metastore::object_metadata::ntp_metadata) override; std::expected finish(object_id, size_t footer_pos, size_t object_size) override; bool is_empty() const override; diff --git a/src/v/cloud_topics/level_one/metastore/state_update.cc b/src/v/cloud_topics/level_one/metastore/state_update.cc index 33c59bfe451f1..f1c0dd39d2cd2 100644 --- a/src/v/cloud_topics/level_one/metastore/state_update.cc +++ b/src/v/cloud_topics/level_one/metastore/state_update.cc @@ -84,6 +84,31 @@ void remove_extents_below_start_offset_for_tp( } } +// The offset a batch of extents must connect at for a partition. +// +// An empty partition marked migrating (via set_migrating) adopts its log at +// the first incoming extent's base, allowing a non-zero start. +// Note, this may happen more than once if, during the migration process, +// segments happen to be gc'd past the latest point so far mirrored once more +// emptying the partition. In such a case, we want to still adopt the first +// incoming extent's base as there might be a gap. +// +// A brand-new (untracked) partition starts at 0; an existing non-empty one +// continues from its next_offset. +kafka::offset expected_next_offset( + const state& s, + const model::topic_id_partition& tidp, + kafka::offset first_base) { + auto p = s.partition_state(tidp); + if (p.has_value() && !p->get().extents.empty()) { + return p->get().next_offset; + } + if (p.has_value() && p->get().migrating) { + return first_base; + } + return kafka::offset{0}; +} + // Validates the basic shape of new_objects against the state: // - non-empty // - every object is pre-registered and not yet finalized @@ -302,15 +327,22 @@ std::expected add_objects_update::can_apply( } sorted_extents_by_tidp_t new_extents; for (const auto& o : new_objects) { - auto it = state.objects.find(o.oid); - if (it == state.objects.end()) { - return std::unexpected( - stm_update_error{ - fmt::format("Object {} not pre-registered", o.oid)}); - } - if (!it->second.is_preregistration) { - return std::unexpected( - stm_update_error{fmt::format("Object {} already exists", o.oid)}); + if (!o.imported_ts_location.has_value()) { + // Native objects are reserved before being written, so they must + // already be pre-registered. An imported object references an + // existing tiered-storage segment (nothing is written to L1), so + // there is no reservation to honor -- it is added directly. + auto it = state.objects.find(o.oid); + if (it == state.objects.end()) { + return std::unexpected( + stm_update_error{ + fmt::format("Object {} not pre-registered", o.oid)}); + } + if (!it->second.is_preregistration) { + return std::unexpected( + stm_update_error{ + fmt::format("Object {} already exists", o.oid)}); + } } o.collect_extents_by_tidp(&new_extents); } @@ -318,11 +350,8 @@ std::expected add_objects_update::can_apply( chunked_hash_map corrected_next_offsets; for (const auto& [tidp, extents] : new_extents) { - // TODO: maybe we need some mount operation that adopts a partition log - // and allows it to start a specific offset. - auto p_state = state.partition_state(tidp); - auto expected_next = p_state ? p_state->get().next_offset - : kafka::offset{0}; + auto expected_next = expected_next_offset( + state, tidp, extents.begin()->base_offset); if (extents.begin()->base_offset != expected_next) { // If the start of the new extents for this partition aren't @@ -466,17 +495,21 @@ add_objects_update::apply(state& state) { .object_size = o.object_size, .last_updated = model::timestamp::now(), .is_preregistration = false, + .imported_ts_location = o.imported_ts_location, }; } for (const auto& [tidp, extents] : extents_by_tp) { - auto p_state = state.partition_state(tidp); - auto expected_next = p_state ? p_state->get().next_offset - : kafka::offset{0}; + auto expected_next = expected_next_offset( + state, tidp, extents.begin()->base_offset); if (extents.begin()->base_offset == expected_next) { auto& t_state = state.topic_to_state[tidp.topic_id]; auto& p_state = t_state.pid_to_state[tidp.partition]; // We've validated that all extents form a contiguous offset space. - // Accept them all. + // Accept them all. A migrating partition adopting its log at a + // non-zero start seeds its start offset at the first extent. + if (p_state.extents.empty() && p_state.migrating) { + p_state.start_offset = extents.begin()->base_offset; + } for (const auto& e : extents) { p_state.extents.emplace(e); } diff --git a/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc b/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc index 32bee675bb23f..94d1aece9c028 100644 --- a/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc +++ b/src/v/cloud_topics/level_one/metastore/tests/state_update_test.cc @@ -362,6 +362,88 @@ class StateUpdateParamTest : public ::testing::TestWithParam { return (*meta)->migrating; } + // Reads (start_offset, next_offset) for a partition from whichever backend + // is active. + std::pair + read_offsets(const model::topic_id_partition& tp) { + if (GetParam() == state_backend::simple) { + auto prt = state_.partition_state(tp); + if (!prt.has_value()) { + return {kafka::offset{}, kafka::offset{}}; + } + return {prt->get().start_offset, prt->get().next_offset}; + } + auto reader = state_reader(db_->create_snapshot()); + auto meta = reader.get_metadata(tp).get(); + if (!meta.has_value() || !meta->has_value()) { + return {kafka::offset{}, kafka::offset{}}; + } + return {(*meta)->start_offset, (*meta)->next_offset}; + } + + // Imported objects are added through the normal add_objects path (no + // pre-registration); the partition must be marked migrating first for a + // fresh partition to adopt its log at a non-zero start. Mirrors the + // add_imported builder path: a batch that does not connect at the tail + // (non-empty corrections) is reported as an error rather than silently + // dropped. + std::expected apply_append_imported( + chunked_vector objs, term_state_update_t terms) { + chunked_hash_map corrections; + if (GetParam() == state_backend::simple) { + auto update = add_objects_update::build( + state_, std::move(objs), std::move(terms), &corrections); + if (!update.has_value()) { + return std::unexpected( + stm_update_error{fmt::format("{}", update.error())}); + } + if (!corrections.empty()) { + return std::unexpected( + stm_update_error{"imported batch did not connect at tail"}); + } + return update->apply(state_); + } + add_objects_db_update db_update{ + .new_objects = std::move(objs), + .new_terms = std::move(terms), + }; + auto reader = state_reader(db_->create_snapshot()); + chunked_vector rows; + auto result = db_update.build_rows(reader, rows, &corrections).get(); + if (!result.has_value()) { + return std::unexpected( + stm_update_error{fmt::format("{}", result.error())}); + } + if (!corrections.empty()) { + return std::unexpected( + stm_update_error{"imported batch did not connect at tail"}); + } + apply_rows_to_db(rows); + return std::monostate{}; + } + + // Builds a single-extent imported new_object covering [base, last]. + static new_object make_imported_object( + object_id oid, + std::string_view tidp_sv, + kafka::offset base, + kafka::offset last) { + auto o = new_obj_builder(oid, 0, 100) + .add(tidp_sv, base, last, model::timestamp{1}, 0, 100) + .build(); + // ts_path is an object property; the segment descriptor (delta/term) is + // an extent property carried on each extent entry. + o.imported_ts_location = imported_ts_object_location{ + .ts_path = ts_segment_path{ + fmt::format("seg-{}-{}.log", base(), last())}}; + for (auto& [tid, parts] : o.extent_metas) { + for (auto& [p, meta] : parts) { + meta.imported_ts_info = imported_ts_segment_info{}; + } + } + return o; + } + std::expected apply_remove_topics(std::initializer_list topics_list) { chunked_vector topics; @@ -619,10 +701,75 @@ TEST_P(StateUpdateParamTest, TestSetMigratingPreservedAcrossAddObjects) { .add_term_start(tidp_a, 0_tm, 0_o) .build(); ASSERT_TRUE(apply_add_objects(std::move(update)).has_value()); - // Adding objects must not clobber the migration phase. + // Adding objects must not clobber the migrating flag. EXPECT_TRUE(read_migrating(tp)); } +// Importing on a migrating partition (add_imported committed via add_objects) +// seeds a fresh partition's start/next at the first imported extent's base -- a +// non-zero migration start. +TEST_P(StateUpdateParamTest, TestAppendImportedSeedsNonZeroStart) { + auto tp = model::topic_id_partition::from(tidp_a); + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + chunked_vector objs; + objs.push_back(make_imported_object(oid1, tidp_a, 100_o, 104_o)); + term_state_update_t terms; + terms[tp].push_back(term_start{.term_id = 0_tm, .start_offset = 100_o}); + ASSERT_TRUE( + apply_append_imported(std::move(objs), std::move(terms)).has_value()); + + auto [start, next] = read_offsets(tp); + EXPECT_EQ(start, 100_o); + EXPECT_EQ(next, 105_o); + EXPECT_TRUE(read_migrating(tp)); +} + +// Successive imported batches forward-append at the tail. +TEST_P(StateUpdateParamTest, TestAppendImportedForwardAppends) { + auto tp = model::topic_id_partition::from(tidp_a); + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + { + chunked_vector objs; + objs.push_back(make_imported_object(oid1, tidp_a, 100_o, 104_o)); + term_state_update_t terms; + terms[tp].push_back(term_start{.term_id = 0_tm, .start_offset = 100_o}); + ASSERT_TRUE( + apply_append_imported(std::move(objs), std::move(terms)).has_value()); + } + { + chunked_vector objs; + objs.push_back(make_imported_object(oid2, tidp_a, 105_o, 109_o)); + term_state_update_t terms; + terms[tp].push_back(term_start{.term_id = 0_tm, .start_offset = 105_o}); + ASSERT_TRUE( + apply_append_imported(std::move(objs), std::move(terms)).has_value()); + } + auto [start, next] = read_offsets(tp); + EXPECT_EQ(start, 100_o); + EXPECT_EQ(next, 110_o); +} + +// A batch that does not connect at the tail is rejected (the mirror is +// ordered). +TEST_P(StateUpdateParamTest, TestAppendImportedRejectsNonContiguous) { + auto tp = model::topic_id_partition::from(tidp_a); + ASSERT_TRUE(apply_set_migrating(tp, true).has_value()); + { + chunked_vector objs; + objs.push_back(make_imported_object(oid1, tidp_a, 100_o, 104_o)); + term_state_update_t terms; + terms[tp].push_back(term_start{.term_id = 0_tm, .start_offset = 100_o}); + ASSERT_TRUE( + apply_append_imported(std::move(objs), std::move(terms)).has_value()); + } + chunked_vector objs; + objs.push_back(make_imported_object(oid2, tidp_a, 200_o, 204_o)); + term_state_update_t terms; + terms[tp].push_back(term_start{.term_id = 0_tm, .start_offset = 200_o}); + auto res = apply_append_imported(std::move(objs), std::move(terms)); + EXPECT_FALSE(res.has_value()); +} + TEST_P(StateUpdateParamTest, TestDuplicateAddSingleUpdate) { ASSERT_TRUE(apply_preregister_objects({oid1, oid2}).has_value()); auto update = add_objects_builder() diff --git a/src/v/cloud_topics/read_replica/snapshot_metastore.cc b/src/v/cloud_topics/read_replica/snapshot_metastore.cc index 3269e1820d94b..468f8e80a8cfa 100644 --- a/src/v/cloud_topics/read_replica/snapshot_metastore.cc +++ b/src/v/cloud_topics/read_replica/snapshot_metastore.cc @@ -71,6 +71,8 @@ snapshot_metastore::get_first_ge( .object_size = obj.object_size, .first_offset = extent.base_offset, .last_offset = extent.last_offset, + .imported = l1::to_imported_ts_info( + obj.imported_ts_location, extent.imported_ts_info), }; } @@ -122,6 +124,8 @@ snapshot_metastore::get_first_ge( .object_size = obj.object_size, .first_offset = key->base_offset, .last_offset = extent.last_offset, + .imported = l1::to_imported_ts_info( + obj.imported_ts_location, extent.imported_ts_info), }; } } @@ -312,6 +316,8 @@ snapshot_metastore::get_extent_metadata_forwards( .oid = val.oid, .footer_pos = obj.footer_pos, .object_size = obj.object_size, + .imported = l1::to_imported_ts_info( + obj.imported_ts_location, val.imported_ts_info), }; } extents.push_back(std::move(em)); From ec471e9494940425724827116a894de02d3fbb21 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Thu, 25 Jun 2026 11:16:04 -0700 Subject: [PATCH 5/9] cloud_topics/l1: add a chunked data source A generic, header-only data source that fetches a byte range one fixed-size chunk at a time, lazily, and presents it as a single contiguous stream: only the chunks a reader actually consumes are fetched, so a read that stops early never pulls the tail. It is generic over the fetch callable and its error type, so it is not tied to tiered storage. A `use_chunk_aligned_reads` bool_class controls where the first chunk starts: when yes, chunk boundaries are chunk_size aligned so overlapping reads request identical (position, length) chunks that a downstream cache can dedup; when no, the first chunk starts exactly at the read's position (for a caller with no cache to dedup against). It has no default -- every caller states its choice. get() checks the abort source on every read, so a cancelled read fails loudly rather than ending as a truncated clean end-of-stream. Later commits build the reader and the open_object read path on top of this. Co-Authored-By: Claude Opus 4.8 --- src/v/cloud_topics/level_one/common/BUILD | 10 + .../level_one/common/chunk_data_source.h | 164 ++++++++++ .../cloud_topics/level_one/common/tests/BUILD | 17 ++ .../common/tests/chunk_data_source_test.cc | 282 ++++++++++++++++++ 4 files changed, 473 insertions(+) create mode 100644 src/v/cloud_topics/level_one/common/chunk_data_source.h create mode 100644 src/v/cloud_topics/level_one/common/tests/chunk_data_source_test.cc diff --git a/src/v/cloud_topics/level_one/common/BUILD b/src/v/cloud_topics/level_one/common/BUILD index fa7e98c15ef7f..a50bc24c4566c 100644 --- a/src/v/cloud_topics/level_one/common/BUILD +++ b/src/v/cloud_topics/level_one/common/BUILD @@ -152,3 +152,13 @@ redpanda_cc_library( "@seastar", ], ) + +redpanda_cc_library( + name = "chunk_data_source", + hdrs = ["chunk_data_source.h"], + deps = [ + "//src/v/base", + "@fmt", + "@seastar", + ], +) diff --git a/src/v/cloud_topics/level_one/common/chunk_data_source.h b/src/v/cloud_topics/level_one/common/chunk_data_source.h new file mode 100644 index 0000000000000..facc3aebb40ea --- /dev/null +++ b/src/v/cloud_topics/level_one/common/chunk_data_source.h @@ -0,0 +1,164 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ +#pragma once + +#include "base/vassert.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace cloud_topics::l1 { + +/// Controls where the first chunk starts. When yes, chunk boundaries are +/// chunk_size aligned, so overlapping reads request identical +/// (position, length) chunks that a downstream cache can dedup. When no, the +/// first chunk starts exactly at start_pos -- for a caller with no cache to +/// dedup against, which would otherwise fetch and discard the bytes between the +/// chunk boundary and start_pos. +using use_chunk_aligned_reads + = ss::bool_class; + +/// A data source that fetches the byte range [start_pos, start_pos+total_len) +/// one fixed-size chunk at a time, lazily, and presents the result as a single +/// contiguous stream. +/// +/// Only the chunks actually consumed are fetched: a reader that stops early +/// simply stops pulling, so the un-read tail is never fetched. Chunks are fixed +/// byte ranges and need not be aligned to any structure in the data -- the +/// source concatenates one chunk's tail with the next chunk's head, so a record +/// spanning a boundary is read seamlessly. EOF is signalled only at the true +/// end (start_pos+total_len), so a consumer never mistakes a chunk boundary for +/// the end of the data. +/// +/// `Fetch` is any callable +/// (size_t file_position, size_t length, ss::abort_source*) +/// -> ss::future, E>> +/// for any formattable error `E`. A chunk-fetch failure (an unexpected result) +/// surfaces as an exception from get(); a record-batch reader propagates it +/// (its read API has no error variant), matching how such readers already +/// report stream errors. +template +class chunk_data_source final : public ss::data_source_impl { +public: + /// See use_chunk_aligned_reads. When yes the first fetch is aligned + /// at or before start_pos and the leading `start_pos % chunk_size` bytes + /// are skipped; when no the first chunk starts exactly at start_pos with no + /// skipped head. + chunk_data_source( + Fetch fetch, + size_t start_pos, + size_t total_len, + size_t chunk_size, + ss::abort_source* as, + use_chunk_aligned_reads aligned) + : _fetch(std::move(fetch)) + , _end(start_pos + total_len) + , _chunk_size(chunk_size) + , _next_chunk_start( + aligned ? start_pos - (start_pos % chunk_size) : start_pos) + , _skip_head(aligned ? start_pos % chunk_size : 0) + , _as(as) { + vassert( + chunk_size > 0, "chunk_data_source requires a non-zero chunk size"); + } + + ss::future> get() override { + while (true) { + // Surface an abort as an exception on every read -- including while + // draining a buffered chunk or between chunks -- so a cancelled + // read fails loudly rather than terminating as a (truncated) clean + // end-of-stream. + if (_as != nullptr) { + _as->check(); + } + if (!_cur.has_value()) { + if (_next_chunk_start >= _end) { + // Past the requested range: true EOF. + co_return ss::temporary_buffer{}; + } + auto chunk_end = std::min( + _next_chunk_start + _chunk_size, _end); + auto len = chunk_end - _next_chunk_start; + auto stream = co_await _fetch(_next_chunk_start, len, _as); + if (!stream.has_value()) { + throw std::runtime_error( + fmt::format( + "failed to fetch chunk at byte {} (len {}): {}", + _next_chunk_start, + len, + stream.error())); + } + _cur = std::move(*stream); + if (_skip_head > 0) { + co_await _cur->skip(_skip_head); + _skip_head = 0; + } + _next_chunk_start += _chunk_size; + } + auto buf = co_await _cur->read(); + if (buf.empty()) { + // Current chunk exhausted; advance to the next. + co_await _cur->close(); + _cur.reset(); + continue; + } + co_return buf; + } + } + + ss::future<> close() override { + if (_cur.has_value()) { + co_await _cur->close(); + _cur.reset(); + } + } + +private: + Fetch _fetch; + size_t _end; + size_t _chunk_size; + // Start of the next chunk to fetch. When chunk-aligned this is the chunk + // boundary at or before start_pos; otherwise it is start_pos itself. + size_t _next_chunk_start; + // Bytes to discard from the first chunk so the stream starts at start_pos + // (non-zero only when chunk-aligned and start_pos sits mid-chunk). Zeroed + // after the first chunk. + size_t _skip_head; + std::optional> _cur; + ss::abort_source* _as; +}; + +/// Deduce `Fetch` and wrap a chunk_data_source in an ss::data_source. +template +ss::data_source make_chunk_data_source( + Fetch fetch, + size_t start_pos, + size_t total_len, + size_t chunk_size, + ss::abort_source* as, + use_chunk_aligned_reads aligned) { + return ss::data_source{std::make_unique>( + std::move(fetch), start_pos, total_len, chunk_size, as, aligned)}; +} + +} // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/common/tests/BUILD b/src/v/cloud_topics/level_one/common/tests/BUILD index db21f449d655b..c041e3bf516a6 100644 --- a/src/v/cloud_topics/level_one/common/tests/BUILD +++ b/src/v/cloud_topics/level_one/common/tests/BUILD @@ -77,3 +77,20 @@ redpanda_cc_gtest( "@seastar", ], ) + +redpanda_cc_gtest( + name = "chunk_data_source_test", + timeout = "short", + srcs = [ + "chunk_data_source_test.cc", + ], + deps = [ + "//src/v/bytes:iobuf", + "//src/v/bytes:iostream", + "//src/v/cloud_topics/level_one/common:abstract_io", + "//src/v/cloud_topics/level_one/common:chunk_data_source", + "//src/v/test_utils:gtest", + "@googletest//:gtest", + "@seastar", + ], +) diff --git a/src/v/cloud_topics/level_one/common/tests/chunk_data_source_test.cc b/src/v/cloud_topics/level_one/common/tests/chunk_data_source_test.cc new file mode 100644 index 0000000000000..3a9c85f7e0bec --- /dev/null +++ b/src/v/cloud_topics/level_one/common/tests/chunk_data_source_test.cc @@ -0,0 +1,282 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ + +#include "bytes/iobuf.h" +#include "bytes/iostream.h" +#include "cloud_topics/level_one/common/abstract_io.h" +#include "cloud_topics/level_one/common/chunk_data_source.h" + +#include +#include + +#include + +#include +#include +#include + +using namespace cloud_topics::l1; + +namespace { + +// A deterministic byte sequence so reads can be checked positionally. +std::string make_bytes(size_t n) { + std::string s(n, '\0'); + for (size_t i = 0; i < n; ++i) { + s[i] = static_cast(i % 251); + } + return s; +} + +iobuf to_iobuf(const std::string& s) { + iobuf b; + b.append(s.data(), s.size()); + return b; +} + +// A fetch over an in-memory buffer that records every requested (pos, len) so +// tests can assert which chunks were fetched and in what granularity. The fetch +// is returned as a plain lambda -- chunk_data_source is generic over the fetch +// type (io::errc here is just this test's error type). +struct recording_fetcher { + iobuf data; + std::vector> calls; + + explicit recording_fetcher(const std::string& s) + : data(to_iobuf(s)) {} + + auto fn() { + return + [this](size_t pos, size_t len, ss::abort_source*) + -> ss::future, io::errc>> { + calls.emplace_back(pos, len); + co_return make_iobuf_input_stream(data.share(pos, len)); + }; + } + + static auto failing_fn() { + return + [](size_t, size_t, ss::abort_source*) + -> ss::future, io::errc>> { + co_return std::unexpected(io::errc::cloud_op_error); + }; + } +}; + +template +ss::input_stream make_stream( + Fetch fetch, + size_t start, + size_t total, + size_t chunk_size, + ss::abort_source& as, + use_chunk_aligned_reads aligned) { + return ss::input_stream{make_chunk_data_source( + std::move(fetch), start, total, chunk_size, &as, aligned)}; +} + +std::string read_all(ss::input_stream& in) { + std::string out; + while (true) { + auto buf = in.read().get(); + if (buf.empty()) { + break; + } + out.append(buf.get(), buf.size()); + } + return out; +} + +} // namespace + +// A full read across many chunks yields exactly the requested byte range, i.e. +// chunking is transparent: the stream is byte-identical to a single download. +TEST(ChunkDataSourceTest, FullReadConcatenatesChunks) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 1000, 256, as, use_chunk_aligned_reads::yes); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out, bytes); + // ceil(1000/256) = 4 chunks: (0,256)(256,256)(512,256)(768,232). + ASSERT_EQ(f.calls.size(), 4u); + EXPECT_EQ(f.calls[0], (std::pair{0, 256})); + EXPECT_EQ(f.calls[1], (std::pair{256, 256})); + EXPECT_EQ(f.calls[2], (std::pair{512, 256})); + EXPECT_EQ(f.calls[3], (std::pair{768, 232})); +} + +// A seek into the middle of a chunk: the first fetch is chunk-aligned and the +// leading bytes before the seek point are skipped. +TEST(ChunkDataSourceTest, MidChunkSeekSkipsHead) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + // start at 300 (mid second chunk), read to end. + auto in = make_stream( + f.fn(), 300, 700, 256, as, use_chunk_aligned_reads::yes); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out, bytes.substr(300)); + // First fetch is the chunk containing 300: [256, 512). + ASSERT_FALSE(f.calls.empty()); + EXPECT_EQ(f.calls[0], (std::pair{256, 256})); +} + +// use_chunk_aligned_reads=false: a mid-chunk start reads exactly from start_pos +// -- no chunk alignment and no skipped head, so no bytes before the start are +// fetched (contrast MidChunkSeekSkipsHead, which fetches the [256, 512) chunk +// and discards the leading 44 bytes). Chunks then advance by chunk_size +// from the start. +TEST(ChunkDataSourceTest, UnalignedStartReadsFromStartPos) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 300, 700, 256, as, use_chunk_aligned_reads::no); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out, bytes.substr(300)); + // First fetch starts at 300 itself; subsequent chunks advance by 256. + ASSERT_EQ(f.calls.size(), 3u); + EXPECT_EQ(f.calls[0], (std::pair{300, 256})); + EXPECT_EQ(f.calls[1], (std::pair{556, 256})); + EXPECT_EQ(f.calls[2], (std::pair{812, 188})); +} + +// When start_pos already sits on a chunk boundary the flag makes no difference: +// use_chunk_aligned_reads::no fetches the same chunks as ::yes. +TEST(ChunkDataSourceTest, UnalignedStartOnBoundaryMatchesAligned) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 1000, 256, as, use_chunk_aligned_reads::no); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out, bytes); + ASSERT_EQ(f.calls.size(), 4u); + EXPECT_EQ(f.calls[0], (std::pair{0, 256})); + EXPECT_EQ(f.calls[1], (std::pair{256, 256})); + EXPECT_EQ(f.calls[2], (std::pair{512, 256})); + EXPECT_EQ(f.calls[3], (std::pair{768, 232})); +} + +// A read that stops early only fetches the chunks it touches; the tail is never +// downloaded. This is the win that a whole-suffix download cannot give, and the +// reason a naive length cap (which would EOF early) is unnecessary. +TEST(ChunkDataSourceTest, EarlyTerminationFetchesOnlyTouchedChunks) { + auto bytes = make_bytes(10000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 10000, 256, as, use_chunk_aligned_reads::yes); + + auto head = in.read_exactly(50).get(); + in.close().get(); + + EXPECT_EQ(std::string(head.get(), head.size()), bytes.substr(0, 50)); + // Only the first chunk was needed; a whole-suffix download would have + // pulled all ceil(10000/256)=40 chunks. + EXPECT_EQ(f.calls.size(), 1u); +} + +// A segment smaller than one chunk is a single fetch. +TEST(ChunkDataSourceTest, SingleChunkSegment) { + auto bytes = make_bytes(100); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 100, 256, as, use_chunk_aligned_reads::yes); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out, bytes); + ASSERT_EQ(f.calls.size(), 1u); + EXPECT_EQ(f.calls[0], (std::pair{0, 100})); +} + +// When the length is an exact multiple of the chunk size, EOF lands exactly at +// the end -- it is not mistaken for a chunk boundary. +TEST(ChunkDataSourceTest, ExactMultipleEndsAtTotal) { + auto bytes = make_bytes(512); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 512, 256, as, use_chunk_aligned_reads::yes); + auto out = read_all(in); + in.close().get(); + + EXPECT_EQ(out.size(), 512u); + EXPECT_EQ(out, bytes); + ASSERT_EQ(f.calls.size(), 2u); + EXPECT_EQ(f.calls[1], (std::pair{256, 256})); +} + +// A fetch failure surfaces as an exception from the stream read. +TEST(ChunkDataSourceTest, FetchFailurePropagates) { + ss::abort_source as; + auto in = make_stream( + recording_fetcher::failing_fn(), + 0, + 1000, + 256, + as, + use_chunk_aligned_reads::yes); + EXPECT_THROW(in.read().get(), std::runtime_error); + in.close().get(); +} + +// A mid-stream abort surfaces as an exception rather than a clean EOF: the +// source checks the abort source on every get(), so a cancelled read fails +// loudly instead of looking like a (truncated) end-of-stream -- otherwise a +// cancelled caller could commit a partial read as if it had read everything. +TEST(ChunkDataSourceTest, AbortSurfacesAsError) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 1000, 256, as, use_chunk_aligned_reads::yes); + + // First read succeeds, then abort with bytes still unread. + auto first = in.read().get(); + EXPECT_FALSE(first.empty()); + as.request_abort(); + + EXPECT_THROW( + { + while (!in.read().get().empty()) { + } + }, + ss::abort_requested_exception); + in.close().get(); +} + +// An abort before the first read fails the very first get(), before any chunk +// is fetched -- the abort check sits at the top of get(), ahead of the fetch. +TEST(ChunkDataSourceTest, AbortBeforeFirstReadFetchesNothing) { + auto bytes = make_bytes(1000); + recording_fetcher f(bytes); + ss::abort_source as; + auto in = make_stream( + f.fn(), 0, 1000, 256, as, use_chunk_aligned_reads::yes); + + as.request_abort(); + EXPECT_THROW(in.read().get(), ss::abort_requested_exception); + in.close().get(); + + EXPECT_TRUE(f.calls.empty()); +} From 37e1b7cc0a54f9c1d2ad8a6c0da66db80d1c1d16 Mon Sep 17 00:00:00 2001 From: Samuel Just Date: Mon, 15 Jun 2026 17:29:27 -0700 Subject: [PATCH 6/9] cloud_topics/l1: read L1 objects through an open_object/object_handle abstraction Introduce open_object() and the object_handle/object_index abstraction for reading an L1 object: a handle exposes an index (seek by offset/timestamp to a byte position) and opens readers at a seek result. This commit wires the native L1 object path (footer index + record reader). Chunking lives here, above the cache, not inside read_object. read_object is a single-range primitive -- fetch exactly the requested bytes, via the cache or (skip_cache) a single cache-bypassing ranged GET. open_object wraps that per-range fetch in a chunk_data_source sized for the read: a cache-bypassing read is bounded by the streaming chunk size (request-relative), while a cached native read is left un-chunked because read_object caches the whole extent to a file, which already bounds memory. Keeping the chunking in open_object lets a later imported read chunk at the cache-chunk granularity for dedup, independently of how the cache-bypass path fragments a read. Co-Authored-By: Claude Opus 4.8 --- src/v/cloud_topics/level_one/common/BUILD | 33 +- .../level_one/common/abstract_io.cc | 2 +- .../level_one/common/abstract_io.h | 8 +- .../cloud_topics/level_one/common/fake_io.cc | 12 +- src/v/cloud_topics/level_one/common/fake_io.h | 15 + .../cloud_topics/level_one/common/file_io.cc | 208 ++++------- src/v/cloud_topics/level_one/common/file_io.h | 6 + .../level_one/common/file_io_probe.cc | 4 + .../level_one/common/file_io_probe.h | 5 + .../level_one/common/object_handle.cc | 53 +++ .../level_one/common/object_handle.h | 121 +++++++ .../level_one/common/open_object.cc | 96 +++++ .../level_one/common/open_object.h | 36 ++ .../cloud_topics/level_one/common/tests/BUILD | 25 ++ .../common/tests/file_io_stream_test.cc | 58 +-- .../common/tests/open_object_test.cc | 341 ++++++++++++++++++ .../level_one/frontend_reader/BUILD | 3 +- .../frontend_reader/level_one_reader.cc | 207 ++++------- .../frontend_reader/level_one_reader.h | 26 +- .../frontend_reader/level_one_reader_probe.cc | 4 - .../frontend_reader/level_one_reader_probe.h | 5 - .../level_one/metastore/tests/BUILD | 1 + 22 files changed, 922 insertions(+), 347 deletions(-) create mode 100644 src/v/cloud_topics/level_one/common/object_handle.cc create mode 100644 src/v/cloud_topics/level_one/common/object_handle.h create mode 100644 src/v/cloud_topics/level_one/common/open_object.cc create mode 100644 src/v/cloud_topics/level_one/common/open_object.h create mode 100644 src/v/cloud_topics/level_one/common/tests/open_object_test.cc diff --git a/src/v/cloud_topics/level_one/common/BUILD b/src/v/cloud_topics/level_one/common/BUILD index a50bc24c4566c..18a47bab42ca3 100644 --- a/src/v/cloud_topics/level_one/common/BUILD +++ b/src/v/cloud_topics/level_one/common/BUILD @@ -101,6 +101,37 @@ redpanda_cc_library( ], ) +redpanda_cc_library( + name = "object_handle", + srcs = ["object_handle.cc"], + hdrs = ["object_handle.h"], + deps = [ + ":abstract_io", + ":object", + ":object_id", + "//src/v/model", + "@seastar", + ], +) + +redpanda_cc_library( + name = "open_object", + srcs = ["open_object.cc"], + hdrs = ["open_object.h"], + implementation_deps = [ + ":chunk_data_source", + ":object", + "//src/v/cloud_topics:logger", + "//src/v/config", + ], + deps = [ + ":abstract_io", + ":object_handle", + ":object_id", + "@seastar", + ], +) + redpanda_cc_library( name = "file_io", srcs = [ @@ -113,6 +144,7 @@ redpanda_cc_library( ], implementation_deps = [ "//src/v/base", + "//src/v/bytes:iostream", ], visibility = [ "//src/v/cloud_topics:__pkg__", @@ -144,7 +176,6 @@ redpanda_cc_library( deps = [ ":abstract_io", ":object_id", - ":object_utils", "//src/v/bytes:iobuf", "//src/v/bytes:iostream", "//src/v/cloud_storage_clients", diff --git a/src/v/cloud_topics/level_one/common/abstract_io.cc b/src/v/cloud_topics/level_one/common/abstract_io.cc index 9863c343de8a5..e154a312ea589 100644 --- a/src/v/cloud_topics/level_one/common/abstract_io.cc +++ b/src/v/cloud_topics/level_one/common/abstract_io.cc @@ -18,7 +18,7 @@ ss::future> io::read_file(staging_file* file) { return file->input_stream(); } -ss::future> io::read_object_as_iobuf( +ss::future> io::fetch_native_footer( object_extent extent, ss::abort_source* as, cloud_io::group_id gid, diff --git a/src/v/cloud_topics/level_one/common/abstract_io.h b/src/v/cloud_topics/level_one/common/abstract_io.h index 9b63fec0f6283..22ad57f7e9c26 100644 --- a/src/v/cloud_topics/level_one/common/abstract_io.h +++ b/src/v/cloud_topics/level_one/common/abstract_io.h @@ -95,9 +95,11 @@ class io { cloud_io::group_id g, bool skip_cache = false) = 0; - // The same as `read_object` except that instead of returning an input - // stream, the data is fully buffered into an `iobuf`. - virtual ss::future> read_object_as_iobuf( + // Read a native L1 object's footer region (the passed extent) fully + // buffered into an `iobuf`, so open_object can parse the footer index. + // Like read_object, may be served from or populate the cache unless + // `skip_cache` is set. + virtual ss::future> fetch_native_footer( object_extent, ss::abort_source*, cloud_io::group_id g, diff --git a/src/v/cloud_topics/level_one/common/fake_io.cc b/src/v/cloud_topics/level_one/common/fake_io.cc index 9ca1892f999d2..5520f3aca6b45 100644 --- a/src/v/cloud_topics/level_one/common/fake_io.cc +++ b/src/v/cloud_topics/level_one/common/fake_io.cc @@ -116,8 +116,16 @@ fake_io::read_object( ss::abort_source*, [[maybe_unused]] cloud_io::group_id gid, // fake_io keeps everything in memory and never caches, so it is always - // effectively a streaming read regardless of this flag. - [[maybe_unused]] bool skip_cache) { + // effectively a streaming read regardless of this flag. It is still recorded + // so tests can assert how open_object chunks a read and whether it bypasses + // the cache. + bool skip_cache) { + _read_object_calls.push_back( + read_object_call{ + .position = extent.position, + .size = extent.size, + .skip_cache = skip_cache, + .imported = extent.imported.has_value()}); co_return get_object(extent.id) .transform( [&extent]( diff --git a/src/v/cloud_topics/level_one/common/fake_io.h b/src/v/cloud_topics/level_one/common/fake_io.h index c0a4514508437..53199df870f24 100644 --- a/src/v/cloud_topics/level_one/common/fake_io.h +++ b/src/v/cloud_topics/level_one/common/fake_io.h @@ -14,6 +14,8 @@ #include "bytes/iobuf.h" #include "cloud_topics/level_one/common/abstract_io.h" +#include + namespace cloud_topics::l1 { // The IO implementation that is entirely in-memory, used for testing. @@ -51,8 +53,21 @@ class fake_io : public io { // Return a list of the object IDs that haven't been removed. chunked_vector list_objects() const; + /// A recorded read_object call, so tests can assert how open_object chunks + /// a read (chunk size / chunk alignment) and whether it bypasses the cache. + struct read_object_call { + size_t position; + size_t size; + bool skip_cache; + bool imported; + }; + const std::vector& read_object_calls() const { + return _read_object_calls; + } + private: absl::btree_map _storage; + std::vector _read_object_calls; }; } // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/common/file_io.cc b/src/v/cloud_topics/level_one/common/file_io.cc index f0240ca27ce28..119c8f1281768 100644 --- a/src/v/cloud_topics/level_one/common/file_io.cc +++ b/src/v/cloud_topics/level_one/common/file_io.cc @@ -10,6 +10,7 @@ #include "cloud_topics/level_one/common/file_io.h" +#include "bytes/iostream.h" #include "cloud_io/io_result.h" #include "cloud_io/remote.h" #include "cloud_storage_clients/client.h" @@ -20,7 +21,6 @@ #include "config/configuration.h" #include -#include #include #include #include @@ -96,117 +96,45 @@ struct one_time_stream_provider : public stream_provider { // A streaming download source for L1 objects that chunks downloads per // `chunk_size`. This may result in many GET requests per object download. // -// The object's byte range is read one bounded chunk at a time. `get()` fully -// buffers the next chunk via `download_stream` (a separate ranged GET), which -// returns the lease to the pool as soon as the chunk's body has been read off -// the connection. The buffered chunk is then served from memory, lease-free, -// while subsequent `get()`s drain it; only once it is exhausted is the next -// chunk fetched. Peak in-memory bytes and the connection-hold duration per read -// are both bounded by `chunk_size`. -class streaming_download_source final : public ss::data_source_impl { -public: - // chunk_size bounds both the peak in-memory bytes held per read and the - // span of object bytes downloaded under a single held lease. _next_pos and - // _last_pos are initialized to inclusive byte ranges. - streaming_download_source( - cloud_io::remote* remote, - cloud_storage_clients::bucket_name bucket, - cloud_storage_clients::object_key key, - cloud_storage_clients::http_byte_range range, - ss::abort_source& as, - cloud_io::group_id gid, - size_t chunk_size) - : _remote(remote) - , _bucket(std::move(bucket)) - , _key(std::move(key)) - , _next_pos(range.first) - , _last_pos(range.second) - , _as(as) - , _gid(gid) - , _chunk_size(chunk_size) {} - - streaming_download_source(const streaming_download_source&) = delete; - streaming_download_source& - operator=(const streaming_download_source&) = delete; - streaming_download_source(streaming_download_source&&) = delete; - streaming_download_source& operator=(streaming_download_source&&) = delete; - ~streaming_download_source() override = default; - - ss::future> get() override { - while (true) { - _as.check(); - if (!_current.empty()) { - auto buf = std::move(_current.front()); - _current.pop_front(); - co_return buf; - } - if (_next_pos > _last_pos) { - // An empty buffer signals end-of-stream. - co_return ss::temporary_buffer(); - } - co_await download_next_chunk(); - } - } - - ss::future<> close() override { return ss::now(); } - -private: - // Fetches the next chunk of the object's byte range into `_current`. The - // lease is released when `download_stream` returns, before the buffered - // chunk is served. - ss::future<> download_next_chunk() { - static constexpr auto timeout = 10s; - static constexpr auto backoff = 100ms; - const auto chunk_first = _next_pos; - const auto chunk_last = std::min( - chunk_first + _chunk_size - 1, _last_pos); - _next_pos = chunk_last + 1; - retry_chain_node root(_as, ss::lowres_clock::now() + timeout, backoff); - - cloud_io::try_consume_stream consumer = - [this](uint64_t /*content_length*/, ss::input_stream stream) { - return drain_chunk(std::move(stream)); - }; - - auto result_fut = co_await ss::coroutine::as_future( - _remote->download_stream( - cloud_io::transfer_details{ - .bucket = _bucket, - .key = _key, - .parent_rtc = root, - }, - consumer, - "l1_stream_download", - /*acquire_hydration_units=*/true, - cloud_storage_clients::http_byte_range{chunk_first, chunk_last}, - {}, - _gid)); - if (result_fut.failed()) { - std::rethrow_exception(result_fut.get_exception()); - } - auto result = result_fut.get(); - if (result != cloud_io::download_result::success) { - throw std::runtime_error( - fmt::format("L1 streaming download failed: {}", result)); - } - } +// Cache-bypassing ranged download of the object byte range [pos, pos+len) at +// `key`, fully buffered into an input_stream. `download_stream` (a single +// ranged GET) returns the lease to the pool as soon as the range body has been +// read off the connection, before the buffered bytes are served -- so both the +// peak in-memory bytes and the connection-hold duration are bounded by the +// range length. Backs read_object's skip_cache path without populating the +// cloud cache; the range is bounded because open_object invokes read_object +// once per chunk. A download failure throws (rethrowing the original +// exception, or a runtime_error for a non-success result), surfaced from the +// caller's read of the returned stream. +ss::future, io::errc>> +download_range_bypassing_cache( + cloud_io::remote* remote, + const cloud_storage_clients::bucket_name& bucket, + const cloud_storage_clients::object_key& key, + cloud_io::group_id gid, + size_t pos, + size_t len, + ss::abort_source* as) { + static constexpr auto timeout = 10s; + static constexpr auto backoff = 100ms; + retry_chain_node root(*as, ss::lowres_clock::now() + timeout, backoff); - // Reads a chunk's whole body into `_current`. Cleared up-front so that a - // retried download re-buffers from scratch rather than appending to a - // partially-read result. - ss::future drain_chunk(ss::input_stream stream) { - _current.clear(); + iobuf buf; + cloud_io::try_consume_stream consumer = + [&buf, as]( + uint64_t /*content_length*/, + ss::input_stream stream) -> ss::future { uint64_t total = 0; std::exception_ptr ex; try { while (true) { - _as.check(); - auto buf = co_await stream.read(); - if (buf.empty()) { + as->check(); + auto b = co_await stream.read(); + if (b.empty()) { break; } - total += buf.size(); - _current.push_back(std::move(buf)); + total += b.size(); + buf.append(std::move(b)); } } catch (...) { ex = std::current_exception(); @@ -216,18 +144,30 @@ class streaming_download_source final : public ss::data_source_impl { std::rethrow_exception(ex); } co_return total; + }; + + auto result_fut = co_await ss::coroutine::as_future(remote->download_stream( + cloud_io::transfer_details{ + .bucket = bucket, + .key = key, + .parent_rtc = root, + }, + consumer, + "l1_stream_download", + /*acquire_hydration_units=*/true, + cloud_storage_clients::http_byte_range{pos, pos + len - 1}, + {}, + gid)); + if (result_fut.failed()) { + std::rethrow_exception(result_fut.get_exception()); } - - cloud_io::remote* _remote; - cloud_storage_clients::bucket_name _bucket; - cloud_storage_clients::object_key _key; - size_t _next_pos; - size_t _last_pos; - ss::abort_source& _as; - cloud_io::group_id _gid; - size_t _chunk_size; - ss::chunked_fifo> _current; -}; + auto result = result_fut.get(); + if (result != cloud_io::download_result::success) { + throw std::runtime_error( + fmt::format("L1 streaming download failed: {}", result)); + } + co_return make_iobuf_input_stream(std::move(buf)); +} } // namespace @@ -424,19 +364,18 @@ file_io::read_object( } if (skip_cache) { - // Not in the cache: stream directly from object storage in bounded - // chunks without populating the cache. - co_return ss::input_stream(ss::data_source( - std::make_unique( - _remote, - _bucket, - object_path_factory::level_one_path(extent.id), - cloud_storage_clients::http_byte_range{ - extent.position, extent.position + extent.size - 1}, - *as, - gid, - config::shard_local_cfg() - .cloud_topics_l1_streaming_read_chunk_size()))); + // Not in the cache: serve the requested range directly from object + // storage as a single ranged GET, without populating the cache. Any + // chunking that bounds peak memory is applied a layer up in + // open_object, which invokes read_object once per chunk. + co_return co_await download_range_bypassing_cache( + _remote, + _bucket, + object_path_factory::level_one_path(extent.id), + gid, + extent.position, + extent.size, + as); } // single_flight dedups concurrent downloads for this extent. @@ -463,6 +402,17 @@ file_io::read_object( } } +ss::future> file_io::fetch_native_footer( + object_extent extent, + ss::abort_source* as, + cloud_io::group_id gid, + bool skip_cache) { + if (_probe != nullptr) { + _probe->register_footer_read(extent.size); + } + return io::fetch_native_footer(extent, as, gid, skip_cache); +} + ss::future> file_io::delete_objects(chunked_vector ids, ss::abort_source* as) { static constexpr auto timeout = 10s; diff --git a/src/v/cloud_topics/level_one/common/file_io.h b/src/v/cloud_topics/level_one/common/file_io.h index 66c7f418584c3..99ef9e4d94ca5 100644 --- a/src/v/cloud_topics/level_one/common/file_io.h +++ b/src/v/cloud_topics/level_one/common/file_io.h @@ -60,6 +60,12 @@ class file_io : public io { cloud_io::group_id g, bool skip_cache) override; + ss::future> fetch_native_footer( + object_extent, + ss::abort_source*, + cloud_io::group_id g, + bool skip_cache) override; + ss::future> delete_objects(chunked_vector, ss::abort_source*) override; diff --git a/src/v/cloud_topics/level_one/common/file_io_probe.cc b/src/v/cloud_topics/level_one/common/file_io_probe.cc index 1aa6e19f9f287..8945358273ddf 100644 --- a/src/v/cloud_topics/level_one/common/file_io_probe.cc +++ b/src/v/cloud_topics/level_one/common/file_io_probe.cc @@ -42,6 +42,10 @@ void file_io_probe::setup_metrics() { sm::description( "Cache misses that joined an in-flight download for the " "same extent.")), + sm::make_counter( + "footer_read_bytes", + [this] { return _footer_bytes_read; }, + sm::description("Native L1 object footer bytes read by L1 readers.")), }); } diff --git a/src/v/cloud_topics/level_one/common/file_io_probe.h b/src/v/cloud_topics/level_one/common/file_io_probe.h index d952561343c7c..df9600e9b206e 100644 --- a/src/v/cloud_topics/level_one/common/file_io_probe.h +++ b/src/v/cloud_topics/level_one/common/file_io_probe.h @@ -11,6 +11,7 @@ #include "metrics/metrics.h" +#include #include namespace cloud_topics::l1 { @@ -24,6 +25,8 @@ class file_io_probe { void register_cache_miss() { ++_cache_misses; } void register_concurrent_read_merge() { ++_concurrent_read_merges; } + void register_footer_read(size_t bytes) { _footer_bytes_read += bytes; } + private: void setup_metrics(); @@ -31,6 +34,8 @@ class file_io_probe { uint64_t _cache_misses{0}; uint64_t _concurrent_read_merges{0}; + uint64_t _footer_bytes_read{0}; + metrics::internal_metric_groups _metrics; }; diff --git a/src/v/cloud_topics/level_one/common/object_handle.cc b/src/v/cloud_topics/level_one/common/object_handle.cc new file mode 100644 index 0000000000000..d802a1b9ac49b --- /dev/null +++ b/src/v/cloud_topics/level_one/common/object_handle.cc @@ -0,0 +1,53 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ +#include "cloud_topics/level_one/common/object_handle.h" + +#include "cloud_topics/level_one/common/abstract_io.h" +#include "cloud_topics/level_one/common/object.h" +#include "cloud_topics/level_one/common/object_id.h" + +namespace cloud_topics::l1 { + +l1_footer_index::l1_footer_index(footer f) + : _footer(std::move(f)) {} + +std::optional l1_footer_index::seek_to_offset( + model::topic_id_partition tidp, kafka::offset offset) const { + auto r = _footer.file_position_before_kafka_offset(tidp, offset); + if (r == footer::npos) { + return std::nullopt; + } + return seek_result{.file_position = r.file_position, .length = r.length}; +} + +std::optional l1_footer_index::seek_to_timestamp( + model::topic_id_partition tidp, model::timestamp ts) const { + auto r = _footer.file_position_before_max_timestamp(tidp, ts); + if (r == footer::npos) { + return std::nullopt; + } + return seek_result{.file_position = r.file_position, .length = r.length}; +} + +l1_native_object_handle::l1_native_object_handle(footer f, fetch_range_fn fetch) + : _index(std::move(f)) + , _fetch(std::move(fetch)) {} + +ss::future, io::errc>> +l1_native_object_handle::open_reader( + const seek_result& seek, ss::abort_source* as) { + auto stream_result = co_await _fetch(seek.file_position, seek.length, as); + if (!stream_result.has_value()) { + co_return std::unexpected(stream_result.error()); + } + co_return object_reader::create(std::move(stream_result).value()); +} + +} // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/common/object_handle.h b/src/v/cloud_topics/level_one/common/object_handle.h new file mode 100644 index 0000000000000..42ce81a5c41c5 --- /dev/null +++ b/src/v/cloud_topics/level_one/common/object_handle.h @@ -0,0 +1,121 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ + +#pragma once + +#include "cloud_topics/level_one/common/abstract_io.h" +#include "cloud_topics/level_one/common/object.h" +#include "cloud_topics/level_one/common/object_id.h" +#include "model/fundamental.h" + +#include + +#include +#include +#include +#include + +namespace cloud_topics::l1 { + +/// Result of an index seek. `file_position` is always a byte offset within +/// the object. `delta`, when set, is the log-to-Kafka offset-translation delta +/// at `file_position`, letting the reader translate log offsets to Kafka +/// offsets directly rather than inferring the delta from the first batch; it is +/// `nullopt` when the index already reports positions in Kafka-offset space. +struct seek_result { + size_t file_position{0}; + /// Bytes from file_position to the end of the readable range. + /// Always non-zero for a valid seek. + size_t length{0}; + std::optional delta; +}; + +/// An index over a single L1 object. +/// +/// Abstracts seeking to a byte position by Kafka offset or timestamp, +/// independent of the object's on-disk index format. +class object_index { +public: + virtual ~object_index() = default; + + /// Return the seek position to start reading at or before the given + /// Kafka offset. Returns nullopt when the object has no data at or after + /// the offset. + virtual std::optional + seek_to_offset(model::topic_id_partition, kafka::offset) const = 0; + + /// Return the seek position for data with max_timestamp >= ts. + /// Returns nullopt when no matching data exists. + virtual std::optional + seek_to_timestamp(model::topic_id_partition, model::timestamp) const = 0; +}; + +/// An open reference to a single L1 object. +/// +/// Obtained from l1::open_object. Holds the object's index and can open readers +/// positioned at any seek result. +class object_handle { +public: + virtual ~object_handle() = default; + + /// Return a reference to the index for this object. + virtual const object_index& index() const = 0; + + /// Open a reader starting at the seek result returned by index(). + /// `seek.file_position` is always a byte offset; `seek.delta`, when set, + /// carries the offset delta at the seek point. + virtual ss::future, io::errc>> + open_reader(const seek_result& seek, ss::abort_source*) = 0; +}; + +/// Fetches a stream over an object's raw bytes [file_position, +/// file_position+length). +/// +/// Used by object_handle implementations to read from the underlying extent. +/// The handle just streams whatever this returns; any chunking (bounding peak +/// memory / cache-chunk granularity) is baked into the fetch by open_object +/// when it builds the handle, not applied by the handle itself. Copyable: the +/// reader (and its data source's copy of this fetch) outlives the handle, so it +/// must not capture the handle. +using fetch_range_fn + = std::function, io::errc>>( + size_t file_position, size_t length, ss::abort_source*)>; + +/// object_index over a native L1 object's footer. +class l1_footer_index final : public object_index { +public: + explicit l1_footer_index(footer f); + + std::optional + seek_to_offset(model::topic_id_partition, kafka::offset) const override; + std::optional seek_to_timestamp( + model::topic_id_partition, model::timestamp) const override; + +private: + footer _footer; +}; + +/// object_handle for a native L1 object. Byte ranges are read through the +/// `fetch` callback supplied by open_object. +class l1_native_object_handle final : public object_handle { +public: + l1_native_object_handle(footer f, fetch_range_fn fetch); + + const object_index& index() const override { return _index; } + + ss::future, io::errc>> + open_reader(const seek_result& seek, ss::abort_source* as) override; + +private: + l1_footer_index _index; + fetch_range_fn _fetch; +}; + +} // namespace cloud_topics::l1 diff --git a/src/v/cloud_topics/level_one/common/open_object.cc b/src/v/cloud_topics/level_one/common/open_object.cc new file mode 100644 index 0000000000000..5332acb729f0f --- /dev/null +++ b/src/v/cloud_topics/level_one/common/open_object.cc @@ -0,0 +1,96 @@ +/* + * Copyright 2026 Redpanda Data, Inc. + * + * Licensed as a Redpanda Enterprise file under the Redpanda Community + * License (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://github.com/redpanda-data/redpanda/blob/master/licenses/rcl.md + */ +#include "cloud_topics/level_one/common/open_object.h" + +#include "cloud_topics/level_one/common/chunk_data_source.h" +#include "cloud_topics/level_one/common/object.h" +#include "cloud_topics/logger.h" +#include "config/configuration.h" + +namespace cloud_topics::l1 { + +namespace { + +// Chunking for a read, given whether the cache is bypassed. Applied by wrapping +// the per-range read_object fetch in a chunk_data_source; chunk_size == 0 means +// "don't chunk" -- pass the whole requested range straight through. +struct read_chunking { + size_t chunk_size; + use_chunk_aligned_reads aligned; +}; + +read_chunking chunking_for(bool skip_cache) { + const auto& cfg = config::shard_local_cfg(); + if (skip_cache) { + // Cache-bypassing bulk/one-shot read (leveling, compaction): bound peak + // memory with the streaming chunk size. Nothing is cached, so there is + // no (pos,size) key to align to -- chunks start at the read's position. + return { + cfg.cloud_topics_l1_streaming_read_chunk_size(), + use_chunk_aligned_reads::no}; + } + // Cached native read: read_object stores the whole requested extent in the + // cache and serves it from a file, so the cache file bounds memory and no + // stream-level chunking is needed. + return {0, use_chunk_aligned_reads::yes}; +} + +// Wrap a per-range fetch so a request for [pos, pos+len) is served as a +// chunk_data_source that lazily pulls fixed-size chunks (only the chunks a read +// touches are fetched). chunk_size == 0 passes the whole range straight through +// (the cached-native case). +fetch_range_fn chunked_fetch(fetch_range_fn inner, read_chunking c) { + if (c.chunk_size == 0) { + return inner; + } + return [inner = std::move(inner), + c](size_t pos, size_t len, ss::abort_source* as) + -> ss::future, io::errc>> { + co_return ss::input_stream{ + make_chunk_data_source(inner, pos, len, c.chunk_size, as, c.aligned)}; + }; +} + +} // namespace + +ss::future, io::errc>> open_object( + io& io_impl, + object_extent extent, + ss::abort_source* as, + cloud_io::group_id g, + bool skip_cache) { + // Native L1 object: read and parse the footer, then hand the handle a fetch + // that reads data ranges back through io::read_object, with the read's + // chunking layered on by chunked_fetch. + auto footer_buf = co_await io_impl.fetch_native_footer( + extent, as, g, skip_cache); + if (!footer_buf.has_value()) { + co_return std::unexpected(footer_buf.error()); + } + auto footer_result = co_await footer::read(std::move(footer_buf).value()); + if (!std::holds_alternative