RFC ts-ct-migration: metastore and L1 reader [PR 1]

proto/redpanda/core/admin/internal/cloud_topics/v1/metastore.proto

@@ -237,6 +237,34 @@ message MetadataValue {
     int64 compaction_epoch = 3;
     uint64 size = 4;
     uint64 num_extents = 5;
+    // True while the partition is mid tiered->cloud migration.
+    bool migrating = 6;
+}
+
+// Where the backing tiered-storage segment of an imported object lives. An
+// object property: the imported analog of a native object's id->bucket path.
+message ImportedTsObjectLocation {
+    string ts_path = 1;
+}
+
+// Whether an imported tiered-storage segment has an aborted-transaction (.tx)
+// manifest, resolved at import time so the read path need not probe object
+// storage. UNKNOWN (v1/v2 non-compacted) => probe and tolerate a missing .tx.
+enum ImportedTsTxManifestState {
+    IMPORTED_TS_TX_MANIFEST_STATE_UNKNOWN = 0;
+    IMPORTED_TS_TX_MANIFEST_STATE_ABSENT = 1;
+    IMPORTED_TS_TX_MANIFEST_STATE_PRESENT = 2;
+}
+
+// Properties of the data in an imported tiered-storage extent. An extent
+// property, alongside the extent's Kafka offset bounds (base/last_offset).
+message ImportedTsSegmentInfo {
+    int64 segment_term = 1;
+    // Offset-translation delta at the segment's base (source segment_meta's
+    // delta_offset).
+    int64 delta_base = 2;
+    // Whether the segment's .tx manifest exists (resolved at import time).
+    ImportedTsTxManifestState tx_state = 3;
 }
 
 // Value for an extent row.
@@ -246,6 +274,7 @@ message ExtentValue {
     uint64 filepos = 3;
     uint64 len = 4;
     string object_id = 5;
+    optional ImportedTsSegmentInfo imported_ts_info = 6;
 }
 
 // Value for a term row.
@@ -280,6 +309,7 @@ message ObjectValue {
     uint64 object_size = 4;
     int64 last_updated = 5;
     bool is_preregistration = 6;
+    optional ImportedTsObjectLocation imported_ts_location = 7;
 }
 
 // A typed value for a metastore row.

src/v/cloud_storage/BUILD

@@ -86,6 +86,32 @@ redpanda_cc_library(
     ],
 )
 
+redpanda_cc_library(
+    name = "offset_index",
+    srcs = [
+        "offset_index.cc",
+    ],
+    hdrs = [
+        "offset_index.h",
+    ],
+    implementation_deps = [
+        ":logger",
+        "//src/v/base",
+        "//src/v/bytes:iobuf_parser",
+        "//src/v/serde",
+        "//src/v/serde:iobuf",
+        "//src/v/serde:vector",
+        "@seastar",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//src/v/bytes:iobuf",
+        "//src/v/model",
+        "//src/v/utils:delta_for",
+        "@abseil-cpp//absl/container:btree",
+    ],
+)
+
 redpanda_cc_library(
     name = "cloud_storage",
     srcs = [
@@ -183,6 +209,7 @@ redpanda_cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":logger",
+        ":offset_index",
         ":remote_label",
         ":segment_meta_cstore",
         ":topic_mount_manifest_path",