diff --git a/.github/workflows/build-cnpg-timescaledb-dev.yml b/.github/workflows/build-cnpg-timescaledb-dev.yml index d20db6e21..d795c995f 100644 --- a/.github/workflows/build-cnpg-timescaledb-dev.yml +++ b/.github/workflows/build-cnpg-timescaledb-dev.yml @@ -8,8 +8,8 @@ on: default: '17' required: true tsdb_version: - description: TimescaleDB version (semver, e.g. 2.25.0) - default: '2.25.0' + description: TimescaleDB version (semver, e.g. 2.28.2) + default: '2.28.2' required: true permissions: diff --git a/.github/workflows/build-cnpg-timescaledb-stg.yml b/.github/workflows/build-cnpg-timescaledb-stg.yml index cef0d001e..6450c3a6d 100644 --- a/.github/workflows/build-cnpg-timescaledb-stg.yml +++ b/.github/workflows/build-cnpg-timescaledb-stg.yml @@ -8,8 +8,8 @@ on: default: '17' required: true tsdb_version: - description: TimescaleDB version (semver, e.g. 2.25.0) - default: '2.25.0' + description: TimescaleDB version (semver, e.g. 2.28.2) + default: '2.28.2' required: true permissions: diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index e659730e3..d762a6569 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: timescale/timescaledb:2.25.0-pg17 + image: timescale/timescaledb:2.28.2-pg17 env: POSTGRES_USER: postgres POSTGRES_DB: postgres diff --git a/Dockerfile-timescale-cnpg b/Dockerfile-timescale-cnpg index 5ca5b2d87..51cbf80e4 100644 --- a/Dockerfile-timescale-cnpg +++ b/Dockerfile-timescale-cnpg @@ -5,7 +5,7 @@ # TimescaleDB from the apt repo and layers it onto the CNPG base image. # # Usage: -# docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.25.0 -t cnpg-timescaledb . +# docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.28.2 -t cnpg-timescaledb . ARG PG_MAJOR=17 @@ -15,7 +15,7 @@ ARG PG_MAJOR=17 FROM ghcr.io/cloudnative-pg/postgresql:${PG_MAJOR}-bookworm AS timescaledb-builder ARG PG_MAJOR -ARG TSDB_VERSION=2.25.0 +ARG TSDB_VERSION=2.28.2 USER root diff --git a/docker-compose.yaml b/docker-compose.yaml index bf402fe09..23ce273c8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: db: container_name: db - image: timescale/timescaledb:2.25.0-pg17 + image: timescale/timescaledb:2.28.2-pg17 command: ["postgres", "-c", "timescaledb.enable_chunk_skipping=on", "-c", "timescaledb.enable_sparse_index_bloom=on"] healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] diff --git a/docs/data-migrations/adding-a-protocol.md b/docs/data-migrations/adding-a-protocol.md index db9a1023a..5b5ae8716 100644 --- a/docs/data-migrations/adding-a-protocol.md +++ b/docs/data-migrations/adding-a-protocol.md @@ -166,13 +166,12 @@ CREATE TABLE _ ( last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (...) -- Choose a primary key that fits your state's natural identity ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -- Add indexes based on your query patterns @@ -182,7 +181,19 @@ CREATE TABLE _ ( DROP TABLE IF EXISTS _; ``` -The storage parameters (`fillfactor = 80`, aggressive autovacuum) are tuned for heavy UPSERT workloads during ingestion. Adjust based on your protocol's write pattern. +`fillfactor = 90` is the default: it reserves 10% free space per page for HOT (Heap-Only +Tuple) updates without over-reserving for tables whose UPSERTs only touch non-indexed +columns. Use `fillfactor = 80` instead only for tables whose rows update on essentially +every ledger (e.g. AMM-style reserves that shift on every trade). `autovacuum_vacuum_cost_delay += 0` disables cost-based vacuum throttling for this table's worker; there is no +`autovacuum_vacuum_cost_limit` to set alongside it since a delay of 0 makes any cost +limit a no-op. + +The aggressive autovacuum block above is for tables whose row count grows with users or +usage (positions, balances, allowances). Skip it entirely — keep only `fillfactor = 90` +— for small, protocol-bounded current-state tables (pool configs, reserve configs) whose +row count is capped by the number of pools/reserves your protocol will ever have +deployed; default autovacuum behavior is fine for those. ### Data Model diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 000000000..bd65403cb --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,14 @@ +# Operations + +**Hypertable index usage:** `pg_stat_user_indexes` on the `public` schema always reports `idx_scan = 0` for hypertable indexes — scans accrue on the per-chunk indexes in `_timescaledb_internal`, and the parent's index entry is only a template. To measure real usage (e.g. for an unused-index audit), aggregate chunk-index stats: + +```sql +SELECT ht.table_name AS hypertable, regexp_replace(sui.indexrelname, '^_hyper_\d+_\d+_chunk_', '') AS index_name, + sum(sui.idx_scan) AS scans +FROM pg_stat_user_indexes sui +JOIN _timescaledb_catalog.chunk c ON c.schema_name = sui.schemaname AND c.table_name = sui.relname +JOIN _timescaledb_catalog.hypertable ht ON ht.id = c.hypertable_id +GROUP BY 1, 2 ORDER BY 1, 3 DESC; +``` + +Every index in the schema must have a nameable consumer query; secondary indexes duplicating a primary key's column set are not kept (the PK's column order is chosen to serve its consumers directly — B-tree scans run forward or backward, so ASC/DESC variants of the same key are one index, not two). diff --git a/internal/data/ingest_store.go b/internal/data/ingest_store.go index 7c022cc58..642b924c4 100644 --- a/internal/data/ingest_store.go +++ b/internal/data/ingest_store.go @@ -169,19 +169,35 @@ func (m *IngestStoreModel) UpdateMin(ctx context.Context, dbTx pgx.Tx, cursorNam return nil } -func (m *IngestStoreModel) GetLedgerGaps(ctx context.Context) ([]LedgerRange, error) { +// GetLedgerGaps returns gaps between consecutive present ledger_number values within +// [startLedger, endLedger]. The window bounds the cost two ways: chunks whose recorded +// ledger_number range (enable_chunk_skipping on transactions.ledger_number; ranges are +// captured when a chunk is compressed) doesn't overlap the window are excluded at plan +// time, and any remaining out-of-window batches are dropped by the columnstore's batch +// min/max metadata without decompression. +// +// Edge semantics: callers must pass a startLedger that is itself a present ledger_number (e.g. +// the oldest ingested ledger) — the left edge is NOT synthesized. If startLedger fell strictly +// inside an already-open gap, this function would not report the portion before the first +// present row in-window; this mirrors the original unwindowed behavior, which never reported a +// gap before the very first row in the whole table. The right edge IS handled: COALESCE falls +// back to endLedger+1 when LEAD finds no next row within the window (the next present ledger +// lies beyond endLedger, or doesn't exist yet), so a gap still open at the window's boundary is +// reported clipped to endLedger instead of silently dropped (plain LEAD would return NULL there, +// failing the gap_start <= gap_end filter and losing the trailing partial gap). +func (m *IngestStoreModel) GetLedgerGaps(ctx context.Context, startLedger, endLedger uint32) ([]LedgerRange, error) { const query = ` SELECT gap_start, gap_end FROM ( SELECT ledger_number + 1 AS gap_start, - LEAD(ledger_number) OVER (ORDER BY ledger_number) - 1 AS gap_end - FROM (SELECT DISTINCT ledger_number FROM transactions) t + COALESCE(LEAD(ledger_number) OVER (ORDER BY ledger_number), $2 + 1) - 1 AS gap_end + FROM (SELECT DISTINCT ledger_number FROM transactions WHERE ledger_number BETWEEN $1 AND $2) t ) gaps WHERE gap_start <= gap_end ORDER BY gap_start ` start := time.Now() - ledgerGaps, err := db.QueryMany[LedgerRange](ctx, m.DB, query) + ledgerGaps, err := db.QueryMany[LedgerRange](ctx, m.DB, query, int32(startLedger), int32(endLedger)) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("GetLedgerGaps", "transactions").Observe(duration) m.Metrics.QueriesTotal.WithLabelValues("GetLedgerGaps", "transactions").Inc() diff --git a/internal/data/ingest_store_test.go b/internal/data/ingest_store_test.go index d2f4485e4..5fb8ed802 100644 --- a/internal/data/ingest_store_test.go +++ b/internal/data/ingest_store_test.go @@ -401,15 +401,21 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { testCases := []struct { name string + startLedger uint32 + endLedger uint32 setupDB func(t *testing.T) expectedGaps []LedgerRange }{ { name: "returns_empty_when_no_transactions", + startLedger: 1, + endLedger: 100, expectedGaps: []LedgerRange{}, }, { - name: "returns_empty_when_no_gaps", + name: "returns_empty_when_no_gaps", + startLedger: 100, + endLedger: 102, setupDB: func(t *testing.T) { // Insert consecutive ledgers: 100, 101, 102 for i, ledger := range []uint32{100, 101, 102} { @@ -423,7 +429,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { expectedGaps: []LedgerRange{}, }, { - name: "returns_single_gap", + name: "returns_single_gap", + startLedger: 100, + endLedger: 105, setupDB: func(t *testing.T) { // Insert ledgers 100 and 105, creating gap 101-104 for i, ledger := range []uint32{100, 105} { @@ -439,7 +447,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { }, }, { - name: "returns_multiple_gaps", + name: "returns_multiple_gaps", + startLedger: 100, + endLedger: 110, setupDB: func(t *testing.T) { // Insert ledgers 100, 105, 110, creating gaps 101-104 and 106-109 for i, ledger := range []uint32{100, 105, 110} { @@ -456,7 +466,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { }, }, { - name: "handles_single_ledger_gap", + name: "handles_single_ledger_gap", + startLedger: 100, + endLedger: 102, setupDB: func(t *testing.T) { // Insert ledgers 100 and 102, creating gap of just 101 for i, ledger := range []uint32{100, 102} { @@ -471,6 +483,46 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { {GapStart: 101, GapEnd: 101}, }, }, + { + // SQL-05: the real next ledger (200) lies beyond the requested window (150). The + // trailing gap must be reported clipped to endLedger, not dropped just because LEAD + // finds no next row inside the window. + name: "clips_trailing_gap_still_open_at_window_boundary", + startLedger: 100, + endLedger: 150, + setupDB: func(t *testing.T) { + for i, ledger := range []uint32{100, 200} { + _, err := dbConnectionPool.Exec(ctx, + `INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 150}, + }, + }, + { + // Ledgers outside [startLedger, endLedger] must not affect the result: a gap beyond + // endLedger (106-109) is excluded entirely, proving the query is bounded rather than + // scanning the whole table. + name: "ignores_gaps_entirely_outside_the_window", + startLedger: 100, + endLedger: 104, + setupDB: func(t *testing.T) { + for i, ledger := range []uint32{100, 105, 110} { + _, err := dbConnectionPool.Exec(ctx, + `INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 104}, + }, + }, } for _, tc := range testCases { @@ -490,7 +542,7 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { tc.setupDB(t) } - gaps, err := m.GetLedgerGaps(ctx) + gaps, err := m.GetLedgerGaps(ctx, tc.startLedger, tc.endLedger) require.NoError(t, err) assert.Equal(t, tc.expectedGaps, gaps) }) diff --git a/internal/data/mocks.go b/internal/data/mocks.go index 05e0ca567..83a3aa96f 100644 --- a/internal/data/mocks.go +++ b/internal/data/mocks.go @@ -309,14 +309,6 @@ func (m *ProtocolsModelMock) GetByIDs(ctx context.Context, protocolIDs []string) return args.Get(0).([]Protocols), args.Error(1) } -func (m *ProtocolsModelMock) GetClassified(ctx context.Context) ([]Protocols, error) { - args := m.Called(ctx) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).([]Protocols), args.Error(1) -} - func (m *ProtocolsModelMock) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error { args := m.Called(ctx, dbTx, protocolID) return args.Error(0) diff --git a/internal/data/operations.go b/internal/data/operations.go index 7409d10c8..140d17d5a 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -23,7 +23,7 @@ type OperationModel struct { } func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) (*types.Operation, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") query := fmt.Sprintf(`SELECT %s FROM operations WHERE id = $1`, columns) start := time.Now() operation, err := db.QueryOne[types.Operation](ctx, m.DB, query, id) @@ -38,7 +38,10 @@ func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) } func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") queryBuilder := strings.Builder{} var args []interface{} argIndex := 1 @@ -112,44 +115,59 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 // and the next transaction (index 4096 = 0x1000). // // See SEP-35: https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0035.md -func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") - queryBuilder := strings.Builder{} - // This CTE query implements per-transaction pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all operations from just a few - // transactions, we use ROW_NUMBER() with PARTITION BY to limit results per transaction. - // This guarantees that each transaction gets at most 'limit' operations, providing - // more balanced and predictable pagination across multiple transactions. - // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. - query := ` - WITH - inputs (to_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_operations_per_to_id AS ( - SELECT - o.*, - i.to_id as tx_to_id, - ROW_NUMBER() OVER (PARTITION BY i.to_id ORDER BY o.id %s) AS rn - FROM - operations o - JOIN - inputs i ON o.id > i.to_id AND o.id < i.to_id + 4096 - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM ranked_operations_per_to_id - ` - fmt.Fprintf(&queryBuilder, query, sortOrder, columns) +func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { + if len(toIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") + + // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding + // (see the package doc above). Each key carries its parent transaction's ledger_created_at, + // so the LATERAL probe pins both the TOID band and the partition column, giving per-key + // runtime chunk exclusion instead of a Parallel Append across the whole hypertable. The + // ORDER BY + LIMIT live inside the LATERAL (a per-key top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-transaction cap; a subquery + // containing LIMIT cannot be flattened into a join, so the planner must run this as one + // per-key probe (see BatchGetByAccountAddress). Only the requested columns are selected + // inside the LATERAL, avoiding decompressing full rows before projection — so + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST) rather than + // o.ledger_created_at, which the LATERAL may not have projected; the two are identical for + // any row that survives the WHERE match. o.id is safe to reference directly since + // prepareColumnsWithID always forces "id" into the projection. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column name ("id"), not "o.id" — there is no "o" in scope at that + // level, only the fenced derived table. + args := []interface{}{toIDs, ledgerCreatedAts} + var queryBuilder strings.Builder + fmt.Fprintf(&queryBuilder, ` + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, o.id as cursor_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(to_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM operations o + WHERE o.id > k.to_id AND o.id < k.to_id + 4096 AND o.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY id %s`, columns, columns, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } - query = queryBuilder.String() + queryBuilder.WriteString(` + ) o + `) + + query := queryBuilder.String() if sortOrder == DESC { query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations.cursor_ledger_created_at ASC, operations.cursor_id ASC`, query) } start := time.Now() - operations, err := db.QueryManyPtrs[types.OperationWithCursor](ctx, m.DB, query, toIDs) + operations, err := db.QueryManyPtrs[types.OperationWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByToIDs", "operations").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByToIDs", "operations").Observe(float64(len(toIDs))) @@ -162,15 +180,16 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, col } // BatchGetByToID gets operations for a single transaction ToID with pagination support. -// Operations for a transaction are found using TOID range: (tx_to_id, tx_to_id + 4096). -func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") +// Operations for a transaction are found using TOID range: (tx_to_id, tx_to_id + 4096), pinned +// to the parent transaction's ledger_created_at for partition-column chunk exclusion. +func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") queryBuilder := strings.Builder{} // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. - fmt.Fprintf(&queryBuilder, `SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM operations WHERE id > $1 AND id < $1 + 4096`, columns) + fmt.Fprintf(&queryBuilder, `SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM operations WHERE id > $1 AND id < $1 + 4096 AND ledger_created_at = $2`, columns) - args := []interface{}{toID} - argIndex := 2 + args := []interface{}{toID, ledgerCreatedAt} + argIndex := 3 if cursor != nil { if sortOrder == DESC { @@ -228,7 +247,7 @@ func (m *OperationModel) BatchGetAccountOperationsByToIDs(ctx context.Context, a if len(toIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s FROM UNNEST($2::bigint[], $3::timestamptz[]) AS i(to_id, ledger_created_at) @@ -265,7 +284,7 @@ func (m *OperationModel) BatchGetAccountOperationsByToIDs(ctx context.Context, a // Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization // on the operations_accounts hypertable by ordering on ledger_created_at first. func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder, timeRange *TimeRange) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} @@ -348,26 +367,32 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd } // BatchGetByStateChangeIDs gets the operations that are associated with the given state change IDs. -func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, stateChangeIDs []int64, columns string) ([]*types.OperationWithStateChangeID, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "operations", "id") - - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(stateChangeIDs)) - for i := range stateChangeIDs { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], stateChangeIDs[i]) +// Callers supply each key's parent ledger_created_at (the state change's ledger time, same ledger +// as its operation) so the LATERAL can pin the complete operations primary key (id, +// ledger_created_at) instead of joining through state_changes and hash-joining a full IN-list. +func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, stateChangeIDs []int64, ledgerCreatedAts []time.Time, columns string) ([]*types.OperationWithStateChangeID, error) { + if len(scOpIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("scOpIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scOpIDs), len(ledgerCreatedAts)) } + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") + // ORDER BY reads k.ledger_created_at (from UNNEST), not o.ledger_created_at: the LATERAL + // only projects the caller-requested columns, which may not include ledger_created_at, but + // k.ledger_created_at is always in scope and identical to o.ledger_created_at for any row + // that survives the WHERE match. query := fmt.Sprintf(` - SELECT %s, CONCAT(state_changes.to_id, '-', state_changes.operation_id, '-', state_changes.state_change_id) AS state_change_id - FROM operations - INNER JOIN state_changes ON operations.id = state_changes.operation_id - WHERE (state_changes.to_id, state_changes.operation_id, state_changes.state_change_id) IN (%s) - ORDER BY operations.ledger_created_at DESC - `, columns, strings.Join(tuples, ", ")) + SELECT %s, CONCAT(k.to_id, '-', k.operation_id, '-', k.state_change_id) AS state_change_id + FROM UNNEST($1::bigint[], $2::bigint[], $3::bigint[], $4::timestamptz[]) AS k(to_id, operation_id, state_change_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT %s FROM operations o + WHERE o.id = k.operation_id AND o.ledger_created_at = k.ledger_created_at + LIMIT 1 + ) o + ORDER BY k.ledger_created_at DESC + `, columns, columns) start := time.Now() - operationsWithStateChanges, err := db.QueryManyPtrs[types.OperationWithStateChangeID](ctx, m.DB, query) + operationsWithStateChanges, err := db.QueryManyPtrs[types.OperationWithStateChangeID](ctx, m.DB, query, scToIDs, scOpIDs, stateChangeIDs, ledgerCreatedAts) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByStateChangeIDs", "operations").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByStateChangeIDs", "operations").Observe(float64(len(stateChangeIDs))) diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index 5e65be650..91bd653a4 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -392,7 +392,11 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) { Metrics: dbMetrics, } - operations, err := m.BatchGetByToIDs(ctx, tc.toIDs, "", tc.limit, tc.sortOrder) + ledgerCreatedAts := make([]time.Time, len(tc.toIDs)) + for i := range ledgerCreatedAts { + ledgerCreatedAts[i] = now + } + operations, err := m.BatchGetByToIDs(ctx, tc.toIDs, ledgerCreatedAts, "", tc.limit, tc.sortOrder) require.NoError(t, err) assert.Len(t, operations, tc.expectedCount) @@ -435,6 +439,44 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) { } }) } + + t.Run("wrong ledger_created_at for a key returns no operations for that key (time pin enforced)", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByToIDs(ctx, []int64{4096}, []time.Time{wrongTime}, "", nil, ASC) + require.NoError(t, err) + assert.Empty(t, operations) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + _, err := m.BatchGetByToIDs(ctx, []int64{4096, 8192}, []time.Time{now}, "", nil, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns) — a narrow column set that + // excludes ledgerCreatedAt must not break the query with "column does not exist". + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + operations, err := m.BatchGetByToIDs(ctx, []int64{4096}, []time.Time{now}, "operation_type", nil, ASC) + require.NoError(t, err) + require.Len(t, operations, 3) + for _, op := range operations { + assert.NotZero(t, op.CompositeCursor.LedgerCreatedAt) + assert.NotZero(t, op.CompositeCursor.ID) + } + }) } func int32Ptr(v int32) *int32 { @@ -486,11 +528,18 @@ func TestOperationModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) // Test BatchGetByToID - operations, err := m.BatchGetByToID(ctx, 4096, "", nil, nil, ASC) + operations, err := m.BatchGetByToID(ctx, 4096, now, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 2) assert.Equal(t, xdr1.String(), operations[0].OperationXDR.String()) assert.Equal(t, xdr3.String(), operations[1].OperationXDR.String()) + + t.Run("wrong ledger_created_at returns no operations (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByToID(ctx, 4096, wrongTime, "", nil, nil, ASC) + require.NoError(t, err) + assert.Empty(t, operations) + }) } func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { @@ -663,7 +712,8 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { require.NoError(t, err) // Test BatchGetByStateChangeID - operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192, 12288}, []int64{4097, 8193, 4097}, []int64{1, 1, 1}, "") + ledgerCreatedAts := []time.Time{now, now, now} + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192, 12288}, []int64{4097, 8193, 4097}, []int64{1, 1, 1}, ledgerCreatedAts, "") require.NoError(t, err) assert.Len(t, operations, 3) @@ -675,6 +725,26 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, int64(4097), stateChangeIDsFound["4096-4097-1"]) assert.Equal(t, int64(8193), stateChangeIDsFound["8192-8193-1"]) assert.Equal(t, int64(4097), stateChangeIDsFound["12288-4097-1"]) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096}, []int64{4097}, []int64{1}, []time.Time{wrongTime}, "") + require.NoError(t, err) + assert.Empty(t, operations) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192}, []int64{4097, 8193}, []int64{1, 1}, []time.Time{now}, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: ORDER BY must not read a column absent from the LATERAL's narrowed projection. + t.Run("narrow column selection excluding ledger_created_at does not break ORDER BY", func(t *testing.T) { + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096}, []int64{4097}, []int64{1}, []time.Time{now}, "operation_type") + require.NoError(t, err) + assert.Len(t, operations, 1) + }) } // BenchmarkOperationModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. @@ -833,3 +903,40 @@ func TestOperationModel_BatchGetAccountOperationsByToIDs(t *testing.T) { assert.Equal(t, int64(4097), ops[3].ID) }) } + +// A client selecting no time fields must still get operations whose LedgerCreatedAt is +// hydrated: the state-change and transaction relationship resolvers pin their lookups on the +// parent operation's partition timestamp, so every projection forces ledger_created_at. +func TestOperationModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + m := &OperationModel{DB: dbConnectionPool, Metrics: metrics.NewMetrics(reg).DB} + + now := time.Now().UTC().Truncate(time.Microsecond) + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) + VALUES (4098, 'PAYMENT', 'xdr', 'op_success', true, 1, $1) + `, now) + require.NoError(t, err) + + op, err := m.GetByID(ctx, 4098, "id") + require.NoError(t, err) + assert.True(t, now.Equal(op.LedgerCreatedAt), "GetByID with minimal projection must hydrate ledger_created_at") + + limit := int32(10) + ops, err := m.GetAll(ctx, "id", &limit, nil, DESC) + require.NoError(t, err) + require.Len(t, ops, 1) + assert.True(t, now.Equal(ops[0].Operation.LedgerCreatedAt), "GetAll with minimal projection must hydrate ledger_created_at") + + batched, err := m.BatchGetByToID(ctx, 4096, now, "id", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, batched, 1) + assert.True(t, now.Equal(batched[0].Operation.LedgerCreatedAt), "BatchGetByToID with minimal projection must hydrate ledger_created_at") +} diff --git a/internal/data/protocols.go b/internal/data/protocols.go index 2a22ba4ba..13410ee91 100644 --- a/internal/data/protocols.go +++ b/internal/data/protocols.go @@ -37,7 +37,6 @@ type ProtocolsModelInterface interface { UpdateHistoryMigrationStatus(ctx context.Context, dbTx pgx.Tx, protocolIDs []string, status string) error UpdateCurrentStateMigrationStatus(ctx context.Context, dbTx pgx.Tx, protocolIDs []string, status string) error GetByIDs(ctx context.Context, protocolIDs []string) ([]Protocols, error) - GetClassified(ctx context.Context) ([]Protocols, error) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error } @@ -145,26 +144,6 @@ func (m *ProtocolsModel) GetByIDs(ctx context.Context, protocolIDs []string) ([] return protocols, nil } -// GetClassified returns all protocols with classification_status = 'success'. -func (m *ProtocolsModel) GetClassified(ctx context.Context) ([]Protocols, error) { - const query = ` - SELECT id, classification_status, history_migration_status, current_state_migration_status, created_at, updated_at - FROM protocols - WHERE classification_status = 'success' - ` - - start := time.Now() - protocols, err := db.QueryMany[Protocols](ctx, m.DB, query) - duration := time.Since(start).Seconds() - m.Metrics.QueryDuration.WithLabelValues("GetClassified", "protocols").Observe(duration) - m.Metrics.QueriesTotal.WithLabelValues("GetClassified", "protocols").Inc() - if err != nil { - m.Metrics.QueryErrors.WithLabelValues("GetClassified", "protocols", utils.GetDBErrorType(err)).Inc() - return nil, fmt.Errorf("querying classified protocols: %w", err) - } - return protocols, nil -} - // InsertIfNotExists inserts a protocol if it doesn't already exist (idempotent). func (m *ProtocolsModel) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error { const query = `INSERT INTO protocols (id) VALUES ($1) ON CONFLICT (id) DO NOTHING` diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 78fedbe5a..7d5ba698c 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -49,6 +49,17 @@ const ( DESC SortOrder = "DESC" ) +// validatePositiveLimit rejects a non-nil limit that is zero or negative. A caller-supplied +// page size can go negative (e.g. an int32-overflowed `first` upstream), and silently dropping +// the LIMIT clause in that case turns a bounded page read into an unbounded full-table read. +// limit == nil still means "no limit" (used internally) and is left untouched. +func validatePositiveLimit(limit *int32) error { + if limit != nil && *limit <= 0 { + return fmt.Errorf("limit must be positive, got %d", *limit) + } + return nil +} + // pgtypeTextFromNullString converts sql.NullString to pgtype.Text for efficient binary COPY. func pgtypeTextFromNullString(ns sql.NullString) pgtype.Text { return pgtype.Text{String: ns.String, Valid: ns.Valid} @@ -160,6 +171,12 @@ func getDBColumns(model any) set.Set[string] { // prepareColumnsWithID ensures that all specified identifier columns are always included in the column list. // It appends whatever idColumns are passed in, which is useful for tables (such as state_changes) that use composite keys. +// +// Every transaction/operation/state-change projection also forces ledger_created_at — the +// hypertable partition column — even when the client didn't request that field: relationship +// resolvers pin their child lookups on the parent row's ledger_created_at (see the dataloader +// keys), so a row fetched without it would silently time-pin those lookups to the zero value +// and return empty relationships. func prepareColumnsWithID(columns string, model any, prefix string, idColumns ...string) string { var dbColumns set.Set[string] if columns == "" { diff --git a/internal/data/sac_balances.go b/internal/data/sac_balances.go index c9f936a2a..3ee2df725 100644 --- a/internal/data/sac_balances.go +++ b/internal/data/sac_balances.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/wallet-backend/internal/db" @@ -66,7 +67,7 @@ func (m *SACBalanceModel) GetByAccount(ctx context.Context, accountAddress strin } query := ` - SELECT asb.account_id, asb.contract_id, asb.balance, asb.is_authorized, + SELECT asb.account_id, asb.contract_id, asb.balance::text, asb.is_authorized, asb.is_clawback_enabled, asb.last_modified_ledger, ct.contract_id AS token_id, ct.code, ct.issuer, ct.decimals FROM sac_balances asb @@ -148,7 +149,9 @@ func (m *SACBalanceModel) BatchUpsert(ctx context.Context, dbTx pgx.Tx, upserts INSERT INTO sac_balances ( account_id, contract_id, balance, is_authorized, is_clawback_enabled, last_modified_ledger ) - SELECT * FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::boolean[], $5::boolean[], $6::int[]) + SELECT u.account_id, u.contract_id, u.balance::numeric, u.is_authorized, u.is_clawback_enabled, u.last_modified_ledger + FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::boolean[], $5::boolean[], $6::int[]) + AS u(account_id, contract_id, balance, is_authorized, is_clawback_enabled, last_modified_ledger) ON CONFLICT (account_id, contract_id) DO UPDATE SET balance = EXCLUDED.balance, is_authorized = EXCLUDED.is_authorized, @@ -224,10 +227,18 @@ func (m *SACBalanceModel) BatchCopy(ctx context.Context, dbTx pgx.Tx, balances [ if err != nil { return nil, fmt.Errorf("converting account address to bytes: %w", err) } + // CopyFrom uses the binary protocol, which requires a pgtype value for + // NUMERIC rather than a bare string. Scan also validates the balance is + // well-formed decimal text, catching malformed decoder output at the + // write boundary instead of storing it silently. + var balanceNumeric pgtype.Numeric + if err := balanceNumeric.Scan(bal.Balance); err != nil { + return nil, fmt.Errorf("parsing SAC balance %q for account %s: %w", bal.Balance, bal.AccountID, err) + } return []any{ accountIDBytes, bal.ContractID, - bal.Balance, + balanceNumeric, bal.IsAuthorized, bal.IsClawbackEnabled, bal.LedgerNumber, diff --git a/internal/data/sep41/allowances.go b/internal/data/sep41/allowances.go index d2f9db795..d4c55ac98 100644 --- a/internal/data/sep41/allowances.go +++ b/internal/data/sep41/allowances.go @@ -82,7 +82,7 @@ func (m *AllowanceModel) GetByOwner(ctx context.Context, ownerAddress string, li query := ` SELECT - a.spender_id, a.contract_id, a.amount, a.expiration_ledger, a.last_modified_ledger, + a.spender_id, a.contract_id, a.amount::text, a.expiration_ledger, a.last_modified_ledger, ct.contract_id AS token_id FROM sep41_allowances a INNER JOIN contract_tokens ct ON ct.id = a.contract_id @@ -196,10 +196,12 @@ func (m *AllowanceModel) BatchUpsert(ctx context.Context, dbTx pgx.Tx, upserts [ owner_id, spender_id, contract_id, amount, expiration_ledger, last_modified_ledger ) - SELECT * FROM UNNEST( + SELECT u.owner_id, u.spender_id, u.contract_id, + u.amount::numeric, u.expiration_ledger, u.last_modified_ledger + FROM UNNEST( $1::bytea[], $2::bytea[], $3::uuid[], $4::text[], $5::integer[], $6::integer[] - ) + ) AS u(owner_id, spender_id, contract_id, amount, expiration_ledger, last_modified_ledger) ON CONFLICT (owner_id, spender_id, contract_id) DO UPDATE SET amount = EXCLUDED.amount, expiration_ledger = EXCLUDED.expiration_ledger, diff --git a/internal/data/sep41/balances.go b/internal/data/sep41/balances.go index de3fe67ba..e89b10684 100644 --- a/internal/data/sep41/balances.go +++ b/internal/data/sep41/balances.go @@ -73,7 +73,7 @@ func (m *BalanceModel) GetByAccount(ctx context.Context, accountAddress string, query := ` SELECT - b.contract_id, b.balance, b.last_modified_ledger, + b.contract_id, b.balance::text, b.last_modified_ledger, ct.contract_id AS token_id, ct.name, ct.symbol, ct.decimals FROM sep41_balances b INNER JOIN contract_tokens ct ON ct.id = b.contract_id @@ -171,13 +171,16 @@ func (m *BalanceModel) BatchApplyDeltas(ctx context.Context, dbTx pgx.Tx, deltas ledgers[i] = int32(d.LedgerNumber) } - // Sum in SQL: existing + delta. The TEXT columns are cast to numeric for arithmetic - // and back to text for storage so the column type stays TEXT (preserves full i128 range). + // Sum in SQL: existing + delta. balance is stored as NUMERIC so this is native + // arithmetic; the delta arrives as text (i128 decimal string) and is cast once at + // the boundary, since Postgres has no implicit text->numeric cast. const upsertQuery = ` INSERT INTO sep41_balances (account_id, contract_id, balance, last_modified_ledger) - SELECT * FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::integer[]) + SELECT u.account_id, u.contract_id, u.balance::numeric, u.last_modified_ledger + FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::integer[]) + AS u(account_id, contract_id, balance, last_modified_ledger) ON CONFLICT (account_id, contract_id) DO UPDATE SET - balance = (sep41_balances.balance::numeric + EXCLUDED.balance::numeric)::text, + balance = sep41_balances.balance + EXCLUDED.balance, last_modified_ledger = EXCLUDED.last_modified_ledger` if _, err := dbTx.Exec(ctx, upsertQuery, accountIDs, contractIDs, balances, ledgers); err != nil { m.Metrics.QueryErrors.WithLabelValues("BatchApplyDeltas", "sep41_balances", utils.GetDBErrorType(err)).Inc() @@ -186,7 +189,7 @@ func (m *BalanceModel) BatchApplyDeltas(ctx context.Context, dbTx pgx.Tx, deltas const deleteZeroesQuery = ` DELETE FROM sep41_balances - WHERE balance::numeric = 0 + WHERE balance = 0 AND (account_id, contract_id) IN ( SELECT * FROM UNNEST($1::bytea[], $2::uuid[]) )` diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 40fb21574..7da6bcdbd 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -2,8 +2,6 @@ package data import ( "context" - "crypto/rand" - "encoding/binary" "fmt" "strings" "time" @@ -34,7 +32,7 @@ var _ StateChangeWriter = (*StateChangeModel)(nil) // BatchGetByAccountAddress gets the state changes that are associated with the given account address. // Optional filters: txHash, operationID, category, and reason can be used to further filter results. func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder, timeRange *TimeRange) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} argIndex := 2 @@ -48,9 +46,11 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account // Time range filter: enables TimescaleDB chunk pruning on the state_changes hypertable args, argIndex = appendTimeRangeConditions(&queryBuilder, "ledger_created_at", timeRange, args, argIndex) - // Add transaction hash filter if provided (uses subquery to find to_id by hash) + // Add transaction hash filter if provided (uses subquery to find to_id(s) by hash). + // idx_transactions_hash is non-unique (TimescaleDB can't enforce unique(hash) on a + // hypertable), so this must tolerate more than one matching row instead of erroring. if txHash != nil { - fmt.Fprintf(&queryBuilder, " AND to_id = (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex) + fmt.Fprintf(&queryBuilder, " AND to_id IN (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex) args = append(args, types.HashBytea(*txHash)) argIndex++ } @@ -125,7 +125,10 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account } func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder var args []interface{} argIndex := 1 @@ -156,7 +159,7 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, to_id ASC, operation_id ASC, state_change_id ASC") } - if limit != nil && *limit > 0 { + if limit != nil { fmt.Fprintf(&queryBuilder, " LIMIT $%d", argIndex) args = append(args, *limit) } @@ -185,9 +188,16 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in // BatchCopy inserts state changes using pgx's binary COPY protocol. // Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). // +// Each StateChange must already carry its final, deterministic StateChangeID +// (see types.AssignStateChangeOrdinals) — BatchCopy writes it as-is and never +// generates one. +// // IMPORTANT: BatchCopy will FAIL if any duplicate records exist. The PostgreSQL COPY // protocol does not support conflict handling. Callers must ensure no duplicates exist -// before calling this method, or handle the unique constraint violation error appropriately. +// before calling this method, or handle the unique constraint violation error +// appropriately. Because state_change_id is now deterministic, this is also the +// idempotency backstop: re-copying an already-committed ledger's rows collides on the +// primary key and fails loudly instead of inserting duplicates. func (m *StateChangeModel) BatchCopy( ctx context.Context, pgxTx pgx.Tx, @@ -216,13 +226,6 @@ func (m *StateChangeModel) BatchCopy( pgx.CopyFromSlice(len(stateChanges), func(i int) ([]any, error) { sc := stateChanges[i] - // Generate a fresh random ID at insertion time so retries on PK collision - // produce new IDs automatically. - stateChangeID, err := generateStateChangeID() - if err != nil { - return nil, err - } - // Convert account_id to BYTEA (required field) accountBytes, err := sc.AccountID.Value() if err != nil { @@ -265,7 +268,7 @@ func (m *StateChangeModel) BatchCopy( return []any{ pgtype.Int8{Int64: sc.ToID, Valid: true}, - pgtype.Int8{Int64: stateChangeID, Valid: true}, + pgtype.Int8{Int64: sc.StateChangeID, Valid: true}, pgtype.Text{String: string(sc.StateChangeCategory), Valid: true}, pgtypeTextFromReason(sc.StateChangeReason), pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true}, @@ -321,28 +324,19 @@ func (m *StateChangeModel) BatchCopy( return len(stateChanges), nil } -// generateStateChangeID produces a random positive int64 from crypto/rand. -// Full 63 bits of entropy, masked to ensure a positive value. -func generateStateChangeID() (int64, error) { - var buf [8]byte - if _, err := rand.Read(buf[:]); err != nil { - return 0, fmt.Errorf("generating state change ID: %w", err) - } - return int64(binary.BigEndian.Uint64(buf[:]) & 0x7FFFFFFFFFFFFFFF), nil -} - -// BatchGetByToID gets state changes for a single transaction with pagination support. -func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// BatchGetByToID gets state changes for a single transaction with pagination support, pinned to +// the parent transaction's ledger_created_at for partition-column chunk exclusion. +func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM state_changes - WHERE to_id = $1 + WHERE to_id = $1 AND ledger_created_at = $2 `, columns) - args := []interface{}{toID} - argIndex := 2 + args := []interface{}{toID, ledgerCreatedAt} + argIndex := 3 // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. @@ -390,35 +384,55 @@ func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, colum return stateChanges, nil } -// BatchGetByToIDs gets the state changes that are associated with the given to_ids. -func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// BatchGetByToIDs gets the state changes that are associated with the given to_ids. Callers +// supply each to_id's parent ledger_created_at so the LATERAL can pin the partition column, +// giving per-key runtime chunk exclusion instead of a hash join across the whole hypertable. +func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + if len(toIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") + + // The ORDER BY + LIMIT live inside the LATERAL (a per-transaction top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-to_id cap; a subquery containing + // LIMIT cannot be flattened into a join, so the planner must run this as one per-key PK-band + // probe (see BatchGetByAccountAddress) instead of decompressing the whole hypertable. + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST), not sc.ledger_created_at: + // the LATERAL only projects the caller-requested columns, which may not include + // ledger_created_at, but k.ledger_created_at is always in scope and identical to + // sc.ledger_created_at for any row that survives the WHERE match. sc.to_id/operation_id/ + // state_change_id are safe to reference directly since prepareColumnsWithID always forces + // them into the projection. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column names, not "sc."-qualified ones — there is no "sc" in scope + // at that level, only the fenced derived table. ledger_created_at and to_id are dropped from + // the ORDER BY (not just unqualified): both are pinned to a single constant by the WHERE + // clause within one lateral probe, so they're no-op leading sort keys; operation_id and + // state_change_id are the only columns that actually vary among a probe's matched rows. + args := []interface{}{toIDs, ledgerCreatedAts} var queryBuilder strings.Builder - // This CTE query implements per-transaction pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all state changes from just a few - // transactions, we use ROW_NUMBER() with PARTITION BY to_id to limit results per transaction. - // This guarantees that each transaction gets at most 'limit' state changes, providing - // more balanced and predictable pagination across multiple transactions. fmt.Fprintf(&queryBuilder, ` - WITH - inputs (to_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_state_changes_per_to_id AS ( - SELECT - sc.*, - ROW_NUMBER() OVER (PARTITION BY sc.to_id ORDER BY sc.ledger_created_at %s, sc.to_id %s, sc.operation_id %s, sc.state_change_id %s) AS rn - FROM - state_changes sc - JOIN - inputs i ON sc.to_id = i.to_id - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM ranked_state_changes_per_to_id - `, sortOrder, sortOrder, sortOrder, sortOrder, columns) + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, sc.to_id as cursor_to_id, sc.operation_id as cursor_operation_id, sc.state_change_id as cursor_state_change_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(to_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM state_changes sc + WHERE sc.to_id = k.to_id AND sc.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY operation_id %s, state_change_id %s`, columns, columns, sortOrder, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } + queryBuilder.WriteString(` + ) sc + `) + query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. @@ -429,7 +443,7 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, c } start := time.Now() - stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, toIDs) + stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByToIDs", "state_changes").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByToIDs", "state_changes").Observe(float64(len(toIDs))) @@ -441,18 +455,19 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, c return stateChanges, nil } -// BatchGetByOperationID gets state changes for a single operation with pagination support. -func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// BatchGetByOperationID gets state changes for a single operation with pagination support, +// pinned to the parent operation's ledger_created_at for partition-column chunk exclusion. +func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM state_changes - WHERE operation_id = $1 + WHERE operation_id = $1 AND ledger_created_at = $2 `, columns) - args := []interface{}{operationID} - argIndex := 2 + args := []interface{}{operationID, ledgerCreatedAt} + argIndex := 3 // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. @@ -530,7 +545,7 @@ func (m *StateChangeModel) BatchGetAccountStateChangesByToIDs(ctx context.Contex hi = t } } - columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s FROM state_changes sc @@ -555,34 +570,53 @@ func (m *StateChangeModel) BatchGetAccountStateChangesByToIDs(ctx context.Contex } // BatchGetByOperationIDs gets the state changes that are associated with the given operation IDs. -func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// Callers supply each operation's parent ledger_created_at so the LATERAL can pin the partition +// column, giving per-key runtime chunk exclusion instead of a hash join across the whole +// hypertable. +func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + if len(operationIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("operationIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(operationIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") + + // The ORDER BY + LIMIT live inside the LATERAL (a per-operation top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-operation cap; a subquery + // containing LIMIT cannot be flattened into a join, so the planner must run this as one + // per-key probe (see BatchGetByAccountAddress) instead of decompressing the whole hypertable. + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST), not sc.ledger_created_at: + // the LATERAL only projects the caller-requested columns, which may not include + // ledger_created_at, but k.ledger_created_at is always in scope and identical to + // sc.ledger_created_at for any row that survives the WHERE match. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column names, not "sc."-qualified ones — there is no "sc" in scope + // at that level, only the fenced derived table. ledger_created_at and operation_id are + // dropped from the ORDER BY (not just unqualified): both are pinned to a single constant by + // the WHERE clause within one lateral probe, so they're no-op leading sort keys; to_id and + // state_change_id are the only columns that actually vary among a probe's matched rows. + args := []interface{}{operationIDs, ledgerCreatedAts} var queryBuilder strings.Builder - // This CTE query implements per-operation pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all state changes from just a few - // operations, we use ROW_NUMBER() with PARTITION BY operation_id to limit results per operation. - // This guarantees that each operation gets at most 'limit' state changes, providing - // more balanced and predictable pagination across multiple operations. fmt.Fprintf(&queryBuilder, ` - WITH - inputs (operation_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_state_changes_per_operation_id AS ( - SELECT - sc.*, - ROW_NUMBER() OVER (PARTITION BY sc.operation_id ORDER BY sc.ledger_created_at %s, sc.to_id %s, sc.operation_id %s, sc.state_change_id %s) AS rn - FROM - state_changes sc - JOIN - inputs i ON sc.operation_id = i.operation_id - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM ranked_state_changes_per_operation_id - `, sortOrder, sortOrder, sortOrder, sortOrder, columns) + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, sc.to_id as cursor_to_id, sc.operation_id as cursor_operation_id, sc.state_change_id as cursor_state_change_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(operation_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM state_changes sc + WHERE sc.operation_id = k.operation_id AND sc.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY to_id %s, state_change_id %s`, columns, columns, sortOrder, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } + queryBuilder.WriteString(` + ) sc + `) + query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. @@ -593,7 +627,7 @@ func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operation } start := time.Now() - stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, operationIDs) + stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByOperationIDs", "state_changes").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByOperationIDs", "state_changes").Observe(float64(len(operationIDs))) diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 24f7156f9..9b691ab19 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -215,6 +215,71 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { } } +// TestStateChangeModel_BatchCopy_DuplicateFailsOnPK verifies the SQL-06 +// idempotency backstop: since state_change_id is now a deterministic ordinal +// (assigned by types.AssignStateChangeOrdinals before BatchCopy is called, +// not randomly generated inside BatchCopy), re-copying an already-committed +// ledger's rows collides on the state_changes primary key and fails loudly +// instead of silently inserting duplicate rows. +func TestStateChangeModel_BatchCopy_DuplicateFailsOnPK(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + now := time.Now() + kp := keypair.MustRandom() + + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, "f176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", int64(1), 100, "TransactionResultCodeTxSuccess", uint32(1), now, false) + require.NoError(t, err) + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + conn, err := pgx.Connect(ctx, dbt.DSN) + require.NoError(t, err) + defer conn.Close(ctx) + + // Same (to_id, operation_id, state_change_id, ledger_created_at) as a + // deterministic emitter would recompute for the same ledger on a re-ingest. + sc := types.StateChange{ + ToID: 1, + StateChangeID: 1, + StateChangeCategory: types.StateChangeCategoryBalance, + StateChangeReason: types.StateChangeReasonAdd, + LedgerCreatedAt: now, + LedgerNumber: 1, + AccountID: types.AddressBytea(kp.Address()), + OperationID: 123, + } + + pgxTx, err := conn.Begin(ctx) + require.NoError(t, err) + gotCount, err := m.BatchCopy(ctx, pgxTx, []types.StateChange{sc}) + require.NoError(t, err) + require.NoError(t, pgxTx.Commit(ctx)) + assert.Equal(t, 1, gotCount) + + // Re-copying the identical row (simulating a re-ingest of the same ledger) + // must fail on the primary key rather than inserting a duplicate. + pgxTx2, err := conn.Begin(ctx) + require.NoError(t, err) + _, err = m.BatchCopy(ctx, pgxTx2, []types.StateChange{sc}) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") + pgxTx2.Rollback(ctx) + + dbInsertedIDs, err := db.QueryMany[string](ctx, dbConnectionPool, "SELECT CONCAT(to_id, '-', operation_id, '-', state_change_id) FROM state_changes") + require.NoError(t, err) + assert.Len(t, dbInsertedIDs, 1, "the failed re-copy must not have left duplicate rows") +} + func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -494,6 +559,57 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { }) } +// TestStateChangeModel_BatchGetByAccountAddress_DuplicateHashTolerated covers SQL-07: idx_transactions_hash +// is non-unique (TimescaleDB can't enforce unique(hash) on a hypertable), so a data bug could +// duplicate a hash across two to_ids. The txHash filter must tolerate this ("IN" rather than "=" / +// a scalar subquery) instead of erroring with "more than one row returned by a subquery". +func TestStateChangeModel_BatchGetByAccountAddress_DuplicateHashTolerated(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + now := time.Now() + address := keypair.MustRandom().Address() + dupHash := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000009") + + // Two different transactions (different to_id, the hypertable's real uniqueness boundary) + // sharing the same hash. + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES + ($1, 1, 100, 'TransactionResultCodeTxSuccess', 1, $2), + ($1, 2, 100, 'TransactionResultCodeTxSuccess', 2, $2) + `, dupHash, now) + require.NoError(t, err) + + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO state_changes (to_id, state_change_id, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) + VALUES + (1, 1, 'BALANCE', 'CREDIT', $1, 1, $2, 123), + (2, 1, 'BALANCE', 'CREDIT', $1, 2, $2, 456) + `, now, types.AddressBytea(address)) + require.NoError(t, err) + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + txHash := dupHash.String() + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC, nil) + require.NoError(t, err, "duplicate hash must not error as 'more than one row returned by a subquery'") + assert.Len(t, stateChanges, 2, "state changes from both to_ids sharing the hash must be returned") + + toIDsFound := make(map[int64]bool) + for _, sc := range stateChanges { + toIDsFound[sc.StateChange.ToID] = true + } + assert.True(t, toIDsFound[1]) + assert.True(t, toIDsFound[2]) +} + func TestStateChangeModel_GetAll(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -671,7 +787,11 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { Metrics: dbMetrics, } - stateChanges, err := m.BatchGetByToIDs(ctx, tc.toIDs, "", tc.limit, tc.sortOrder) + ledgerCreatedAts := make([]time.Time, len(tc.toIDs)) + for i := range ledgerCreatedAts { + ledgerCreatedAts[i] = now + } + stateChanges, err := m.BatchGetByToIDs(ctx, tc.toIDs, ledgerCreatedAts, "", tc.limit, tc.sortOrder) require.NoError(t, err) assert.Len(t, stateChanges, tc.expectedCount) @@ -689,6 +809,43 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { } }) } + + t.Run("wrong ledger_created_at for a key returns no state changes for that key (time pin enforced)", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByToIDs(ctx, []int64{1}, []time.Time{wrongTime}, "", nil, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + _, err := m.BatchGetByToIDs(ctx, []int64{1, 2}, []time.Time{now}, "", nil, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns) — a narrow column set that + // excludes ledgerCreatedAt must not break the query with "column does not exist". + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + stateChanges, err := m.BatchGetByToIDs(ctx, []int64{1}, []time.Time{now}, "state_change_category", nil, ASC) + require.NoError(t, err) + require.Len(t, stateChanges, 3) + for _, sc := range stateChanges { + assert.NotZero(t, sc.StateChangeCursor.LedgerCreatedAt) + } + }) } func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { @@ -736,7 +893,8 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { // Test BatchGetByOperationID limit := int32(10) - stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, "", &limit, ASC) + ledgerCreatedAts := []time.Time{now, now} + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, ledgerCreatedAts, "", &limit, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -747,6 +905,30 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { } assert.Equal(t, 2, operationIDsFound[123]) assert.Equal(t, 1, operationIDsFound[456]) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123}, []time.Time{wrongTime}, "", &limit, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, []time.Time{now}, "", &limit, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns). + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123}, []time.Time{now}, "state_change_category", &limit, ASC) + require.NoError(t, err) + require.Len(t, stateChanges, 2) + for _, sc := range stateChanges { + assert.NotZero(t, sc.StateChangeCursor.LedgerCreatedAt) + } + }) } func TestStateChangeModel_BatchGetByToID(t *testing.T) { @@ -792,7 +974,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) t.Run("get all state changes for single to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -809,7 +991,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with pagination - first", func(t *testing.T) { limit := int32(2) - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", &limit, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -820,7 +1002,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with cursor pagination", func(t *testing.T) { limit := int32(2) cursor := &types.StateChangeCursor{LedgerCreatedAt: now, ToID: 1, OperationID: 123, StateChangeID: 1} - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, cursor, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", &limit, cursor, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -830,7 +1012,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("get state changes with DESC ordering", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, DESC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", nil, nil, DESC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -841,7 +1023,14 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("no state changes for non-existent to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 999, now, "", nil, nil, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("wrong ledger_created_at returns no state changes (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByToID(ctx, 1, wrongTime, "", nil, nil, ASC) require.NoError(t, err) assert.Empty(t, stateChanges) }) @@ -976,6 +1165,24 @@ func TestStateChangeModel_BatchGetAccountStateChangesByToIDs(t *testing.T) { assert.Equal(t, int64(1), scs[1].StateChangeID) }) + t.Run("minimal projection still hydrates the loader-key columns", func(t *testing.T) { + // The operation/transaction relationship resolvers build their dataloader + // key from (to_id, operation_id, state_change_id). A client selection that + // maps to none of those scalars must still come back with them hydrated, + // or the key collapses to "-0-0": nested operation/transaction + // resolve to null and the loader dedups every row into one. + scs, err := m.BatchGetAccountStateChangesByToIDs(ctx, acct, []int64{4096}, []time.Time{now}, "state_change_category") + require.NoError(t, err) + require.Len(t, scs, 2) + for _, sc := range scs { + assert.Equal(t, int64(4096), sc.ToID, "to_id must be hydrated") + assert.Equal(t, int64(4097), sc.OperationID, "operation_id must be hydrated for the loader key") + assert.NotZero(t, sc.StateChangeID, "state_change_id must be hydrated for the loader key") + assert.Equal(t, acct, sc.AccountID.String(), "account_id must be hydrated") + assert.True(t, now.Equal(sc.LedgerCreatedAt), "ledger_created_at must be hydrated") + } + }) + t.Run("empty when account has no state changes in the transaction", func(t *testing.T) { none := keypair.MustRandom().Address() scs, err := m.BatchGetAccountStateChangesByToIDs(ctx, none, []int64{4096}, []time.Time{now}, "") @@ -1009,3 +1216,37 @@ func TestStateChangeModel_BatchGetAccountStateChangesByToIDs(t *testing.T) { assert.Equal(t, int64(1), scs[2].StateChangeID) }) } + +// A client selecting no time fields must still get state changes whose LedgerCreatedAt is +// hydrated: the operation and transaction relationship resolvers pin their lookups on the +// parent state change's partition timestamp, so every projection forces ledger_created_at. +func TestStateChangeModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + m := &StateChangeModel{DB: dbConnectionPool, Metrics: metrics.NewMetrics(reg).DB} + + now := time.Now().UTC().Truncate(time.Microsecond) + address := keypair.MustRandom().Address() + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO state_changes (to_id, state_change_id, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) + VALUES (4096, 1, 'BALANCE', 'CREDIT', $1, 1, $2, 4098) + `, now, types.AddressBytea(address)) + require.NoError(t, err) + + limit := int32(10) + byToID, err := m.BatchGetByToID(ctx, 4096, now, "state_change_category", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, byToID, 1) + assert.True(t, now.Equal(byToID[0].StateChange.LedgerCreatedAt), "BatchGetByToID with minimal projection must hydrate ledger_created_at") + + byOpID, err := m.BatchGetByOperationID(ctx, 4098, now, "state_change_category", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, byOpID, 1) + assert.True(t, now.Equal(byOpID[0].StateChange.LedgerCreatedAt), "BatchGetByOperationID with minimal projection must hydrate ledger_created_at") +} diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 9ec07b9f3..c547f3191 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -23,7 +23,7 @@ type TransactionModel struct { } func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns string) (*types.Transaction, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id", "ledger_created_at") query := fmt.Sprintf(`SELECT %s FROM transactions WHERE hash = $1`, columns) start := time.Now() hashBytea := types.HashBytea(hash) @@ -39,7 +39,10 @@ func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns s } func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.TransactionWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } + columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id", "ledger_created_at") queryBuilder := strings.Builder{} var args []interface{} argIndex := 1 @@ -188,7 +191,7 @@ func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, account // // See SEP-35: https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0035.md func (m *TransactionModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string) ([]*types.TransactionWithOperationID, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id", "ledger_created_at") // Join operations to transactions using TOID encoding: // An operation ID's lower 12 bits encode the operation index within the transaction. // Masking these bits (id &~ 0xFFF) gives the transaction's to_id. @@ -212,26 +215,33 @@ func (m *TransactionModel) BatchGetByOperationIDs(ctx context.Context, operation return transactions, nil } -// BatchGetByStateChangeIDs gets the transactions that are associated with the given state changes -func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, columns string) ([]*types.TransactionWithStateChangeID, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") - - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(scOrders)) - for i := range scOrders { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], scOrders[i]) +// BatchGetByStateChangeIDs gets the transactions that are associated with the given state changes. +// The state-change key triple's to_id already identifies the parent transaction (to_id is the PK +// prefix of state_changes), so this probes transactions directly instead of joining through +// state_changes. Callers supply each key's parent ledger_created_at (the state change's ledger +// time, same ledger as its transaction) so the LATERAL can pin the complete transactions primary +// key (to_id, ledger_created_at). The LATERAL LIMIT 1 is load-bearing: a subquery containing LIMIT +// cannot be flattened into a join, so the planner must run this as one per-key PK probe (see +// BatchGetByAccountAddress) instead of decompressing the whole hypertable to hash-join a full +// IN-list of (to_id, operation_id, state_change_id) tuples. +func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, ledgerCreatedAts []time.Time, columns string) ([]*types.TransactionWithStateChangeID, error) { + if len(scToIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("scToIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scToIDs), len(ledgerCreatedAts)) } + columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id", "ledger_created_at") query := fmt.Sprintf(` - SELECT %s, CONCAT(sc.to_id, '-', sc.operation_id, '-', sc.state_change_id) as state_change_id - FROM transactions - INNER JOIN state_changes sc ON transactions.to_id = sc.to_id - WHERE (sc.to_id, sc.operation_id, sc.state_change_id) IN (%s) - `, columns, strings.Join(tuples, ", ")) + SELECT %s, CONCAT(k.to_id, '-', k.operation_id, '-', k.state_change_id) AS state_change_id + FROM UNNEST($1::bigint[], $2::bigint[], $3::bigint[], $4::timestamptz[]) AS k(to_id, operation_id, state_change_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT %s FROM transactions t + WHERE t.to_id = k.to_id AND t.ledger_created_at = k.ledger_created_at + LIMIT 1 + ) t + `, columns, columns) start := time.Now() - transactions, err := db.QueryManyPtrs[types.TransactionWithStateChangeID](ctx, m.DB, query) + transactions, err := db.QueryManyPtrs[types.TransactionWithStateChangeID](ctx, m.DB, query, scToIDs, scOpIDs, scOrders, ledgerCreatedAts) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByStateChangeIDs", "transactions").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByStateChangeIDs", "transactions").Observe(float64(len(scOrders))) diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index de94141ff..0ae2e5a28 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -429,7 +429,8 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { require.NoError(t, err) // Test BatchGetByStateChangeID - transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2, 3}, []int64{1, 2, 3}, []int64{1, 1, 1}, "") + ledgerCreatedAts := []time.Time{now, now, now} + transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2, 3}, []int64{1, 2, 3}, []int64{1, 1, 1}, ledgerCreatedAts, "") require.NoError(t, err) assert.Len(t, transactions, 3) @@ -442,6 +443,19 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, scTestHash1, stateChangeIDsFound["1-1-1"]) // to_id=1 -> scTestHash1 (to_id=1) assert.Equal(t, scTestHash2, stateChangeIDsFound["2-2-1"]) // to_id=2 -> scTestHash2 (to_id=2) assert.Equal(t, scTestHash3, stateChangeIDsFound["3-3-1"]) // to_id=3 -> scTestHash3 (to_id=3) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1}, []int64{1}, []int64{1}, []time.Time{wrongTime}, "") + require.NoError(t, err) + assert.Empty(t, transactions) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2}, []int64{1, 2}, []int64{1, 1}, []time.Time{now}, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) } // BenchmarkTransactionModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. @@ -509,3 +523,49 @@ func BenchmarkTransactionModel_BatchCopy(b *testing.B) { }) } } + +// A client selecting no time fields must still get rows whose LedgerCreatedAt is hydrated: +// relationship resolvers pin their child lookups (operations, state changes) on the parent +// transaction's partition timestamp, so every projection forces ledger_created_at. +func TestTransactionModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &TransactionModel{DB: dbConnectionPool, Metrics: dbMetrics} + opModel := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + now := time.Now().UTC().Truncate(time.Microsecond) + txHash := types.HashBytea("0076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $2, false) + `, txHash, now) + require.NoError(t, err) + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) + VALUES (4098, 'PAYMENT', 'xdr', 'op_success', true, 1, $1) + `, now) + require.NoError(t, err) + + tx, err := m.GetByHash(ctx, txHash.String(), "hash") + require.NoError(t, err) + assert.True(t, now.Equal(tx.LedgerCreatedAt), "GetByHash with minimal projection must hydrate ledger_created_at") + + limit := int32(10) + txs, err := m.GetAll(ctx, "hash", &limit, nil, DESC) + require.NoError(t, err) + require.Len(t, txs, 1) + assert.True(t, now.Equal(txs[0].Transaction.LedgerCreatedAt), "GetAll with minimal projection must hydrate ledger_created_at") + + // End-to-end: the hydrated timestamp pins the child operations lookup. + ops, err := opModel.BatchGetByToID(ctx, tx.ToID, tx.LedgerCreatedAt, "id", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, ops, 1, "time-pinned child lookup must find the operation via the hydrated timestamp") + assert.Equal(t, int64(4098), ops[0].Operation.ID) +} diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 6043f34a2..af0bbd882 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -20,6 +20,19 @@ CREATE TABLE transactions ( ); SELECT enable_chunk_skipping('transactions', 'to_id'); +-- ledger_number rises monotonically with the partition column, so per-chunk min/max +-- ranges let ledger_number-windowed queries (e.g. backfill gap detection) exclude +-- non-overlapping chunks at plan time. +SELECT enable_chunk_skipping('transactions', 'ledger_number'); + +-- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB +-- auto-creates on the hypertable's partition column. The root GraphQL connection +-- sorts by (ledger_created_at DESC, to_id DESC), which the default index can't serve +-- for a top-N first page (it still needs a heapsort on to_id within each timestamp). +-- This composite index matches the API sort key directly and makes the auto-created +-- one redundant. +DROP INDEX IF EXISTS transactions_ledger_created_at_idx; +CREATE INDEX idx_transactions_created_at_sort ON transactions (ledger_created_at DESC, to_id DESC); CREATE INDEX idx_transactions_hash ON transactions(hash); @@ -28,7 +41,12 @@ CREATE TABLE transactions_accounts ( tx_to_id BIGINT NOT NULL, account_id BYTEA NOT NULL, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, tx_to_id, ledger_created_at) + -- Column order matches BatchGetByAccountAddress's access shape: account_id is pinned by + -- equality, then the walk sorts by (ledger_created_at, tx_to_id). With account_id fixed, + -- the planner serves that sort via an index (only) scan over this PK -- forward or + -- backward -- so the dedicated (account_id, ledger_created_at DESC, tx_to_id DESC) index + -- once needed alongside it is redundant. + PRIMARY KEY (account_id, ledger_created_at, tx_to_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -39,8 +57,12 @@ CREATE TABLE transactions_accounts ( SELECT enable_chunk_skipping('transactions_accounts', 'tx_to_id'); +-- TimescaleDB's default single-column index on the partition column. Retention drops chunks +-- by range metadata, and no query path filters this table by bare ledger_created_at, so it +-- has zero consumers. +DROP INDEX IF EXISTS transactions_accounts_ledger_created_at_idx; + CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); -CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC, tx_to_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 77929329f..9b0abd0df 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -34,12 +34,28 @@ CREATE TABLE operations ( SELECT enable_chunk_skipping('operations', 'id'); +-- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB +-- auto-creates on the hypertable's partition column. The root GraphQL connection +-- sorts by (ledger_created_at DESC, id DESC), which the default index can't serve +-- for a top-N first page (it still needs a heapsort on id within each timestamp). +-- This composite index matches the API sort key directly and makes the auto-created +-- one redundant. +DROP INDEX IF EXISTS operations_ledger_created_at_idx; +CREATE INDEX idx_operations_created_at_sort ON operations (ledger_created_at DESC, id DESC); + -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE operations_accounts ( operation_id BIGINT NOT NULL, account_id BYTEA NOT NULL, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, operation_id, ledger_created_at) + -- Column order (account_id, ledger_created_at, operation_id) serves both consumers as a + -- single index: BatchGetByAccountAddress pins account_id by equality and sorts by + -- (ledger_created_at, operation_id), which the planner walks as an index (only) scan over + -- this PK -- forward or backward; BatchGetAccountOperationsByToIDs probes equality on + -- account_id, equality on ledger_created_at, and a range on operation_id, which fits this + -- key as a perfect prefix. The dedicated (account_id, ledger_created_at DESC, operation_id + -- DESC) index once needed for the former is now redundant. + PRIMARY KEY (account_id, ledger_created_at, operation_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -50,8 +66,12 @@ CREATE TABLE operations_accounts ( SELECT enable_chunk_skipping('operations_accounts', 'operation_id'); +-- TimescaleDB's default single-column index on the partition column. Retention drops chunks +-- by range metadata, and no query path filters this table by bare ledger_created_at, so it +-- has zero consumers. +DROP INDEX IF EXISTS operations_accounts_ledger_created_at_idx; + CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); -CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC, operation_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index f4fb26150..2d7e426e2 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -45,7 +45,12 @@ CREATE TABLE state_changes ( key_value JSONB, to_muxed_id TEXT, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (to_id, operation_id, state_change_id, ledger_created_at) + -- The root GraphQL connection sorts by (ledger_created_at DESC, to_id DESC, operation_id + -- DESC, state_change_id DESC). This PK's column order matches that sort key directly, so + -- a top-N first page is served by scanning the PK itself -- forward or backward, no + -- heapsort -- which also makes it this table's replacement for the default single-column + -- index TimescaleDB would otherwise auto-create on the partition column. + PRIMARY KEY (ledger_created_at, to_id, operation_id, state_change_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -58,6 +63,10 @@ CREATE TABLE state_changes ( SELECT enable_chunk_skipping('state_changes', 'to_id'); SELECT enable_chunk_skipping('state_changes', 'operation_id'); +-- TimescaleDB's default single-column index on the partition column; the reordered PK above +-- is now a superset of it (ledger_created_at is its leading column), so it's redundant. +DROP INDEX IF EXISTS state_changes_ledger_created_at_idx; + CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); CREATE INDEX idx_state_changes_account_category ON state_changes(account_id, state_change_category, state_change_reason, ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); diff --git a/internal/db/migrations/2026-01-12.0-trustline_balances.sql b/internal/db/migrations/2026-01-12.0-trustline_balances.sql index 6b886d6c5..585a7ca64 100644 --- a/internal/db/migrations/2026-01-12.0-trustline_balances.sql +++ b/internal/db/migrations/2026-01-12.0-trustline_balances.sql @@ -20,27 +20,27 @@ CREATE TABLE trustline_balances ( FOREIGN KEY (asset_id) REFERENCES trustline_assets(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, -- Base dead-row count added to (scale_factor * total_rows). Default is fine here -- since the scale factor already keeps the threshold low. autovacuum_vacuum_threshold = 50, - -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, - -- so stale stats can cause bad query plans (e.g. on the GetByAccount JOIN). + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger. + -- The current readers are PK/nested-loop lookups and stats-insensitive; fresh stats + -- matter for range/aggregate queries. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-15.0-native_balances.sql b/internal/db/migrations/2026-01-15.0-native_balances.sql index 282a9952a..c76b7d90a 100644 --- a/internal/db/migrations/2026-01-15.0-native_balances.sql +++ b/internal/db/migrations/2026-01-15.0-native_balances.sql @@ -14,10 +14,12 @@ CREATE TABLE native_balances ( num_subentries INTEGER NOT NULL DEFAULT 0, last_modified_ledger BIGINT NOT NULL DEFAULT 0 ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, @@ -28,13 +30,10 @@ CREATE TABLE native_balances ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-16.0-sac-balances.sql b/internal/db/migrations/2026-01-16.0-sac-balances.sql index db97ecf43..8fdf1bd14 100644 --- a/internal/db/migrations/2026-01-16.0-sac-balances.sql +++ b/internal/db/migrations/2026-01-16.0-sac-balances.sql @@ -9,7 +9,7 @@ CREATE TABLE sac_balances ( account_id BYTEA NOT NULL, contract_id UUID NOT NULL, - balance TEXT NOT NULL DEFAULT '0', + balance NUMERIC NOT NULL DEFAULT 0, is_authorized BOOLEAN NOT NULL DEFAULT true, is_clawback_enabled BOOLEAN NOT NULL DEFAULT false, last_modified_ledger INTEGER NOT NULL DEFAULT 0, @@ -18,10 +18,12 @@ CREATE TABLE sac_balances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, @@ -32,13 +34,10 @@ CREATE TABLE sac_balances ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-04-17.0-sep41_balances.sql b/internal/db/migrations/2026-04-17.0-sep41_balances.sql index 00904116d..866c31c36 100644 --- a/internal/db/migrations/2026-04-17.0-sep41_balances.sql +++ b/internal/db/migrations/2026-04-17.0-sep41_balances.sql @@ -7,20 +7,19 @@ CREATE TABLE sep41_balances ( account_id BYTEA NOT NULL, contract_id UUID NOT NULL, - balance TEXT NOT NULL DEFAULT '0', + balance NUMERIC NOT NULL DEFAULT 0, last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (account_id, contract_id), CONSTRAINT fk_sep41_contract_token FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql index 9242fa3ce..9465524e5 100644 --- a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql +++ b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql @@ -7,7 +7,7 @@ CREATE TABLE sep41_allowances ( owner_id BYTEA NOT NULL, spender_id BYTEA NOT NULL, contract_id UUID NOT NULL, - amount TEXT NOT NULL DEFAULT '0', + amount NUMERIC NOT NULL DEFAULT 0, expiration_ledger INTEGER NOT NULL DEFAULT 0, last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (owner_id, spender_id, contract_id), @@ -15,17 +15,20 @@ CREATE TABLE sep41_allowances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -CREATE INDEX idx_sep41_allowances_owner ON sep41_allowances(owner_id); -CREATE INDEX idx_sep41_allowances_spender ON sep41_allowances(spender_id); +-- owner_id is the PK's leading column, so a lookup by owner alone already uses the PK +-- index; a dedicated single-column index on it would be redundant. +-- No code path filters by spender_id alone, so a dedicated index for it would sit unused. +-- Backs the expired-allowance sweep's ORDER BY expiration_ledger LIMIT n. BatchUpsert +-- updates expiration_ledger, so this makes those updates non-HOT — an accepted +-- tradeoff at allowance write volumes. CREATE INDEX idx_sep41_allowances_expiration_ledger ON sep41_allowances(expiration_ledger); -- +migrate Down diff --git a/internal/db/migrations/2026-07-02.0-liquidity_pools.sql b/internal/db/migrations/2026-07-02.0-liquidity_pools.sql index ec58deb79..b7e3c6742 100644 --- a/internal/db/migrations/2026-07-02.0-liquidity_pools.sql +++ b/internal/db/migrations/2026-07-02.0-liquidity_pools.sql @@ -17,8 +17,9 @@ CREATE TABLE liquidity_pools ( last_modified_ledger BIGINT NOT NULL DEFAULT 0 ) WITH ( -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. - -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. + -- Unlike the other balance/state tables, this table's reserves update on essentially + -- every row per day (each pool's amount_a/amount_b shifts on every ledger it trades), + -- so it needs the full 20% reserve to sustain HOT updates. fillfactor = 80, -- Trigger vacuum when 2% of rows are dead (default 20%). autovacuum_vacuum_scale_factor = 0.02, @@ -27,12 +28,10 @@ CREATE TABLE liquidity_pools ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql b/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql index 80c0e4802..5ba35f9ed 100644 --- a/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql +++ b/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql @@ -13,23 +13,24 @@ CREATE TABLE liquidity_pool_balances ( last_modified_ledger BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (account_id, pool_id) ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, - -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, - -- so stale stats can cause bad query plans (e.g. on the GetByAccount JOIN). + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger. + -- The current readers are PK/nested-loop lookups and stats-insensitive; fresh stats + -- matter for range/aggregate queries. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 6f67a7146..5deb57b4b 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -70,8 +70,19 @@ type ParticipantsProcessorInterface interface { } type OperationProcessorInterface interface { + // ProcessOperation returns this operation's state changes. The slice must be + // in canonical, reproducible order — derived from the transaction meta / XDR + // walk order, never Go map iteration, goroutine completion, or channel + // arrival order — because types.AssignStateChangeOrdinals freezes each + // element's state_change_id from its position within the (to_id, operation_id) + // group. Cross-operation ordering is free: ordinals are keyed per group, so + // only order WITHIN one call's returned slice is load-bearing. ProcessOperation(ctx context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) Name() string + // StateChangeSubBase is this processor's slot in the indexer's state_change_id + // sub-namespace registry (see types.StateChangeSubBase*). It is part of the + // on-disk ID layout and must never change once rows exist with it. + StateChangeSubBase() int64 } // LedgerChangeProcessor is a generic interface for processors that extract data from ledger changes. @@ -110,8 +121,14 @@ type Indexer struct { // during ledger meta processing; classification is performed downstream // (per-batch) by services.DispatchClassification — this keeps the indexer // agnostic of any specific protocol or its validator shape. -func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metrics.IngestionMetrics) *Indexer { - return &Indexer{ +// +// It validates the state_change_id sub-base registry of the built processor +// set (see validateStateChangeSubBases), so a stale or duplicated sub-base +// copied into a new processor fails fast at startup rather than surfacing as +// a state_changes primary-key violation on the first ledger where two +// processors emit for the same operation. +func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metrics.IngestionMetrics) (*Indexer, error) { + indexer := &Indexer{ participantsProcessor: processors.NewParticipantsProcessor(networkPassphrase), tokenTransferProcessor: processors.NewTokenTransferProcessor(networkPassphrase, ingestionMetrics), sacBalancesProcessor: processors.NewSACBalancesProcessor(networkPassphrase, ingestionMetrics), @@ -131,6 +148,34 @@ func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metr ingestionMetrics: ingestionMetrics, networkPassphrase: networkPassphrase, } + if err := validateStateChangeSubBases(indexer.processors); err != nil { + return nil, fmt.Errorf("validating state_change_id sub-bases: %w", err) + } + return indexer, nil +} + +// validateStateChangeSubBases validates the state_change_id sub-base registry +// of the given processor set (see types.StateChangeSubBase*): every sub-base +// must be a non-negative multiple of types.StateChangeSubNamespaceWidth that +// fits inside the indexer's emitter namespace, and no two streams may share +// one. The token-transfer stream's slot is reserved up front since it is +// emitted outside the processors slice (see getTransactionStateChanges). +func validateStateChangeSubBases(procs []OperationProcessorInterface) error { + streamBySubBase := map[int64]string{types.StateChangeSubBaseTokenTransfer: "token_transfer"} + for _, p := range procs { + subBase := p.StateChangeSubBase() + if subBase <= 0 || subBase%types.StateChangeSubNamespaceWidth != 0 || subBase >= types.StateChangeOrdinalNamespaceWidth { + return fmt.Errorf("processor %q has invalid state_change_id sub-base %d: "+ + "must be a positive multiple of %d below %d", + p.Name(), subBase, types.StateChangeSubNamespaceWidth, types.StateChangeOrdinalNamespaceWidth) + } + if other, dup := streamBySubBase[subBase]; dup { + return fmt.Errorf("indexer streams %q and %q share state_change_id sub-base %d", + other, p.Name(), subBase) + } + streamBySubBase[subBase] = p.Name() + } + return nil } // ProcessLedgerTransactions processes all transactions in a ledger in parallel. @@ -342,13 +387,9 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa } } - // Insert state changes + // Insert state changes (already filtered and ID-assigned per processor + // stream by getTransactionStateChanges) for _, stateChange := range stateChanges { - // Skip empty state changes (no account to associate with) - if stateChange.AccountID == "" { - continue - } - // Get the correct operation for this state change var operation types.Operation if stateChange.OperationID != 0 { @@ -366,19 +407,23 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa return allParticipants.Cardinality(), nil } -// getTransactionStateChanges processes operations of a transaction and calculates all state changes +// getTransactionStateChanges processes operations of a transaction and calculates all state +// changes. Each emitting processor's stream is filtered and given its deterministic +// state_change_ids independently, inside that processor's own sub-namespace (see the sub-base +// registry in types), so the returned changes are ready to persist and no processor's IDs +// depend on another processor's output or on registration order. func (i *Indexer) getTransactionStateChanges(ctx context.Context, transaction ingest.LedgerTransaction, opsParticipants map[int64]processors.OperationParticipants) ([]types.StateChange, error) { - stateChanges := []types.StateChange{} + streams := make([][]types.StateChange, len(i.processors)) // Process operations sequentially since there are only 3 processors per operation // Creating a worker pool here adds unnecessary overhead for _, opParticipants := range opsParticipants { - for _, processor := range i.processors { + for procIdx, processor := range i.processors { processorStateChanges, processorErr := processor.ProcessOperation(ctx, opParticipants.OpWrapper) if processorErr != nil && !errors.Is(processorErr, processors.ErrInvalidOpType) { return nil, fmt.Errorf("processing %s state changes: %w", processor.Name(), processorErr) } - stateChanges = append(stateChanges, processorStateChanges...) + streams[procIdx] = append(streams[procIdx], processorStateChanges...) } } @@ -387,11 +432,31 @@ func (i *Indexer) getTransactionStateChanges(ctx context.Context, transaction in if err != nil { return nil, fmt.Errorf("processing token transfer state changes: %w", err) } - stateChanges = append(stateChanges, tokenTransferStateChanges...) + var stateChanges []types.StateChange + for procIdx, processor := range i.processors { + stateChanges = assignStateChangeStream(stateChanges, streams[procIdx], processor.StateChangeSubBase()) + } + stateChanges = assignStateChangeStream(stateChanges, tokenTransferStateChanges, types.StateChangeSubBaseTokenTransfer) return stateChanges, nil } +// assignStateChangeStream drops stream entries with no account to associate with, assigns +// the retained ones their deterministic state_change_ids at the emitting processor's slot in +// the indexer namespace, and appends them to dst. Filtering before assignment keeps ordinals +// gap-free (1..N per (to_id, operation_id) group) for what's actually persisted. +func assignStateChangeStream(dst, stream []types.StateChange, subBase int64) []types.StateChange { + retained := stream[:0] + for _, stateChange := range stream { + if stateChange.AccountID == "" { + continue + } + retained = append(retained, stateChange) + } + types.AssignStateChangeOrdinals(retained, types.StateChangeOrdinalBaseIndexer+subBase) + return append(dst, retained...) +} + // GetLedgerTransactions extracts transactions from ledger close meta. func GetLedgerTransactions(ctx context.Context, networkPassphrase string, ledgerMeta xdr.LedgerCloseMeta) ([]ingest.LedgerTransaction, error) { ledgerTxReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, ledgerMeta) diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 11ef00f7a..811c98e0c 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -145,8 +145,9 @@ func TestIndexer_NewIndexer(t *testing.T) { networkPassphrase := network.TestNetworkPassphrase pool := pond.NewPool(runtime.NumCPU()) - indexer := NewIndexer(networkPassphrase, pool, nil) + indexer, err := NewIndexer(networkPassphrase, pool, nil) + require.NoError(t, err) require.NotNil(t, indexer) assert.NotNil(t, indexer.participantsProcessor) assert.NotNil(t, indexer.tokenTransferProcessor) @@ -155,6 +156,67 @@ func TestIndexer_NewIndexer(t *testing.T) { assert.Len(t, indexer.processors, 3) // effects, contract deploy, SAC events } +func TestValidateStateChangeSubBases(t *testing.T) { + newProc := func(name string, subBase int64) *MockOperationProcessor { + p := &MockOperationProcessor{SubBase: subBase} + p.On("Name").Return(name).Maybe() + return p + } + + testCases := []struct { + name string + procs []OperationProcessorInterface + wantErrContains string + }{ + { + name: "🟢 distinct aligned sub-bases", + procs: []OperationProcessorInterface{ + newProc("effects", types.StateChangeSubBaseEffects), + newProc("contract_deploy", types.StateChangeSubBaseContractDeploy), + }, + }, + { + name: "🔴 duplicate sub-base between two processors", + procs: []OperationProcessorInterface{ + newProc("effects", types.StateChangeSubBaseEffects), + newProc("copy_pasted", types.StateChangeSubBaseEffects), + }, + wantErrContains: `indexer streams "effects" and "copy_pasted" share state_change_id sub-base`, + }, + { + name: "🔴 sub-base 0 is reserved for the token transfer stream", + procs: []OperationProcessorInterface{newProc("zero", 0)}, + wantErrContains: `processor "zero" has invalid state_change_id sub-base 0`, + }, + { + name: "🔴 sub-base not a multiple of the sub-namespace width", + procs: []OperationProcessorInterface{newProc("misaligned", types.StateChangeSubNamespaceWidth+1)}, + wantErrContains: "invalid state_change_id sub-base", + }, + { + name: "🔴 negative sub-base", + procs: []OperationProcessorInterface{newProc("negative", -types.StateChangeSubNamespaceWidth)}, + wantErrContains: "invalid state_change_id sub-base", + }, + { + name: "🔴 sub-base outside the indexer emitter namespace", + procs: []OperationProcessorInterface{newProc("oversized", types.StateChangeOrdinalNamespaceWidth)}, + wantErrContains: "invalid state_change_id sub-base", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateStateChangeSubBases(tc.procs) + if tc.wantErrContains == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.wantErrContains) + } + }) + } +} + func TestIndexer_ProcessLedgerTransactions(t *testing.T) { t.Run("🟢 single transaction with participants", func(t *testing.T) { // Create mocks @@ -764,9 +826,10 @@ func TestIndexer_ProcessLedgerTransactions_FeePhaseBalances(t *testing.T) { }, } - indexer := NewIndexer(network.TestNetworkPassphrase, pond.NewPool(runtime.NumCPU()), nil) + indexer, err := NewIndexer(network.TestNetworkPassphrase, pond.NewPool(runtime.NumCPU()), nil) + require.NoError(t, err) buffer := NewIndexerBuffer() - _, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{feeTx}, buffer) + _, err = indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{feeTx}, buffer) require.NoError(t, err) accountChanges := buffer.GetAccountChanges() @@ -1355,3 +1418,54 @@ func TestExtractContractEventsForLedger_CorpusCoversWrapperVersions(t *testing.T require.True(t, wrapperWithEvents[1], "corpus must include a wrapper-V1 (Protocol 20-22) ledger with extracted contract events") require.True(t, wrapperWithEvents[2], "corpus must include a wrapper-V2 (Protocol 23+) ledger with extracted contract events") } + +func TestAssignStateChangeStream_SubNamespaces(t *testing.T) { + effects := []types.StateChange{ + {OperationID: 42, AccountID: "GA"}, + {OperationID: 42, AccountID: "GB"}, + {OperationID: 43, AccountID: "GC"}, + } + tokenTransfers := []types.StateChange{ + {OperationID: 42, AccountID: "GD"}, + {OperationID: 42, AccountID: ""}, // no account — dropped before assignment + {OperationID: 42, AccountID: "GE"}, + } + + var merged []types.StateChange + merged = assignStateChangeStream(merged, effects, types.StateChangeSubBaseEffects) + merged = assignStateChangeStream(merged, tokenTransfers, types.StateChangeSubBaseTokenTransfer) + + // Each stream numbers its own emissions 1..N per operation inside its own + // sub-namespace; the account-less entry is filtered out first so ordinals + // stay gap-free. + require.Len(t, merged, 5) + assert.Equal(t, types.StateChangeSubBaseEffects+1, merged[0].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseEffects+2, merged[1].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseEffects+1, merged[2].StateChangeID) // op 43: its own group + assert.Equal(t, types.StateChangeSubBaseTokenTransfer+1, merged[3].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseTokenTransfer+2, merged[4].StateChangeID) + + // The IDs a stream gets are a function of that stream alone: assigning the + // same effects emissions with no other stream present yields identical IDs. + // This is the property that makes IDs immune to processors being added, + // removed, or reordered around this one. + alone := assignStateChangeStream(nil, []types.StateChange{ + {OperationID: 42, AccountID: "GA"}, + {OperationID: 42, AccountID: "GB"}, + {OperationID: 43, AccountID: "GC"}, + }, types.StateChangeSubBaseEffects) + require.Len(t, alone, 3) + for i := range alone { + assert.Equal(t, merged[i].StateChangeID, alone[i].StateChangeID) + } + + // All IDs across streams sharing operation 42 are distinct. + seen := map[int64]bool{} + for _, sc := range merged { + if sc.OperationID != 42 { + continue + } + assert.False(t, seen[sc.StateChangeID], "duplicate state_change_id %d", sc.StateChangeID) + seen[sc.StateChangeID] = true + } +} diff --git a/internal/indexer/mocks.go b/internal/indexer/mocks.go index c1951e78f..30351f9dc 100644 --- a/internal/indexer/mocks.go +++ b/internal/indexer/mocks.go @@ -38,6 +38,13 @@ func (m *MockTokenTransferProcessor) ProcessTransaction(ctx context.Context, tx type MockOperationProcessor struct { mock.Mock + // SubBase is returned by StateChangeSubBase as a plain field so tests can + // place a mock processor in a sub-namespace without mock.On plumbing. + SubBase int64 +} + +func (m *MockOperationProcessor) StateChangeSubBase() int64 { + return m.SubBase } func (m *MockOperationProcessor) ProcessOperation(ctx context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) { diff --git a/internal/indexer/processors/contract_deploy.go b/internal/indexer/processors/contract_deploy.go index 0293901f0..8c1aaa2e2 100644 --- a/internal/indexer/processors/contract_deploy.go +++ b/internal/indexer/processors/contract_deploy.go @@ -28,6 +28,10 @@ func (p *ContractDeployProcessor) Name() string { return "contract_deploy" } +func (p *ContractDeployProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseContractDeploy +} + // ProcessOperation emits a state change for each contract deployment (including subinvocations). func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *TransactionOperationWrapper) ([]types.StateChange, error) { startTime := time.Now() @@ -49,19 +53,24 @@ func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *Transa WithCategory(types.StateChangeCategoryAccount). WithReason(types.StateChangeReasonCreate) - deployedContractsMap := map[string]types.StateChange{} + var stateChanges []types.StateChange + seen := map[string]struct{}{} processCreate := func(fromAddr xdr.ContractIdPreimageFromAddress) error { contractID, err := calculateContractID(p.networkPassphrase, fromAddr) if err != nil { return fmt.Errorf("calculating contract ID: %w", err) } + if _, ok := seen[contractID]; ok { + return nil + } deployerAddr, err := fromAddr.Address.String() if err != nil { return fmt.Errorf("deployer address to string: %w", err) } - deployedContractsMap[contractID] = builder.Clone().WithAccount(contractID).WithDeployer(deployerAddr).Build() + seen[contractID] = struct{}{} + stateChanges = append(stateChanges, builder.Clone().WithAccount(contractID).WithDeployer(deployerAddr).Build()) return nil } @@ -119,9 +128,5 @@ func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *Transa } } - stateChanges := make([]types.StateChange, 0, len(deployedContractsMap)) - for _, sc := range deployedContractsMap { - stateChanges = append(stateChanges, sc) - } return stateChanges, nil } diff --git a/internal/indexer/processors/contract_deploy_test.go b/internal/indexer/processors/contract_deploy_test.go index be42a420e..a97892aa4 100644 --- a/internal/indexer/processors/contract_deploy_test.go +++ b/internal/indexer/processors/contract_deploy_test.go @@ -146,6 +146,93 @@ func Test_ContractDeployProcessor_Process_createContract(t *testing.T) { } } +// Test_ContractDeployProcessor_Process_multipleContractsDeterministicOrder pins the emission +// order for an operation that deploys several contracts across its root host function and its +// auth invocation tree. AssignStateChangeOrdinals (internal/indexer/types/types.go) derives each +// state change's ID from its slice position within the operation, so re-ingesting the same ledger +// must yield the same slice order every time. This test asserts the exact order (not +// ElementsMatch) and also exercises dedup: the auth tree redeclares the same contract creation +// that a sibling subinvocation already produced, and it must not be emitted twice. +func Test_ContractDeployProcessor_Process_multipleContractsDeterministicOrder(t *testing.T) { + const ( + rootDeployer = "GCQIH6MRLCJREVE76LVTKKEZXRIT6KSX7KU65HPDDBYFKFYHIYSJE57R" + subDeployerB = "GDG2KKXC62BINMUZNBTLG235323N6BOIR33JBF4ELTOUKUG5BDE6HJZT" + subDeployerC = "GAGWN4445WLODCXT7RUZXJLQK5XWX4GICXDOAAZZGK2N3BR67RIIVWJ7" + + contractA = "CA7UGIYR2H63C2ETN2VE4WDQ6YX5XNEWNWC2DP7A64B2ZR7VJJWF3SBF" // rootDeployer + TestSalt, root host function + ) + saltB := xdr.Uint256{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} + saltC := xdr.Uint256{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2} + + preimageB := xdr.ContractIdPreimageFromAddress{Address: makeScAddress(subDeployerB), Salt: saltB} + preimageC := xdr.ContractIdPreimageFromAddress{Address: makeScAddress(subDeployerC), Salt: saltC} + + contractB, err := calculateContractID(network.TestNetworkPassphrase, preimageB) + require.NoError(t, err) + contractC, err := calculateContractID(network.TestNetworkPassphrase, preimageC) + require.NoError(t, err) + + ctx := context.Background() + + op := makeBasicSorobanOp() + setFromAddress(op, xdr.HostFunctionTypeHostFunctionTypeCreateContract, rootDeployer) + op.Operation.Body.InvokeHostFunctionOp.Auth = []xdr.SorobanAuthorizationEntry{ + { + RootInvocation: xdr.SorobanAuthorizedInvocation{ + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeContractFn, + }, + SubInvocations: []xdr.SorobanAuthorizedInvocation{ + { + // Deploys contractB, then walks depth-first into its own SubInvocations + // (deploying contractC) before the loop moves on to the next sibling. + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractHostFn, + CreateContractHostFn: &xdr.CreateContractArgs{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageB}}, + }, + SubInvocations: []xdr.SorobanAuthorizedInvocation{ + { + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractV2HostFn, + CreateContractV2HostFn: &xdr.CreateContractArgsV2{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageC}}, + }, + }, + }, + }, + { + // Redeclares the same contractB creation as the sibling above; must be + // deduped and not appear a second time in the output. + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractHostFn, + CreateContractHostFn: &xdr.CreateContractArgs{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageB}}, + }, + }, + }, + }, + }, + } + + proc := NewContractDeployProcessor(network.TestNetworkPassphrase, nil) + stateChanges, err := proc.ProcessOperation(ctx, op) + require.NoError(t, err) + + builder := NewStateChangeBuilder(12345, closeTime.Unix(), 53021371269120, nil). + WithOperationID(53021371269121). + WithReason(types.StateChangeReasonCreate). + WithCategory(types.StateChangeCategoryAccount) + + wantOrder := []types.StateChange{ + builder.Clone().WithDeployer(rootDeployer).WithAccount(contractA).Build(), + builder.Clone().WithDeployer(subDeployerB).WithAccount(contractB).Build(), + builder.Clone().WithDeployer(subDeployerC).WithAccount(contractC).Build(), + } + + require.Len(t, stateChanges, len(wantOrder)) + for i := range wantOrder { + assertStateChangeEqual(t, wantOrder[i], stateChanges[i]) + } +} + func Test_ContractDeployProcessor_Process_invokeContract(t *testing.T) { const ( opSourceAccount = "GBZURSTQQRSU3XB66CHJ3SH2ZWLG663V5SWM6HF3FL72BOMYHDT4QTUF" diff --git a/internal/indexer/processors/contracts/sac.go b/internal/indexer/processors/contracts/sac.go index df7a7e1cb..e0ebd2a82 100644 --- a/internal/indexer/processors/contracts/sac.go +++ b/internal/indexer/processors/contracts/sac.go @@ -49,6 +49,10 @@ func (p *SACEventsProcessor) Name() string { return "sac" } +func (p *SACEventsProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseSACEvents +} + // ProcessOperation processes contract events and converts them into state changes. func (p *SACEventsProcessor) ProcessOperation(_ context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) { startTime := time.Now() diff --git a/internal/indexer/processors/contracts_test_utils.go b/internal/indexer/processors/contracts_test_utils.go index ad62e8058..d89b97465 100644 --- a/internal/indexer/processors/contracts_test_utils.go +++ b/internal/indexer/processors/contracts_test_utils.go @@ -152,7 +152,9 @@ func makeFeeBumpOp(feeBumpSourceAccount string, baseOp *TransactionOperationWrap } // assertStateChangeEqual compares two state changes and fails if they are not equal. -// It normalizes IngestedAt (non-deterministic wall clock) and StateChangeID (random). +// It normalizes IngestedAt (non-deterministic wall clock). StateChangeID is left at its +// Build()-time zero value on both sides — ProcessOperation output is compared before the +// emitter assigns ordinals (see types.AssignStateChangeOrdinals), so no normalization is needed. func assertStateChangeEqual(t *testing.T, want types.StateChange, got types.StateChange) { t.Helper() diff --git a/internal/indexer/processors/effects.go b/internal/indexer/processors/effects.go index b6fb18bed..16297477a 100644 --- a/internal/indexer/processors/effects.go +++ b/internal/indexer/processors/effects.go @@ -40,6 +40,9 @@ var ( "med_threshold": types.StateChangeReasonMedium, "high_threshold": types.StateChangeReasonHigh, } + // thresholdOrder fixes the emission order of threshold state changes so + // ordinals within a (to_id, operation_id) group are deterministic across runs. + thresholdOrder = []string{"low_threshold", "med_threshold", "high_threshold"} // signerEffectToReasonMap maps Stellar signer effect types to our internal state change reasons // for tracking when signers are added, removed, or updated on accounts signerEffectToReasonMap = map[int32]types.StateChangeReason{ @@ -81,6 +84,10 @@ func (p *EffectsProcessor) Name() string { return "effects" } +func (p *EffectsProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseEffects +} + // ProcessOperation extracts effects from a Stellar operation and converts them into state changes. // It processes account state changes like signer modifications, threshold updates, flag changes, // home domain updates, data entry changes, and sponsorship relationship modifications. @@ -719,9 +726,11 @@ func (p *EffectsProcessor) parseThresholds(changeBuilder *StateChangeBuilder, ef // Find the account entry change to get old threshold values prevLedgerEntryState := p.getPrevLedgerEntryState(effect, xdr.LedgerEntryTypeAccount, changes) - // Create a separate state change for each threshold that was modified + // Create a separate state change for each threshold that was modified, in a fixed + // order so ordinal assignment is deterministic when multiple thresholds change together. var thresholdChanges []types.StateChange - for threshold, reason := range thresholdToReasonMap { + for _, threshold := range thresholdOrder { + reason := thresholdToReasonMap[threshold] if value, ok := effect.Details[threshold]; ok { // Convert XDR threshold value to int16 for storage newValue := int16(value.(xdr.Uint32)) diff --git a/internal/indexer/processors/effects_test.go b/internal/indexer/processors/effects_test.go index dbbf770ad..8941e8405 100644 --- a/internal/indexer/processors/effects_test.go +++ b/internal/indexer/processors/effects_test.go @@ -454,3 +454,29 @@ func TestEffects_ProcessTransaction(t *testing.T) { assert.Equal(t, strkey.MustEncode(strkey.VersionByteContract, assetContractID[:]), changes[0].TokenID.String()) }) } + +// TestEffects_ParseThresholds_DeterministicOrder pins the emission order of threshold state +// changes: when a single SetOptions effect updates low, medium, and high thresholds together, +// the resulting state changes must always come back in low -> medium -> high order so that +// ordinals assigned within a (to_id, operation_id) group are reproducible across re-ingests. +func TestEffects_ParseThresholds_DeterministicOrder(t *testing.T) { + processor := NewEffectsProcessor(networkPassphrase, nil) + changeBuilder := NewStateChangeBuilder(12345, 12345*100, toid.New(12345, 1, 1).ToInt64(), nil). + WithAccount("GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH"). + WithCategory(types.StateChangeCategorySignatureThreshold) + effect := &EffectOutput{ + Address: "GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH", + Details: map[string]interface{}{ + "low_threshold": xdr.Uint32(1), + "med_threshold": xdr.Uint32(2), + "high_threshold": xdr.Uint32(3), + }, + } + + thresholdChanges := processor.parseThresholds(changeBuilder, effect, nil) + + require.Len(t, thresholdChanges, 3) + assert.Equal(t, types.StateChangeReasonLow, thresholdChanges[0].StateChangeReason) + assert.Equal(t, types.StateChangeReasonMedium, thresholdChanges[1].StateChangeReason) + assert.Equal(t, types.StateChangeReasonHigh, thresholdChanges[2].StateChangeReason) +} diff --git a/internal/indexer/processors/state_change_builder.go b/internal/indexer/processors/state_change_builder.go index 660ed022e..7c7cbc0db 100644 --- a/internal/indexer/processors/state_change_builder.go +++ b/internal/indexer/processors/state_change_builder.go @@ -178,8 +178,9 @@ func (b *StateChangeBuilder) WithSponsoredData(dataName string) *StateChangeBuil } // Build returns the constructed state change. -// Note: StateChangeID is assigned at insertion time by BatchCopy, not here. -// This ensures retries on PK collision generate fresh IDs automatically. +// Note: StateChangeID is left zero here — the emitter assigns it afterward via +// types.AssignStateChangeOrdinals, numbering each transaction's or window's +// state changes in deterministic emission order before persisting. func (b *StateChangeBuilder) Build() types.StateChange { return b.base } diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index be64f677a..bb218c409 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -687,6 +687,101 @@ type StateChangeCursorGetter interface { GetCursor() StateChangeCursor } +// State change ID namespace registry. +// +// state_changes rows are written by more than one independent emitter: the +// main indexer (effects, token transfers, etc.) and protocol processors +// (SEP-41, Blend) that persist their own history inside a separately +// CAS-guarded transaction. Two emitters can legitimately target the same +// (to_id, operation_id) — e.g. a SEP-41 or Blend contract invocation is one +// operation the main indexer also sees. Each emitter numbers its own state +// changes 1..N in deterministic emission order within an (to_id, operation_id) +// group (see AssignStateChangeOrdinals) and adds its base below, so the +// resulting state_change_id values can never collide across emitters even +// when they share an operation. Bases are consecutive multiples of +// StateChangeOrdinalNamespaceWidth and must never change once rows exist with +// them. +// +// Why the split at bit 40: an int64 has 63 usable positive bits. Reserving the +// low 40 bits per namespace carves the space into 2^23 (~8.4M) emitter +// namespaces, each holding 2^40 (~1.1e12) ordinals per (to_id, operation_id) +// group. Transaction meta size limits cap a single operation's emissions at a +// few thousand, and new emitters are added only at code-review speed, so both +// sides of the split sit orders of magnitude beyond their physical bounds — +// neither dimension can become the binding constraint, and frozen bases never +// face renumbering pressure. +// +// The indexer subdivides its namespace further, one sub-namespace per +// emitting processor — see the sub-base registry below. +const ( + // StateChangeOrdinalNamespaceWidth is the span of state_change_id values + // reserved for each emitter namespace; the bases below are consecutive + // multiples of it. Changing it renumbers every base and is forbidden once + // rows exist. + StateChangeOrdinalNamespaceWidth int64 = 1 << 40 + + StateChangeOrdinalBaseIndexer int64 = 0 + StateChangeOrdinalBaseSEP41 int64 = 1 * StateChangeOrdinalNamespaceWidth + StateChangeOrdinalBaseBlend int64 = 2 * StateChangeOrdinalNamespaceWidth +) + +// Sub-namespace registry within the indexer's emitter namespace. +// +// The indexer is itself multi-stream: several processors (token transfers, +// effects, contract deploys, SAC events) can emit state changes for the same +// operation. Each processor numbers its own emissions independently (see +// Indexer.getTransactionStateChanges) and adds StateChangeOrdinalBaseIndexer +// plus its sub-base below, so IDs never depend on the order processors are +// registered or invoked in — adding or reordering processors cannot shift +// another processor's IDs. Within one processor, emission order derives from +// the transaction meta (canonical on-chain data), which is what makes the +// scheme reproducible across runs. +// +// Sub-bases split the indexer's 1<<40-wide namespace at bit 28: 2^12 = 4096 +// sub-streams fit inside it, each with 2^28 (~268M) ordinals per +// (to_id, operation_id) group. As with the bit-40 emitter split, both sides +// sit orders of magnitude beyond their physical bounds — transaction meta size +// limits cap per-operation emissions at a few thousand, and new sub-streams +// are added only at code-review speed — so neither dimension can become the +// binding constraint. Protocol emitters (SEP-41, Blend) are single-stream and use their +// base directly; a future multi-stream emitter subdivides its own namespace +// the same way. Like the emitter bases, sub-bases must never change once +// rows exist with them; new processors take the next unused slot. +const ( + // StateChangeSubNamespaceWidth is the span of state_change_id values + // reserved for each indexer sub-stream; the sub-bases below are consecutive + // multiples of it. Changing it renumbers every sub-base and is forbidden + // once rows exist. + StateChangeSubNamespaceWidth int64 = 1 << 28 + + StateChangeSubBaseTokenTransfer int64 = 0 * StateChangeSubNamespaceWidth + StateChangeSubBaseEffects int64 = 1 * StateChangeSubNamespaceWidth + StateChangeSubBaseContractDeploy int64 = 2 * StateChangeSubNamespaceWidth + StateChangeSubBaseSACEvents int64 = 3 * StateChangeSubNamespaceWidth +) + +// AssignStateChangeOrdinals assigns each state change in changes a +// deterministic state_change_id: an ordinal numbered 1..N in slice order +// within each distinct (to_id, operation_id) group, plus base. base is the calling +// emitter's namespace base — plus its processor's sub-base for multi-stream +// emitters like the indexer. Processing the same input twice (e.g. a +// re-ingested ledger) yields byte-identical IDs, so a duplicate BatchCopy +// fails loudly on the state_changes primary key instead of inserting +// duplicate rows. +// +// Callers must pass one emission stream at a time — the final, filtered slice +// actually handed to BatchCopy — so ordinals come out contiguous (1..N, no +// gaps) per group within the stream's namespace. +func AssignStateChangeOrdinals(changes []StateChange, base int64) { + type ordinalKey struct{ toID, opID int64 } + next := make(map[ordinalKey]int64, len(changes)) + for i := range changes { + k := ordinalKey{changes[i].ToID, changes[i].OperationID} + next[k]++ + changes[i].StateChangeID = base + next[k] + } +} + type NullableJSONB map[string]any var _ sql.Scanner = (*NullableJSONB)(nil) diff --git a/internal/indexer/types/types_test.go b/internal/indexer/types/types_test.go index c8422f890..a0662f4ec 100644 --- a/internal/indexer/types/types_test.go +++ b/internal/indexer/types/types_test.go @@ -379,3 +379,98 @@ func TestHashBytea_String(t *testing.T) { }) } } + +// buildOrdinalTestInput returns a fixed slice of state changes spanning two +// operations, interleaved in emission order: op 10 gets three, op 20 gets two. +func buildOrdinalTestInput() []StateChange { + return []StateChange{ + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 20, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 20, StateChangeCategory: StateChangeCategoryBalance}, + } +} + +func TestAssignStateChangeOrdinals_OrdinalsPerOperationGroup(t *testing.T) { + changes := buildOrdinalTestInput() + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseIndexer) + + // op 10's three changes are numbered 1..3 in the order they appear; op 20's + // two changes are numbered 1..2 independently, interleaved with op 10's. + assert.Equal(t, []int64{1, 1, 2, 3, 2}, []int64{ + changes[0].StateChangeID, changes[1].StateChangeID, changes[2].StateChangeID, + changes[3].StateChangeID, changes[4].StateChangeID, + }) +} + +func TestAssignStateChangeOrdinals_NamespaceBaseIsAdded(t *testing.T) { + changes := buildOrdinalTestInput() + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseBlend) + + for _, sc := range changes { + assert.GreaterOrEqual(t, sc.StateChangeID, StateChangeOrdinalBaseBlend) + assert.Less(t, sc.StateChangeID, StateChangeOrdinalBaseBlend+1000) + } + // Same relative ordinals as the unnamespaced case, offset by the base. + assert.Equal(t, StateChangeOrdinalBaseBlend+1, changes[0].StateChangeID) + assert.Equal(t, StateChangeOrdinalBaseBlend+3, changes[3].StateChangeID) +} + +func TestAssignStateChangeOrdinals_Deterministic(t *testing.T) { + first := buildOrdinalTestInput() + second := buildOrdinalTestInput() + + AssignStateChangeOrdinals(first, StateChangeOrdinalBaseSEP41) + AssignStateChangeOrdinals(second, StateChangeOrdinalBaseSEP41) + + // Processing the same input twice (e.g. a re-ingested ledger) must produce + // byte-identical state_change_ids, not just equal-length results. + require.Len(t, second, len(first)) + for i := range first { + assert.Equal(t, first[i].StateChangeID, second[i].StateChangeID, "index %d", i) + } +} + +func TestAssignStateChangeOrdinals_DifferentEmittersNeverCollide(t *testing.T) { + indexerChanges := []StateChange{{ToID: 1, OperationID: 42}} + sep41Changes := []StateChange{{ToID: 1, OperationID: 42}} + blendChanges := []StateChange{{ToID: 1, OperationID: 42}} + + AssignStateChangeOrdinals(indexerChanges, StateChangeOrdinalBaseIndexer) + AssignStateChangeOrdinals(sep41Changes, StateChangeOrdinalBaseSEP41) + AssignStateChangeOrdinals(blendChanges, StateChangeOrdinalBaseBlend) + + // Same (to_id, operation_id) from three independent emitters: without + // namespacing these would all be state_change_id=1 and collide on the + // state_changes primary key. + ids := map[int64]bool{ + indexerChanges[0].StateChangeID: true, + sep41Changes[0].StateChangeID: true, + blendChanges[0].StateChangeID: true, + } + assert.Len(t, ids, 3, "each emitter's ID for the same operation must be distinct") +} + +func TestAssignStateChangeOrdinals_GroupsByToIDAndOperationID(t *testing.T) { + // A window-scoped emitter can stage transaction-level changes with + // OperationID=0 for several transactions (distinct ToIDs) in one call. + // Ordinals restart per (to_id, operation_id) pair, not per operation_id + // alone: otherwise a row's ID would depend on which other transactions + // happen to share the window, and re-running the same ledger under + // different window bounds would produce different IDs for the same logical + // row — silently bypassing the state_changes primary key on re-ingest. + changes := []StateChange{ + {ToID: 100, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 100, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 200, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 200, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + } + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseIndexer) + + // Keying on operation_id alone would yield 1,2,3,4 (one shared bucket). + assert.Equal(t, []int64{1, 2, 1, 2}, []int64{ + changes[0].StateChangeID, changes[1].StateChangeID, + changes[2].StateChangeID, changes[3].StateChangeID, + }) +} diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 4fe7cc9f9..a6515e3c0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -134,6 +134,10 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("connecting to the database: %w", err) } + // Retention and compression policies live in shared database state, so only + // the live deployment configures them. A backfill runs with ad-hoc flags and + // must not mutate or remove the policies the live pods rely on; and an active + // retention policy would drop the very history a backfill is writing. if cfg.IngestionMode == services.IngestionModeLive { if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { return nil, fmt.Errorf("configuring hypertable settings: %w", err) diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go index 3d3a4d9a2..231a30a1c 100644 --- a/internal/ingest/timescaledb.go +++ b/internal/ingest/timescaledb.go @@ -4,8 +4,10 @@ package ingest import ( "context" + "errors" "fmt" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/go-stellar-sdk/support/log" ) @@ -21,13 +23,15 @@ var hypertables = []string{ // configureHypertableSettings applies chunk interval, retention policy, and // compression schedule settings to all hypertables. Chunk interval only affects -// future chunks. Retention policy is idempotent: any existing policy is removed -// before re-adding. When retention is enabled, a reconciliation job keeps -// oldest_ingest_ledger in sync with the actual minimum ledger remaining after -// chunk drops. Compression schedule interval updates how frequently existing -// compression policy jobs run (does not create new policies). -// Compress after updates how long after a chunk closes before it becomes -// eligible for compression. +// future chunks. Retention policy and the reconciliation job are converged +// in place: an existing job is altered when its config differs from the +// desired value and left untouched when it already matches, so job_id and +// run history survive repeated calls (e.g. every process restart). When +// retention is enabled, the reconciliation job keeps oldest_ingest_ledger in +// sync with the actual minimum ledger remaining after chunk drops. Compression +// schedule interval updates how frequently existing compression policy jobs +// run (does not create new policies). Compress after updates how long after a +// chunk closes before it becomes eligible for compression. func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkInterval, retentionPeriod, oldestCursorName, compressionScheduleInterval, compressAfter string, maxChunksToCompress int) error { for _, table := range hypertables { if _, err := pool.Exec(ctx, @@ -39,73 +43,14 @@ func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkI log.Ctx(ctx).Infof("Set chunk interval %q on %s", chunkInterval, table) } - // We first remove existing retention policy for _, table := range hypertables { - if _, err := pool.Exec(ctx, - "SELECT remove_retention_policy($1::regclass, if_exists => true)", - table, - ); err != nil { - return fmt.Errorf("removing retention policy on %s: %w", table, err) + if err := configureRetentionPolicy(ctx, pool, table, retentionPeriod); err != nil { + return fmt.Errorf("configuring retention policy on %s: %w", table, err) } } - // Reconciliation job: keeps oldestCursorName in sync after retention drops chunks. - // Remove any existing job first (idempotent re-registration on every restart). - if _, err := pool.Exec(ctx, - "SELECT delete_job(job_id) FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", - ); err != nil { - return fmt.Errorf("removing existing reconciliation job: %w", err) - } - if retentionPeriod != "" { - // Add new retention period policy - for _, table := range hypertables { - if _, err := pool.Exec(ctx, - "SELECT add_retention_policy($1::regclass, drop_after => $2::interval)", - table, retentionPeriod, - ); err != nil { - return fmt.Errorf("adding retention policy on %s: %w", table, err) - } - log.Ctx(ctx).Infof("Set retention policy %q on %s", retentionPeriod, table) - } - - // Create or replace the PL/pgSQL function that advances the cursor. - if _, err := pool.Exec(ctx, ` - CREATE OR REPLACE FUNCTION reconcile_oldest_cursor(job_id INT, config JSONB) - RETURNS VOID LANGUAGE plpgsql AS $$ - DECLARE - actual_min INTEGER; - stored INTEGER; - BEGIN - SELECT ledger_number INTO actual_min FROM transactions - ORDER BY ledger_created_at ASC, to_id ASC LIMIT 1; - IF actual_min IS NULL THEN RETURN; END IF; - SELECT value::integer INTO stored FROM ingest_store WHERE key = config->>'cursor_name'; - IF stored IS NULL OR actual_min <= stored THEN RETURN; END IF; - UPDATE ingest_store SET value = actual_min::text WHERE key = config->>'cursor_name'; - RAISE LOG 'reconcile_oldest_cursor: advanced % from % to %', config->>'cursor_name', stored, actual_min; - END $$; - `); err != nil { - return fmt.Errorf("creating reconcile_oldest_cursor function: %w", err) - } - // Schedule the reconciliation job to run every 1 hour. - // - // The job checks whether retention has dropped chunks and advances the - // oldest ledger cursor if so. It is idempotent — a no-op when the cursor - // is already correct — and the query is microsecond-cheap (reads oldest - // chunk metadata + 1 row from ingest_store). Running on a fixed 1-hour - // interval keeps the cursor at most 1 hour stale after retention fires, - // with no coordination required with the retention job schedule. - if _, err := pool.Exec(ctx, ` - SELECT add_job( - 'reconcile_oldest_cursor', - '1 hour', - fixed_schedule => true, - config => $1::jsonb)`, - fmt.Sprintf(`{"cursor_name":"%s"}`, oldestCursorName), - ); err != nil { - return fmt.Errorf("scheduling reconciliation job: %w", err) - } - log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job (1h fixed interval) for cursor %q", oldestCursorName) + if err := configureReconciliationJob(ctx, pool, retentionPeriod, oldestCursorName); err != nil { + return fmt.Errorf("configuring reconciliation job: %w", err) } if compressionScheduleInterval != "" { @@ -185,3 +130,146 @@ func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkI return nil } + +// configureRetentionPolicy converges table's retention policy on the desired +// drop_after. An empty retentionPeriod disables retention, removing any +// existing policy. Otherwise, a missing policy is added; an existing one +// whose drop_after differs is altered in place (job_id and run history +// survive); one that already matches is left untouched. +func configureRetentionPolicy(ctx context.Context, pool *pgxpool.Pool, table, retentionPeriod string) error { + if retentionPeriod == "" { + if _, err := pool.Exec(ctx, + "SELECT remove_retention_policy($1::regclass, if_exists => true)", + table, + ); err != nil { + return fmt.Errorf("removing retention policy: %w", err) + } + return nil + } + + var jobID int + err := pool.QueryRow(ctx, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ).Scan(&jobID) + switch { + case errors.Is(err, pgx.ErrNoRows): + if _, err = pool.Exec(ctx, + "SELECT add_retention_policy($1::regclass, drop_after => $2::interval)", + table, retentionPeriod, + ); err != nil { + return fmt.Errorf("adding retention policy: %w", err) + } + log.Ctx(ctx).Infof("Added retention policy %q on %s", retentionPeriod, table) + return nil + case err != nil: + return fmt.Errorf("looking up existing retention policy: %w", err) + } + + var matches bool + if err := pool.QueryRow(ctx, + `SELECT (config->>'drop_after')::interval = $2::interval + FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, retentionPeriod, + ).Scan(&matches); err != nil { + return fmt.Errorf("comparing existing retention policy (job %d): %w", jobID, err) + } + if matches { + return nil + } + + if _, err := pool.Exec(ctx, + `SELECT alter_job($1, config => jsonb_set( + (SELECT config FROM timescaledb_information.jobs WHERE job_id = $1), + '{drop_after}', to_jsonb($2::text)))`, + jobID, retentionPeriod, + ); err != nil { + return fmt.Errorf("altering retention policy (job %d): %w", jobID, err) + } + log.Ctx(ctx).Infof("Updated retention policy to %q on %s (job %d)", retentionPeriod, table, jobID) + return nil +} + +// configureReconciliationJob converges the reconcile_oldest_cursor job, which +// keeps oldestCursorName in sync with the actual minimum ledger remaining +// after retention drops chunks. An empty retentionPeriod removes the job +// entirely — it has nothing to reconcile when retention is disabled. +// Otherwise, a missing job is added; an existing one whose cursor_name +// differs is altered in place (job_id and run history survive); one that +// already matches is left untouched. +func configureReconciliationJob(ctx context.Context, pool *pgxpool.Pool, retentionPeriod, oldestCursorName string) error { + if retentionPeriod == "" { + if _, err := pool.Exec(ctx, + "SELECT delete_job(job_id) FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", + ); err != nil { + return fmt.Errorf("removing reconciliation job: %w", err) + } + return nil + } + + // Create or replace the PL/pgSQL function that advances the cursor. + if _, err := pool.Exec(ctx, ` + CREATE OR REPLACE FUNCTION reconcile_oldest_cursor(job_id INT, config JSONB) + RETURNS VOID LANGUAGE plpgsql AS $$ + DECLARE + actual_min INTEGER; + stored INTEGER; + BEGIN + SELECT ledger_number INTO actual_min FROM transactions + ORDER BY ledger_created_at ASC, to_id ASC LIMIT 1; + IF actual_min IS NULL THEN RETURN; END IF; + SELECT value::integer INTO stored FROM ingest_store WHERE key = config->>'cursor_name'; + IF stored IS NULL OR actual_min <= stored THEN RETURN; END IF; + UPDATE ingest_store SET value = actual_min::text WHERE key = config->>'cursor_name'; + RAISE LOG 'reconcile_oldest_cursor: advanced % from % to %', config->>'cursor_name', stored, actual_min; + END $$; + `); err != nil { + return fmt.Errorf("creating reconcile_oldest_cursor function: %w", err) + } + + var jobID int + err := pool.QueryRow(ctx, + "SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", + ).Scan(&jobID) + switch { + case errors.Is(err, pgx.ErrNoRows): + // Runs every 1 hour: cheap enough (oldest chunk metadata + 1 row from + // ingest_store) to not need coordination with the retention job's own + // schedule, and idempotent — a no-op once the cursor is already correct. + if _, err = pool.Exec(ctx, ` + SELECT add_job( + 'reconcile_oldest_cursor', + '1 hour', + fixed_schedule => true, + config => $1::jsonb)`, + fmt.Sprintf(`{"cursor_name":"%s"}`, oldestCursorName), + ); err != nil { + return fmt.Errorf("scheduling reconciliation job: %w", err) + } + log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job (1h fixed interval) for cursor %q", oldestCursorName) + return nil + case err != nil: + return fmt.Errorf("looking up existing reconciliation job: %w", err) + } + + var matches bool + if err := pool.QueryRow(ctx, + `SELECT config->>'cursor_name' = $2 FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, oldestCursorName, + ).Scan(&matches); err != nil { + return fmt.Errorf("comparing existing reconciliation job (job %d): %w", jobID, err) + } + if matches { + return nil + } + + if _, err := pool.Exec(ctx, + "SELECT alter_job($1, config => jsonb_build_object('cursor_name', $2::text))", + jobID, oldestCursorName, + ); err != nil { + return fmt.Errorf("altering reconciliation job (job %d): %w", jobID, err) + } + log.Ctx(ctx).Infof("Updated reconcile_oldest_cursor cursor to %q (job %d)", oldestCursorName, jobID) + return nil +} diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go index f582b7cdc..0cdd61f13 100644 --- a/internal/ingest/timescaledb_test.go +++ b/internal/ingest/timescaledb_test.go @@ -114,6 +114,89 @@ func TestConfigureHypertableSettings(t *testing.T) { } }) + t.Run("retention_policy_job_id_stable_across_calls", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDsBefore := make(map[string]int) + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + jobIDsBefore[table] = jobID + } + + // Re-applying the same retention period must not delete/recreate the job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + assert.Equal(t, jobIDsBefore[table], jobID, "retention job_id should be stable across identical re-application for %s", table) + } + }) + + t.Run("retention_policy_altered_not_recreated", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDsBefore := make(map[string]int) + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + jobIDsBefore[table] = jobID + } + + // Changing the retention period must alter the existing job in place, + // not delete and recreate it. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + assert.Equal(t, jobIDsBefore[table], jobID, "retention job_id should be stable when the period changes for %s", table) + + dropAfterMatches, err := db.QueryOne[bool](ctx, dbConnectionPool, + `SELECT (config->>'drop_after')::interval = '90 days'::interval + FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, + ) + require.NoError(t, err, "querying drop_after for %s", table) + assert.True(t, dropAfterMatches, "drop_after should be updated to 90 days for %s", table) + } + }) + t.Run("reconciliation_job_created", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -160,6 +243,69 @@ func TestConfigureHypertableSettings(t *testing.T) { assert.Equal(t, 1, count, "expected exactly 1 reconciliation job after re-application") }) + t.Run("reconciliation_job_id_stable_across_calls", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDBefore, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + + // Re-applying with a different retention period but the same cursor name + // must not delete/recreate the reconciliation job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDAfter, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, jobIDBefore, jobIDAfter, "reconciliation job_id should be stable across re-application") + }) + + t.Run("reconciliation_job_altered_not_recreated_when_cursor_changes", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDBefore, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + + // A changed cursor name must alter the existing job's config in place, + // not delete and recreate the job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "renamed_cursor", "", "", 0) + require.NoError(t, err) + + jobIDAfter, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, jobIDBefore, jobIDAfter, "reconciliation job_id should be stable when the cursor name changes") + + cursorName, err := db.QueryOne[string](ctx, dbConnectionPool, + `SELECT config->>'cursor_name' FROM timescaledb_information.jobs WHERE job_id = $1`, + jobIDAfter, + ) + require.NoError(t, err) + assert.Equal(t, "renamed_cursor", cursorName, "cursor_name should be updated in the reconciliation job's config") + }) + t.Run("no_reconciliation_job_without_retention", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() diff --git a/internal/loadtest/runner.go b/internal/loadtest/runner.go index 09a269a5c..7455edfe1 100644 --- a/internal/loadtest/runner.go +++ b/internal/loadtest/runner.go @@ -88,7 +88,10 @@ func Run(ctx context.Context, cfg RunConfig) error { metrics.RegisterPoolMetrics(m.Registry(), "loadtest_indexer", indexerPool) // Loadtest doesn't exercise classification; the indexer just buffers raw // WASM bytecode and the dispatcher is bypassed (no validators wired). - ledgerIndexer := indexer.NewIndexer(cfg.NetworkPassphrase, indexerPool, m.Ingestion) + ledgerIndexer, err := indexer.NewIndexer(cfg.NetworkPassphrase, indexerPool, m.Ingestion) + if err != nil { + return fmt.Errorf("creating ledger indexer: %w", err) + } // Create TokenIngestionService for token change processing tokenIngestionService := services.NewTokenIngestionService(services.TokenIngestionServiceConfig{ diff --git a/internal/serve/graphql/dataloaders/operation_loaders.go b/internal/serve/graphql/dataloaders/operation_loaders.go index a99826461..3b0a51ea6 100644 --- a/internal/serve/graphql/dataloaders/operation_loaders.go +++ b/internal/serve/graphql/dataloaders/operation_loaders.go @@ -48,15 +48,17 @@ func operationsByToIDLoader(models *data.Models) *dataloadgen.Loader[OperationCo // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.Operations.BatchGetByToID(ctx, keys[0].ToID, columns, limit, keys[0].Cursor, sortOrder) + return models.Operations.BatchGetByToID(ctx, keys[0].ToID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } toIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) maxLimit := min(*limit, MaxOperationsPerBatch) for i, key := range keys { toIDs[i] = key.ToID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.Operations.BatchGetByToIDs(ctx, toIDs, columns, &maxLimit, sortOrder) + return models.Operations.BatchGetByToIDs(ctx, toIDs, ledgerCreatedAts, columns, &maxLimit, sortOrder) }, func(item *types.OperationWithCursor) int64 { // Derive tx_to_id from operation ID using TOID bit masking @@ -96,14 +98,16 @@ func operationByStateChangeIDLoader(models *data.Models) *dataloadgen.Loader[Ope func(ctx context.Context, keys []OperationColumnsKey) ([]*types.OperationWithStateChangeID, error) { columns := keys[0].Columns scIDs := make([]string, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { scIDs[i] = key.StateChangeID + ledgerCreatedAts[i] = key.LedgerCreatedAt } scToIDs, scOpIDs, scOrders, err := parseStateChangeIDs(scIDs) if err != nil { return nil, fmt.Errorf("parsing state change IDs: %w", err) } - return models.Operations.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, columns) + return models.Operations.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, ledgerCreatedAts, columns) }, func(item *types.OperationWithStateChangeID) string { return item.StateChangeID diff --git a/internal/serve/graphql/dataloaders/statechange_loaders.go b/internal/serve/graphql/dataloaders/statechange_loaders.go index ba26010fb..871881864 100644 --- a/internal/serve/graphql/dataloaders/statechange_loaders.go +++ b/internal/serve/graphql/dataloaders/statechange_loaders.go @@ -45,15 +45,17 @@ func stateChangesByToIDLoader(models *data.Models) *dataloadgen.Loader[StateChan // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.StateChanges.BatchGetByToID(ctx, keys[0].ToID, columns, limit, keys[0].Cursor, sortOrder) + return models.StateChanges.BatchGetByToID(ctx, keys[0].ToID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } toIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) maxLimit := min(*limit, MaxStateChangesPerBatch) for i, key := range keys { toIDs[i] = key.ToID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.StateChanges.BatchGetByToIDs(ctx, toIDs, columns, &maxLimit, sortOrder) + return models.StateChanges.BatchGetByToIDs(ctx, toIDs, ledgerCreatedAts, columns, &maxLimit, sortOrder) }, func(item *types.StateChangeWithCursor) int64 { return item.StateChange.ToID @@ -85,14 +87,16 @@ func stateChangesByOperationIDLoader(models *data.Models) *dataloadgen.Loader[St // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.StateChanges.BatchGetByOperationID(ctx, keys[0].OperationID, columns, limit, keys[0].Cursor, sortOrder) + return models.StateChanges.BatchGetByOperationID(ctx, keys[0].OperationID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } operationIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { operationIDs[i] = key.OperationID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.StateChanges.BatchGetByOperationIDs(ctx, operationIDs, columns, limit, sortOrder) + return models.StateChanges.BatchGetByOperationIDs(ctx, operationIDs, ledgerCreatedAts, columns, limit, sortOrder) }, func(item *types.StateChangeWithCursor) int64 { return item.StateChange.OperationID diff --git a/internal/serve/graphql/dataloaders/transaction_loaders.go b/internal/serve/graphql/dataloaders/transaction_loaders.go index 36249761d..4c50a370a 100644 --- a/internal/serve/graphql/dataloaders/transaction_loaders.go +++ b/internal/serve/graphql/dataloaders/transaction_loaders.go @@ -3,6 +3,7 @@ package dataloaders import ( "context" "fmt" + "time" "github.com/vikstrous/dataloadgen" @@ -11,10 +12,11 @@ import ( ) type TransactionColumnsKey struct { - AccountID string - OperationID int64 - StateChangeID string - Columns string + AccountID string + OperationID int64 + StateChangeID string + Columns string + LedgerCreatedAt time.Time // parent state change's ledger time; pins the partition column for BatchGetByStateChangeIDs } // txByOperationIDLoader creates a dataloader for fetching transactions by operation ID @@ -50,14 +52,16 @@ func transactionByStateChangeIDLoader(models *data.Models) *dataloadgen.Loader[T func(ctx context.Context, keys []TransactionColumnsKey) ([]*types.TransactionWithStateChangeID, error) { columns := keys[0].Columns scIDs := make([]string, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { scIDs[i] = key.StateChangeID + ledgerCreatedAts[i] = key.LedgerCreatedAt } scToIDs, scOpIDs, scOrders, err := parseStateChangeIDs(scIDs) if err != nil { return nil, fmt.Errorf("parsing state change IDs: %w", err) } - return models.Transactions.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, columns) + return models.Transactions.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, ledgerCreatedAts, columns) }, func(item *types.TransactionWithStateChangeID) string { return item.StateChangeID diff --git a/internal/serve/graphql/resolvers/operation.resolvers.go b/internal/serve/graphql/resolvers/operation.resolvers.go index 1debac4a6..191dbecae 100644 --- a/internal/serve/graphql/resolvers/operation.resolvers.go +++ b/internal/serve/graphql/resolvers/operation.resolvers.go @@ -82,11 +82,12 @@ func (r *operationResolver) StateChanges(ctx context.Context, obj *types.Operati loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.StateChangeColumnsKey{ - OperationID: obj.ID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.StateChangeCursor, - SortOrder: params.SortOrder, + OperationID: obj.ID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.StateChangeCursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } stateChanges, err := loaders.StateChangesByOperationIDLoader.Load(ctx, loaderKey) diff --git a/internal/serve/graphql/resolvers/operation_resolvers_test.go b/internal/serve/graphql/resolvers/operation_resolvers_test.go index 9b6fd8c41..a42ec875a 100644 --- a/internal/serve/graphql/resolvers/operation_resolvers_test.go +++ b/internal/serve/graphql/resolvers/operation_resolvers_test.go @@ -28,7 +28,7 @@ func TestOperationResolver_Transaction(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -74,7 +74,7 @@ func TestOperationResolver_Accounts(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -124,7 +124,7 @@ func TestOperationResolver_StateChanges(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} nonExistentOperation := &types.Operation{ID: 9999} t.Run("success without pagination", func(t *testing.T) { diff --git a/internal/serve/graphql/resolvers/resolver.go b/internal/serve/graphql/resolvers/resolver.go index 7ea29efa2..d3a79c634 100644 --- a/internal/serve/graphql/resolvers/resolver.go +++ b/internal/serve/graphql/resolvers/resolver.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/google/uuid" @@ -128,14 +129,15 @@ func (r *Resolver) resolveStateChangeAccount(accountID types.AddressBytea) (*typ // resolveStateChangeOperation resolves the operation field for any state change type // Reuses the existing logic from the original StateChange resolver -func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, operationID int64, stateChangeID int64) (*types.Operation, error) { +func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, operationID int64, stateChangeID int64, ledgerCreatedAt time.Time) (*types.Operation, error) { loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) stateChangeKey := fmt.Sprintf("%d-%d-%d", toID, operationID, stateChangeID) loaderKey := dataloaders.OperationColumnsKey{ - StateChangeID: stateChangeKey, - Columns: strings.Join(dbColumns, ", "), + StateChangeID: stateChangeKey, + Columns: strings.Join(dbColumns, ", "), + LedgerCreatedAt: ledgerCreatedAt, } operations, err := loaders.OperationByStateChangeIDLoader.Load(ctx, loaderKey) if err != nil { @@ -146,14 +148,15 @@ func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, // resolveStateChangeTransaction resolves the transaction field for any state change type // Reuses the existing logic from the original StateChange resolver -func (r *Resolver) resolveStateChangeTransaction(ctx context.Context, toID int64, operationID int64, stateChangeID int64) (*types.Transaction, error) { +func (r *Resolver) resolveStateChangeTransaction(ctx context.Context, toID int64, operationID int64, stateChangeID int64, ledgerCreatedAt time.Time) (*types.Transaction, error) { loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) stateChangeKey := fmt.Sprintf("%d-%d-%d", toID, operationID, stateChangeID) loaderKey := dataloaders.TransactionColumnsKey{ - StateChangeID: stateChangeKey, - Columns: strings.Join(dbColumns, ", "), + StateChangeID: stateChangeKey, + Columns: strings.Join(dbColumns, ", "), + LedgerCreatedAt: ledgerCreatedAt, } transaction, err := loaders.TransactionByStateChangeIDLoader.Load(ctx, loaderKey) if err != nil { diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index 6c7562368..d4e8bc153 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -30,12 +30,12 @@ func (r *accountChangeResolver) Account(ctx context.Context, obj *types.AccountS // Operation is the resolver for the operation field. func (r *accountChangeResolver) Operation(ctx context.Context, obj *types.AccountStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *accountChangeResolver) Transaction(ctx context.Context, obj *types.AccountStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // FunderAddress is the resolver for the funderAddress field. @@ -70,12 +70,12 @@ func (r *balanceAuthorizationChangeResolver) Account(ctx context.Context, obj *t // Operation is the resolver for the operation field. func (r *balanceAuthorizationChangeResolver) Operation(ctx context.Context, obj *types.BalanceAuthorizationStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *balanceAuthorizationChangeResolver) Transaction(ctx context.Context, obj *types.BalanceAuthorizationStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. @@ -114,12 +114,12 @@ func (r *flagsChangeResolver) Account(ctx context.Context, obj *types.FlagsState // Operation is the resolver for the operation field. func (r *flagsChangeResolver) Operation(ctx context.Context, obj *types.FlagsStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *flagsChangeResolver) Transaction(ctx context.Context, obj *types.FlagsStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Flags is the resolver for the flags field. @@ -148,12 +148,12 @@ func (r *metadataChangeResolver) Account(ctx context.Context, obj *types.Metadat // Operation is the resolver for the operation field. func (r *metadataChangeResolver) Operation(ctx context.Context, obj *types.MetadataStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *metadataChangeResolver) Transaction(ctx context.Context, obj *types.MetadataStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // KeyValue is the resolver for the keyValue field. @@ -178,12 +178,12 @@ func (r *reservesChangeResolver) Account(ctx context.Context, obj *types.Reserve // Operation is the resolver for the operation field. func (r *reservesChangeResolver) Operation(ctx context.Context, obj *types.ReservesStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *reservesChangeResolver) Transaction(ctx context.Context, obj *types.ReservesStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // SponsoredAddress is the resolver for the sponsoredAddress field. @@ -233,12 +233,12 @@ func (r *signerChangeResolver) Account(ctx context.Context, obj *types.SignerSta // Operation is the resolver for the operation field. func (r *signerChangeResolver) Operation(ctx context.Context, obj *types.SignerStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *signerChangeResolver) Transaction(ctx context.Context, obj *types.SignerStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // SignerAddress is the resolver for the signerAddress field. @@ -286,12 +286,12 @@ func (r *signerThresholdsChangeResolver) Account(ctx context.Context, obj *types // Operation is the resolver for the operation field. func (r *signerThresholdsChangeResolver) Operation(ctx context.Context, obj *types.SignerThresholdsStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *signerThresholdsChangeResolver) Transaction(ctx context.Context, obj *types.SignerThresholdsStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Thresholds is the resolver for the thresholds field. @@ -331,12 +331,12 @@ func (r *standardBalanceChangeResolver) Account(ctx context.Context, obj *types. // Operation is the resolver for the operation field. func (r *standardBalanceChangeResolver) Operation(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *standardBalanceChangeResolver) Transaction(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. @@ -375,12 +375,12 @@ func (r *trustlineChangeResolver) Account(ctx context.Context, obj *types.Trustl // Operation is the resolver for the operation field. func (r *trustlineChangeResolver) Operation(ctx context.Context, obj *types.TrustlineStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *trustlineChangeResolver) Transaction(ctx context.Context, obj *types.TrustlineStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. diff --git a/internal/serve/graphql/resolvers/statechange_resolvers_test.go b/internal/serve/graphql/resolvers/statechange_resolvers_test.go index a9304bc16..78cab3617 100644 --- a/internal/serve/graphql/resolvers/statechange_resolvers_test.go +++ b/internal/serve/graphql/resolvers/statechange_resolvers_test.go @@ -338,6 +338,7 @@ func TestStateChangeResolver_Operation(t *testing.T) { OperationID: opID, StateChangeID: 1, StateChangeCategory: types.StateChangeCategoryBalance, + LedgerCreatedAt: sharedTestLedgerCreatedAt, }, } @@ -394,6 +395,7 @@ func TestStateChangeResolver_Transaction(t *testing.T) { ToID: toid.New(1000, 1, 0).ToInt64(), StateChangeID: 1, StateChangeCategory: types.StateChangeCategoryBalance, + LedgerCreatedAt: sharedTestLedgerCreatedAt, }, } diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index 365fcf782..0aa36cf76 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -51,6 +51,11 @@ var sharedTestAccountAddress = keypair.MustRandom().Address() // sharedNonExistentAccountAddress is a valid Stellar address that doesn't exist in the test DB. var sharedNonExistentAccountAddress = keypair.MustRandom().Address() +// sharedTestLedgerCreatedAt is the ledger_created_at set on every row setupDB inserts. Data-layer +// batch lookups pin this partition column, so any test building a parent node (Transaction, +// Operation, StateChange) that references setupDB's fixture must carry this exact value. +var sharedTestLedgerCreatedAt time.Time + // Test transaction hashes used by setupDB (64-char hex strings for BYTEA storage) // These must match the pattern in setupDB: fmt.Sprintf("...487%x", i) where i = 0,1,2,3 var ( @@ -63,6 +68,7 @@ var ( func setupDB(ctx context.Context, t *testing.T, dbConnectionPool *pgxpool.Pool) { testLedger := int32(1000) now := time.Now() + sharedTestLedgerCreatedAt = now parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} txns := make([]*types.Transaction, 0, 4) ops := make([]*types.Operation, 0, 8) diff --git a/internal/serve/graphql/resolvers/transaction.resolvers.go b/internal/serve/graphql/resolvers/transaction.resolvers.go index 6ffafc610..38794b909 100644 --- a/internal/serve/graphql/resolvers/transaction.resolvers.go +++ b/internal/serve/graphql/resolvers/transaction.resolvers.go @@ -34,11 +34,12 @@ func (r *transactionResolver) Operations(ctx context.Context, obj *types.Transac loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.OperationColumnsKey{ - ToID: obj.ToID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.Cursor, - SortOrder: params.SortOrder, + ToID: obj.ToID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.Cursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } operations, err := loaders.OperationsByToIDLoader.Load(ctx, loaderKey) @@ -98,11 +99,12 @@ func (r *transactionResolver) StateChanges(ctx context.Context, obj *types.Trans loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.StateChangeColumnsKey{ - ToID: obj.ToID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.StateChangeCursor, - SortOrder: params.SortOrder, + ToID: obj.ToID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.StateChangeCursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } stateChanges, err := loaders.StateChangesByToIDLoader.Load(ctx, loaderKey) diff --git a/internal/serve/graphql/resolvers/transaction_resolvers_test.go b/internal/serve/graphql/resolvers/transaction_resolvers_test.go index 70e7d5ff8..00cfcd180 100644 --- a/internal/serve/graphql/resolvers/transaction_resolvers_test.go +++ b/internal/serve/graphql/resolvers/transaction_resolvers_test.go @@ -37,7 +37,7 @@ func TestTransactionResolver_Operations(t *testing.T) { }, }} // ToID=toid.New(1000, 1, 0) matches the test data setup in test_utils.go (testLedger=1000, i=0) - parentTx := &types.Transaction{Hash: types.HashBytea(testTxHash1), ToID: toid.New(1000, 1, 0).ToInt64()} + parentTx := &types.Transaction{Hash: types.HashBytea(testTxHash1), ToID: toid.New(1000, 1, 0).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -219,7 +219,7 @@ func TestTransactionResolver_StateChanges(t *testing.T) { }, }, }} - parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64()} + parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} nonExistentTx := &types.Transaction{Hash: "2376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 0} t.Run("success without pagination", func(t *testing.T) { diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 99605a501..fce9fe6ed 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -162,6 +162,11 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { return nil, fmt.Errorf("building protocol processor map: %w", err) } + ledgerIndexer, err := indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion) + if err != nil { + return nil, fmt.Errorf("creating ledger indexer: %w", err) + } + return &ingestService{ ingestionMode: cfg.IngestionMode, models: cfg.Models, @@ -176,7 +181,7 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { appMetrics: cfg.Metrics, networkPassphrase: cfg.NetworkPassphrase, getLedgersLimit: cfg.GetLedgersLimit, - ledgerIndexer: indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion), + ledgerIndexer: ledgerIndexer, contractMetadataService: cfg.ContractMetadataService, protocolValidators: cfg.ProtocolValidators, wasmSpecExtractor: cfg.WasmSpecExtractor, diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index a632225b0..d98036b65 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -105,7 +105,11 @@ func (m *ingestService) calculateBackfillGaps(ctx context.Context, startLedger, return nil, fmt.Errorf("getting oldest ingest ledger: %w", err) } - currentGaps, err := m.models.IngestStore.GetLedgerGaps(ctx) + // Window at oldestIngestedLedger (guaranteed to be a present ledger_number, never inside a + // gap) rather than startLedger: this lets GetLedgerGaps's chunk-skipping-friendly WHERE + // clause bound the scan without duplicating the explicit "before history began" gap that + // case 2 below already appends, and without ever needing to synthesize a leading gap. + currentGaps, err := m.models.IngestStore.GetLedgerGaps(ctx, oldestIngestedLedger, endLedger) if err != nil { return nil, fmt.Errorf("calculating gaps in ledger range: %w", err) } diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 69b0e221d..470526da3 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1874,6 +1874,10 @@ type testProtocolProcessor struct { func (p *testProtocolProcessor) ProtocolID() string { return p.id } +func (p *testProtocolProcessor) StateChangeOrdinalBase() int64 { + return types.StateChangeOrdinalBaseSEP41 +} + func (p *testProtocolProcessor) Reset() { p.stagedLedgerCount = 0 } func (p *testProtocolProcessor) ProcessLedger(_ context.Context, input ProtocolProcessorInput) error { diff --git a/internal/services/mocks.go b/internal/services/mocks.go index 407679720..ea673acb6 100644 --- a/internal/services/mocks.go +++ b/internal/services/mocks.go @@ -264,6 +264,11 @@ func (m *ProtocolProcessorMock) ProtocolID() string { return args.String(0) } +func (m *ProtocolProcessorMock) StateChangeOrdinalBase() int64 { + args := m.Called() + return args.Get(0).(int64) +} + func (m *ProtocolProcessorMock) ProcessLedger(ctx context.Context, input ProtocolProcessorInput) error { args := m.Called(ctx, input) return args.Error(0) diff --git a/internal/services/processor_registry.go b/internal/services/processor_registry.go index 97683955b..dca9b7462 100644 --- a/internal/services/processor_registry.go +++ b/internal/services/processor_registry.go @@ -3,6 +3,8 @@ package services import ( "fmt" "sort" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) // processorRegistry holds factory functions keyed by protocol ID. Factories @@ -45,8 +47,17 @@ func GetAllProcessorIDs() []string { // BuildProcessors materializes processors for the given protocol IDs from the // registry. Returns an error if any ID has no registered factory. +// +// It also validates the state_change_id namespace bases of the built set: every +// base must be positive and a multiple of types.StateChangeOrdinalNamespaceWidth +// (base 0 is the main indexer's sub-namespaced region and is off-limits to +// protocol processors), and no two processors may share a base. Both boot entry +// points (internal/ingest and cmd/protocol_migrate) funnel through here, so a +// stale or duplicated base copied into a new processor fails fast at startup +// rather than silently colliding state_change_ids on-disk. func BuildProcessors(deps ProtocolDeps, protocolIDs []string) ([]ProtocolProcessor, error) { out := make([]ProtocolProcessor, 0, len(protocolIDs)) + protocolIDByBase := make(map[int64]string, len(protocolIDs)) for _, pid := range protocolIDs { factory, ok := GetProcessor(pid) if !ok { @@ -56,6 +67,18 @@ func BuildProcessors(deps ProtocolDeps, protocolIDs []string) ([]ProtocolProcess if p == nil { return nil, fmt.Errorf("processor factory for protocol %q returned nil", pid) } + + base := p.StateChangeOrdinalBase() + if base <= 0 || base%types.StateChangeOrdinalNamespaceWidth != 0 { + return nil, fmt.Errorf("protocol %q has invalid state_change_id base %d: "+ + "must be a positive multiple of %d", p.ProtocolID(), base, types.StateChangeOrdinalNamespaceWidth) + } + if other, dup := protocolIDByBase[base]; dup { + return nil, fmt.Errorf("protocols %q and %q share state_change_id base %d", + other, p.ProtocolID(), base) + } + protocolIDByBase[base] = p.ProtocolID() + out = append(out, p) } return out, nil diff --git a/internal/services/processor_registry_ext_test.go b/internal/services/processor_registry_ext_test.go new file mode 100644 index 000000000..90f76de07 --- /dev/null +++ b/internal/services/processor_registry_ext_test.go @@ -0,0 +1,43 @@ +package services_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/indexer/types" + "github.com/stellar/wallet-backend/internal/services" + + // Blank-import every protocol package so its init() registers with the + // framework. This test lives in the external services_test package to avoid + // the import cycle a protocol package (which imports services) would create + // with an in-package test. Add new protocol packages here as they land so + // BuildProcessors validates their state_change_id bases in CI automatically. + _ "github.com/stellar/wallet-backend/internal/services/sep41" +) + +// TestBuildProcessorsAllRegistered builds the full registered set with +// zero-value deps and asserts BuildProcessors accepts it: every base is a +// positive multiple of the namespace width and no two collide. Any future +// protocol whose factory tolerates zero-value deps is checked here for free. +func TestBuildProcessorsAllRegistered(t *testing.T) { + ids := services.GetAllProcessorIDs() + require.NotEmpty(t, ids, "expected at least one registered protocol processor") + + procs, err := services.BuildProcessors(services.ProtocolDeps{}, ids) + require.NoError(t, err) + assert.Len(t, procs, len(ids)) + + seen := make(map[int64]string, len(procs)) + for _, p := range procs { + base := p.StateChangeOrdinalBase() + assert.Positive(t, base, "protocol %q base must be positive", p.ProtocolID()) + assert.Zero(t, base%types.StateChangeOrdinalNamespaceWidth, + "protocol %q base %d must be a multiple of %d", p.ProtocolID(), base, types.StateChangeOrdinalNamespaceWidth) + if other, dup := seen[base]; dup { + t.Errorf("protocols %q and %q share base %d", other, p.ProtocolID(), base) + } + seen[base] = p.ProtocolID() + } +} diff --git a/internal/services/processor_registry_test.go b/internal/services/processor_registry_test.go index 3af5809ea..da3c97e37 100644 --- a/internal/services/processor_registry_test.go +++ b/internal/services/processor_registry_test.go @@ -5,8 +5,27 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) +// stubProcessor is a minimal ProtocolProcessor for BuildProcessors validation +// tests: it carries an ID and a state_change_id base and no-ops everything else. +type stubProcessor struct { + ProtocolProcessor // embed the interface; only the methods below are exercised + id string + base int64 +} + +func (s stubProcessor) ProtocolID() string { return s.id } +func (s stubProcessor) StateChangeOrdinalBase() int64 { return s.base } + +func registerStub(id string, base int64) { + RegisterProcessor(id, func(ProtocolDeps) ProtocolProcessor { + return stubProcessor{id: id, base: base} + }) +} + func withCleanProcessorRegistry(t *testing.T) { t.Helper() original := processorRegistry @@ -63,3 +82,46 @@ func TestRegisterProcessor(t *testing.T) { assert.Equal(t, []string{"A", "B"}, ids) }) } + +func TestBuildProcessorsBaseValidation(t *testing.T) { + w := types.StateChangeOrdinalNamespaceWidth + + t.Run("distinct valid bases succeed", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 1*w) + registerStub("B", 2*w) + + procs, err := BuildProcessors(ProtocolDeps{}, []string{"A", "B"}) + require.NoError(t, err) + assert.Len(t, procs, 2) + }) + + t.Run("duplicate bases error names both protocols", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 3*w) + registerStub("B", 3*w) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A", "B"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + assert.Contains(t, err.Error(), "B") + }) + + t.Run("zero base errors", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 0) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + }) + + t.Run("non-multiple base errors", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", w+1) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + }) +} diff --git a/internal/services/protocol_migrate_test.go b/internal/services/protocol_migrate_test.go index c32949fdc..5d0e2c5e6 100644 --- a/internal/services/protocol_migrate_test.go +++ b/internal/services/protocol_migrate_test.go @@ -20,6 +20,7 @@ import ( "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" "github.com/stellar/wallet-backend/internal/utils" ) @@ -184,6 +185,10 @@ type testRecordingProcessor struct { func (p *testRecordingProcessor) ProtocolID() string { return p.id } +func (p *testRecordingProcessor) StateChangeOrdinalBase() int64 { + return types.StateChangeOrdinalBaseSEP41 +} + func (p *testRecordingProcessor) Reset() { p.resetCount++ } func (p *testRecordingProcessor) ProcessLedger(_ context.Context, input ProtocolProcessorInput) error { diff --git a/internal/services/protocol_processor.go b/internal/services/protocol_processor.go index 0f880b712..6e0ccf450 100644 --- a/internal/services/protocol_processor.go +++ b/internal/services/protocol_processor.go @@ -28,8 +28,38 @@ func (m StagingMode) NeedsHistory() bool { return m != StagingModeCurrentState } func (m StagingMode) NeedsCurrentState() bool { return m != StagingModeHistory } // ProtocolProcessor produces and persists protocol-specific state for a ledger. +// +// Determinism contract for state changes: state_change_id values are frozen at +// persist and their base is never renumbered, so every emitted state change +// must land at a byte-identical ID on every re-run of the same input. This +// requires the relative order of all state changes within one +// (to_id, operation_id) group — the slice handed to +// types.AssignStateChangeOrdinals — to be reproducible from canonical on-chain +// data (event index in transaction meta, XDR structure walk order) at the +// moment ordinals are assigned. It must never depend on Go map iteration, +// goroutine completion, or channel arrival order, and one group's emissions +// must never straddle a map-iteration boundary. +// +// Non-determinism is explicitly allowed ACROSS groups: ordinals are keyed per +// (to_id, operation_id), so anything that only reorders whole groups — parallel +// fan-out across operations, transactions, or ledgers, or iterating a map in +// which each key maps to a distinct group (as SEP-41 does over ContractEvents) — +// cannot affect IDs. +// +// Escape hatch: a processor that cannot naturally emit a group in canonical +// order (e.g. it computes one operation's changes concurrently) must sort the +// group by a deterministic total key before calling AssignStateChangeOrdinals. +// IDs need not be semantically chronological — only byte-identical on re-run. +// The ordering step must happen before rows are written. type ProtocolProcessor interface { ProtocolID() string + // StateChangeOrdinalBase returns this protocol's state_change_id namespace + // base (see the registry in internal/indexer/types). It must be a positive + // multiple of types.StateChangeOrdinalNamespaceWidth, unique across all + // registered processors, and must never change once rows exist with it. + // PersistHistory must assign IDs via types.AssignStateChangeOrdinals using + // exactly this base. + StateChangeOrdinalBase() int64 // ProcessLedger accumulates ("folds") this ledger's protocol state into the // processor's staged sets. It does NOT reset between ledgers — the caller owns // reset via Reset(). Folding a window of ledgers then calling a Persist method diff --git a/internal/services/sep41/processor.go b/internal/services/sep41/processor.go index 25faa4595..0f0086485 100644 --- a/internal/services/sep41/processor.go +++ b/internal/services/sep41/processor.go @@ -92,6 +92,9 @@ var _ services.ProtocolProcessor = (*processor)(nil) func (p *processor) ProtocolID() string { return ProtocolID } +// StateChangeOrdinalBase returns the SEP-41 state_change_id namespace base. +func (p *processor) StateChangeOrdinalBase() int64 { return types.StateChangeOrdinalBaseSEP41 } + // ProcessLedger consumes contract events that the indexer (or // ExtractContractEventsForLedger in the migration path) has already // extracted into the buffer. The processor never touches LedgerCloseMeta — @@ -317,6 +320,10 @@ func (p *processor) PersistHistory(ctx context.Context, dbTx pgx.Tx) error { if len(p.stagedStateChanges) == 0 { return nil } + // Assign deterministic state_change_ids in emission order, namespaced under + // the SEP-41 base so they can never collide with the main indexer's or + // Blend's IDs for the same operation (see types.AssignStateChangeOrdinals). + types.AssignStateChangeOrdinals(p.stagedStateChanges, p.StateChangeOrdinalBase()) if _, err := p.stateChanges.BatchCopy(ctx, dbTx, p.stagedStateChanges); err != nil { return fmt.Errorf("persisting %d SEP-41 state changes for ledger %d: %w", len(p.stagedStateChanges), p.ledgerNumber, err)