From f2aa5b1988c9ccbc73fae124d73ee759d30d20aa Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 14 Jul 2026 23:22:05 -0400 Subject: [PATCH 01/14] feat(db): retune storage params, API-sort indexes, idempotent TSDB policy config Storage parameters (migrations edited in place, pre-release): - fillfactor 80->90 on 14 upsert tables; measured churn (~0.1%/day on the large balance tables) uses half the 20% reserve, and COPY-loaded heaps carry the reserve permanently. liquidity_pools keeps 80 (every-row-per-day churn). - autovacuum_vacuum_cost_limit removed everywhere: cost_delay=0 disables cost-based throttling entirely, making cost_limit a no-op. - The aggressive autovacuum block now applies only to usage-scaling tables; protocol-bounded tiny blend tables use defaults. - sep41_allowances drops its redundant owner index (PK-leading column) and unused spender index; the expiration index stays for the sweep with its non-HOT-update tradeoff documented. Root query indexes: transactions/operations/state_changes replace the auto-created single-column time index with composites matching the API keyset sort, turning root first pages from full-chunk seq scan + heapsort into an index walk. TSDB runtime config: configureHypertableSettings now runs in every ingestion mode (a backfill-first bootstrap previously ran TimescaleDB's auto-created columnstore policy with unlimited maxchunks and no retention), and retention/reconcile jobs converge in place via alter_job so job IDs and run history survive restarts. --- docs/data-migrations/adding-a-protocol.md | 19 +- .../migrations/2025-06-10.2-transactions.sql | 9 + .../db/migrations/2025-06-10.3-operations.sql | 9 + .../migrations/2025-06-10.4-statechanges.sql | 9 + .../2026-01-12.0-trustline_balances.sql | 24 +- .../2026-01-15.0-native_balances.sql | 19 +- .../migrations/2026-01-16.0-sac-balances.sql | 19 +- .../2026-04-17.0-sep41_balances.sql | 5 +- .../2026-04-17.1-sep41_allowances.sql | 13 +- .../2026-07-02.0-liquidity_pools.sql | 15 +- .../2026-07-02.1-liquidity_pool_balances.sql | 23 +- internal/ingest/ingest.go | 10 +- internal/ingest/timescaledb.go | 228 ++++++++++++------ internal/ingest/timescaledb_test.go | 146 +++++++++++ 14 files changed, 411 insertions(+), 137 deletions(-) diff --git a/docs/data-migrations/adding-a-protocol.md b/docs/data-migrations/adding-a-protocol.md index db9a1023a..5b5ae8716 100644 --- a/docs/data-migrations/adding-a-protocol.md +++ b/docs/data-migrations/adding-a-protocol.md @@ -166,13 +166,12 @@ CREATE TABLE _ ( last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (...) -- Choose a primary key that fits your state's natural identity ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -- Add indexes based on your query patterns @@ -182,7 +181,19 @@ CREATE TABLE _ ( DROP TABLE IF EXISTS _; ``` -The storage parameters (`fillfactor = 80`, aggressive autovacuum) are tuned for heavy UPSERT workloads during ingestion. Adjust based on your protocol's write pattern. +`fillfactor = 90` is the default: it reserves 10% free space per page for HOT (Heap-Only +Tuple) updates without over-reserving for tables whose UPSERTs only touch non-indexed +columns. Use `fillfactor = 80` instead only for tables whose rows update on essentially +every ledger (e.g. AMM-style reserves that shift on every trade). `autovacuum_vacuum_cost_delay += 0` disables cost-based vacuum throttling for this table's worker; there is no +`autovacuum_vacuum_cost_limit` to set alongside it since a delay of 0 makes any cost +limit a no-op. + +The aggressive autovacuum block above is for tables whose row count grows with users or +usage (positions, balances, allowances). Skip it entirely — keep only `fillfactor = 90` +— for small, protocol-bounded current-state tables (pool configs, reserve configs) whose +row count is capped by the number of pools/reserves your protocol will ever have +deployed; default autovacuum behavior is fine for those. ### Data Model diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index 6043f34a2..c51dd9268 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -21,6 +21,15 @@ CREATE TABLE transactions ( SELECT enable_chunk_skipping('transactions', 'to_id'); +-- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB +-- auto-creates on the hypertable's partition column. The root GraphQL connection +-- sorts by (ledger_created_at DESC, to_id DESC), which the default index can't serve +-- for a top-N first page (it still needs a heapsort on to_id within each timestamp). +-- This composite index matches the API sort key directly and makes the auto-created +-- one redundant. +DROP INDEX IF EXISTS transactions_ledger_created_at_idx; +CREATE INDEX idx_transactions_created_at_sort ON transactions (ledger_created_at DESC, to_id DESC); + CREATE INDEX idx_transactions_hash ON transactions(hash); -- Table: transactions_accounts (TimescaleDB hypertable for automatic cleanup with retention) diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 77929329f..7f4804068 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -34,6 +34,15 @@ CREATE TABLE operations ( SELECT enable_chunk_skipping('operations', 'id'); +-- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB +-- auto-creates on the hypertable's partition column. The root GraphQL connection +-- sorts by (ledger_created_at DESC, id DESC), which the default index can't serve +-- for a top-N first page (it still needs a heapsort on id within each timestamp). +-- This composite index matches the API sort key directly and makes the auto-created +-- one redundant. +DROP INDEX IF EXISTS operations_ledger_created_at_idx; +CREATE INDEX idx_operations_created_at_sort ON operations (ledger_created_at DESC, id DESC); + -- Table: operations_accounts (TimescaleDB hypertable for automatic cleanup with retention) CREATE TABLE operations_accounts ( operation_id BIGINT NOT NULL, diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index f4fb26150..67cf2a7a9 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -58,6 +58,15 @@ CREATE TABLE state_changes ( SELECT enable_chunk_skipping('state_changes', 'to_id'); SELECT enable_chunk_skipping('state_changes', 'operation_id'); +-- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB +-- auto-creates on the hypertable's partition column. The root GraphQL connection +-- sorts by (ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC), +-- which the default index can't serve for a top-N first page (it still needs a +-- heapsort within each timestamp). This composite index matches the API sort key +-- directly and makes the auto-created one redundant. +DROP INDEX IF EXISTS state_changes_ledger_created_at_idx; +CREATE INDEX idx_state_changes_created_at_sort ON state_changes (ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); + CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); CREATE INDEX idx_state_changes_account_category ON state_changes(account_id, state_change_category, state_change_reason, ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); diff --git a/internal/db/migrations/2026-01-12.0-trustline_balances.sql b/internal/db/migrations/2026-01-12.0-trustline_balances.sql index 6b886d6c5..585a7ca64 100644 --- a/internal/db/migrations/2026-01-12.0-trustline_balances.sql +++ b/internal/db/migrations/2026-01-12.0-trustline_balances.sql @@ -20,27 +20,27 @@ CREATE TABLE trustline_balances ( FOREIGN KEY (asset_id) REFERENCES trustline_assets(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, -- Base dead-row count added to (scale_factor * total_rows). Default is fine here -- since the scale factor already keeps the threshold low. autovacuum_vacuum_threshold = 50, - -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, - -- so stale stats can cause bad query plans (e.g. on the GetByAccount JOIN). + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger. + -- The current readers are PK/nested-loop lookups and stats-insensitive; fresh stats + -- matter for range/aggregate queries. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-15.0-native_balances.sql b/internal/db/migrations/2026-01-15.0-native_balances.sql index 282a9952a..c76b7d90a 100644 --- a/internal/db/migrations/2026-01-15.0-native_balances.sql +++ b/internal/db/migrations/2026-01-15.0-native_balances.sql @@ -14,10 +14,12 @@ CREATE TABLE native_balances ( num_subentries INTEGER NOT NULL DEFAULT 0, last_modified_ledger BIGINT NOT NULL DEFAULT 0 ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, @@ -28,13 +30,10 @@ CREATE TABLE native_balances ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-01-16.0-sac-balances.sql b/internal/db/migrations/2026-01-16.0-sac-balances.sql index db97ecf43..a62dd0ff8 100644 --- a/internal/db/migrations/2026-01-16.0-sac-balances.sql +++ b/internal/db/migrations/2026-01-16.0-sac-balances.sql @@ -18,10 +18,12 @@ CREATE TABLE sac_balances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). For a 500K-row table, -- this means vacuum starts at ~10K dead rows instead of waiting for 100K. autovacuum_vacuum_scale_factor = 0.02, @@ -32,13 +34,10 @@ CREATE TABLE sac_balances ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. Per-table cost settings exempt this worker - -- from global cost balancing, so other tables' vacuum workers keep their full budget. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-04-17.0-sep41_balances.sql b/internal/db/migrations/2026-04-17.0-sep41_balances.sql index 00904116d..b49b80d09 100644 --- a/internal/db/migrations/2026-04-17.0-sep41_balances.sql +++ b/internal/db/migrations/2026-04-17.0-sep41_balances.sql @@ -14,13 +14,12 @@ CREATE TABLE sep41_balances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql index 9242fa3ce..2194f82d7 100644 --- a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql +++ b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql @@ -15,17 +15,20 @@ CREATE TABLE sep41_allowances ( FOREIGN KEY (contract_id) REFERENCES contract_tokens(id) DEFERRABLE INITIALLY DEFERRED ) WITH ( - fillfactor = 80, + fillfactor = 90, autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - autovacuum_vacuum_cost_delay = 0, - autovacuum_vacuum_cost_limit = 1000 + autovacuum_vacuum_cost_delay = 0 ); -CREATE INDEX idx_sep41_allowances_owner ON sep41_allowances(owner_id); -CREATE INDEX idx_sep41_allowances_spender ON sep41_allowances(spender_id); +-- owner_id is the PK's leading column, so a lookup by owner alone already uses the PK +-- index; a dedicated single-column index on it would be redundant. +-- No code path filters by spender_id alone, so a dedicated index for it would sit unused. +-- Backs the expired-allowance sweep's ORDER BY expiration_ledger LIMIT n. BatchUpsert +-- updates expiration_ledger, so this makes those updates non-HOT — an accepted +-- tradeoff at allowance write volumes. CREATE INDEX idx_sep41_allowances_expiration_ledger ON sep41_allowances(expiration_ledger); -- +migrate Down diff --git a/internal/db/migrations/2026-07-02.0-liquidity_pools.sql b/internal/db/migrations/2026-07-02.0-liquidity_pools.sql index ec58deb79..b7e3c6742 100644 --- a/internal/db/migrations/2026-07-02.0-liquidity_pools.sql +++ b/internal/db/migrations/2026-07-02.0-liquidity_pools.sql @@ -17,8 +17,9 @@ CREATE TABLE liquidity_pools ( last_modified_ledger BIGINT NOT NULL DEFAULT 0 ) WITH ( -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. - -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. + -- Unlike the other balance/state tables, this table's reserves update on essentially + -- every row per day (each pool's amount_a/amount_b shifts on every ledger it trades), + -- so it needs the full 20% reserve to sustain HOT updates. fillfactor = 80, -- Trigger vacuum when 2% of rows are dead (default 20%). autovacuum_vacuum_scale_factor = 0.02, @@ -27,12 +28,10 @@ CREATE TABLE liquidity_pools ( -- so stale stats can cause bad query plans. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql b/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql index 80c0e4802..5ba35f9ed 100644 --- a/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql +++ b/internal/db/migrations/2026-07-02.1-liquidity_pool_balances.sql @@ -13,23 +13,24 @@ CREATE TABLE liquidity_pool_balances ( last_modified_ledger BIGINT NOT NULL DEFAULT 0, PRIMARY KEY (account_id, pool_id) ) WITH ( - -- Reserve 20% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. + -- Reserve 10% free space per page so PostgreSQL can do HOT (Heap-Only Tuple) updates. -- HOT updates rewrite the row in-place on the same page without creating dead tuples - -- or new index entries, since no indexed column is modified during UPSERTs. - fillfactor = 80, + -- or new index entries, since no indexed column is modified during UPSERTs. Measured + -- churn on this table is ~0.1%/day; a 20% reserve was double what the workload can + -- use, and COPY-loaded heaps carry the reserve permanently regardless. + fillfactor = 90, -- Trigger vacuum when 2% of rows are dead (default 20%). autovacuum_vacuum_scale_factor = 0.02, autovacuum_vacuum_threshold = 50, - -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger, - -- so stale stats can cause bad query plans (e.g. on the GetByAccount JOIN). + -- Refresh planner statistics at 1% change (default 10%). Balances shift every ledger. + -- The current readers are PK/nested-loop lookups and stats-insensitive; fresh stats + -- matter for range/aggregate queries. autovacuum_analyze_scale_factor = 0.01, autovacuum_analyze_threshold = 50, - -- No sleep between vacuum page-processing cycles (default 2ms). Per-table setting, - -- so only workers on this table run full-speed; other tables are unaffected. - autovacuum_vacuum_cost_delay = 0, - -- 5x the default page-processing budget per cycle (default 200). Combined with - -- cost_delay=0, vacuum finishes quickly. - autovacuum_vacuum_cost_limit = 1000 + -- Setting cost_delay=0 disables cost-based throttling for this table's autovacuum + -- worker and, per PostgreSQL's balancing rules, exempts it from cross-worker cost + -- balancing so other tables keep their full budget. + autovacuum_vacuum_cost_delay = 0 ); -- +migrate Down diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 4fe7cc9f9..2add68370 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -134,10 +134,12 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("connecting to the database: %w", err) } - if cfg.IngestionMode == services.IngestionModeLive { - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { - return nil, fmt.Errorf("configuring hypertable settings: %w", err) - } + // Configured regardless of ingestion mode: a backfill-first bootstrap must not + // run under TimescaleDB's auto-created columnstore policy defaults (unlimited + // maxchunks, no retention). All settings applied here are idempotent, so + // running this on every start (live or backfill) is safe. + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { + return nil, fmt.Errorf("configuring hypertable settings: %w", err) } m := metrics.NewMetrics(prometheus.NewRegistry()) diff --git a/internal/ingest/timescaledb.go b/internal/ingest/timescaledb.go index 3d3a4d9a2..231a30a1c 100644 --- a/internal/ingest/timescaledb.go +++ b/internal/ingest/timescaledb.go @@ -4,8 +4,10 @@ package ingest import ( "context" + "errors" "fmt" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/go-stellar-sdk/support/log" ) @@ -21,13 +23,15 @@ var hypertables = []string{ // configureHypertableSettings applies chunk interval, retention policy, and // compression schedule settings to all hypertables. Chunk interval only affects -// future chunks. Retention policy is idempotent: any existing policy is removed -// before re-adding. When retention is enabled, a reconciliation job keeps -// oldest_ingest_ledger in sync with the actual minimum ledger remaining after -// chunk drops. Compression schedule interval updates how frequently existing -// compression policy jobs run (does not create new policies). -// Compress after updates how long after a chunk closes before it becomes -// eligible for compression. +// future chunks. Retention policy and the reconciliation job are converged +// in place: an existing job is altered when its config differs from the +// desired value and left untouched when it already matches, so job_id and +// run history survive repeated calls (e.g. every process restart). When +// retention is enabled, the reconciliation job keeps oldest_ingest_ledger in +// sync with the actual minimum ledger remaining after chunk drops. Compression +// schedule interval updates how frequently existing compression policy jobs +// run (does not create new policies). Compress after updates how long after a +// chunk closes before it becomes eligible for compression. func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkInterval, retentionPeriod, oldestCursorName, compressionScheduleInterval, compressAfter string, maxChunksToCompress int) error { for _, table := range hypertables { if _, err := pool.Exec(ctx, @@ -39,73 +43,14 @@ func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkI log.Ctx(ctx).Infof("Set chunk interval %q on %s", chunkInterval, table) } - // We first remove existing retention policy for _, table := range hypertables { - if _, err := pool.Exec(ctx, - "SELECT remove_retention_policy($1::regclass, if_exists => true)", - table, - ); err != nil { - return fmt.Errorf("removing retention policy on %s: %w", table, err) + if err := configureRetentionPolicy(ctx, pool, table, retentionPeriod); err != nil { + return fmt.Errorf("configuring retention policy on %s: %w", table, err) } } - // Reconciliation job: keeps oldestCursorName in sync after retention drops chunks. - // Remove any existing job first (idempotent re-registration on every restart). - if _, err := pool.Exec(ctx, - "SELECT delete_job(job_id) FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", - ); err != nil { - return fmt.Errorf("removing existing reconciliation job: %w", err) - } - if retentionPeriod != "" { - // Add new retention period policy - for _, table := range hypertables { - if _, err := pool.Exec(ctx, - "SELECT add_retention_policy($1::regclass, drop_after => $2::interval)", - table, retentionPeriod, - ); err != nil { - return fmt.Errorf("adding retention policy on %s: %w", table, err) - } - log.Ctx(ctx).Infof("Set retention policy %q on %s", retentionPeriod, table) - } - - // Create or replace the PL/pgSQL function that advances the cursor. - if _, err := pool.Exec(ctx, ` - CREATE OR REPLACE FUNCTION reconcile_oldest_cursor(job_id INT, config JSONB) - RETURNS VOID LANGUAGE plpgsql AS $$ - DECLARE - actual_min INTEGER; - stored INTEGER; - BEGIN - SELECT ledger_number INTO actual_min FROM transactions - ORDER BY ledger_created_at ASC, to_id ASC LIMIT 1; - IF actual_min IS NULL THEN RETURN; END IF; - SELECT value::integer INTO stored FROM ingest_store WHERE key = config->>'cursor_name'; - IF stored IS NULL OR actual_min <= stored THEN RETURN; END IF; - UPDATE ingest_store SET value = actual_min::text WHERE key = config->>'cursor_name'; - RAISE LOG 'reconcile_oldest_cursor: advanced % from % to %', config->>'cursor_name', stored, actual_min; - END $$; - `); err != nil { - return fmt.Errorf("creating reconcile_oldest_cursor function: %w", err) - } - // Schedule the reconciliation job to run every 1 hour. - // - // The job checks whether retention has dropped chunks and advances the - // oldest ledger cursor if so. It is idempotent — a no-op when the cursor - // is already correct — and the query is microsecond-cheap (reads oldest - // chunk metadata + 1 row from ingest_store). Running on a fixed 1-hour - // interval keeps the cursor at most 1 hour stale after retention fires, - // with no coordination required with the retention job schedule. - if _, err := pool.Exec(ctx, ` - SELECT add_job( - 'reconcile_oldest_cursor', - '1 hour', - fixed_schedule => true, - config => $1::jsonb)`, - fmt.Sprintf(`{"cursor_name":"%s"}`, oldestCursorName), - ); err != nil { - return fmt.Errorf("scheduling reconciliation job: %w", err) - } - log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job (1h fixed interval) for cursor %q", oldestCursorName) + if err := configureReconciliationJob(ctx, pool, retentionPeriod, oldestCursorName); err != nil { + return fmt.Errorf("configuring reconciliation job: %w", err) } if compressionScheduleInterval != "" { @@ -185,3 +130,146 @@ func configureHypertableSettings(ctx context.Context, pool *pgxpool.Pool, chunkI return nil } + +// configureRetentionPolicy converges table's retention policy on the desired +// drop_after. An empty retentionPeriod disables retention, removing any +// existing policy. Otherwise, a missing policy is added; an existing one +// whose drop_after differs is altered in place (job_id and run history +// survive); one that already matches is left untouched. +func configureRetentionPolicy(ctx context.Context, pool *pgxpool.Pool, table, retentionPeriod string) error { + if retentionPeriod == "" { + if _, err := pool.Exec(ctx, + "SELECT remove_retention_policy($1::regclass, if_exists => true)", + table, + ); err != nil { + return fmt.Errorf("removing retention policy: %w", err) + } + return nil + } + + var jobID int + err := pool.QueryRow(ctx, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ).Scan(&jobID) + switch { + case errors.Is(err, pgx.ErrNoRows): + if _, err = pool.Exec(ctx, + "SELECT add_retention_policy($1::regclass, drop_after => $2::interval)", + table, retentionPeriod, + ); err != nil { + return fmt.Errorf("adding retention policy: %w", err) + } + log.Ctx(ctx).Infof("Added retention policy %q on %s", retentionPeriod, table) + return nil + case err != nil: + return fmt.Errorf("looking up existing retention policy: %w", err) + } + + var matches bool + if err := pool.QueryRow(ctx, + `SELECT (config->>'drop_after')::interval = $2::interval + FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, retentionPeriod, + ).Scan(&matches); err != nil { + return fmt.Errorf("comparing existing retention policy (job %d): %w", jobID, err) + } + if matches { + return nil + } + + if _, err := pool.Exec(ctx, + `SELECT alter_job($1, config => jsonb_set( + (SELECT config FROM timescaledb_information.jobs WHERE job_id = $1), + '{drop_after}', to_jsonb($2::text)))`, + jobID, retentionPeriod, + ); err != nil { + return fmt.Errorf("altering retention policy (job %d): %w", jobID, err) + } + log.Ctx(ctx).Infof("Updated retention policy to %q on %s (job %d)", retentionPeriod, table, jobID) + return nil +} + +// configureReconciliationJob converges the reconcile_oldest_cursor job, which +// keeps oldestCursorName in sync with the actual minimum ledger remaining +// after retention drops chunks. An empty retentionPeriod removes the job +// entirely — it has nothing to reconcile when retention is disabled. +// Otherwise, a missing job is added; an existing one whose cursor_name +// differs is altered in place (job_id and run history survive); one that +// already matches is left untouched. +func configureReconciliationJob(ctx context.Context, pool *pgxpool.Pool, retentionPeriod, oldestCursorName string) error { + if retentionPeriod == "" { + if _, err := pool.Exec(ctx, + "SELECT delete_job(job_id) FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", + ); err != nil { + return fmt.Errorf("removing reconciliation job: %w", err) + } + return nil + } + + // Create or replace the PL/pgSQL function that advances the cursor. + if _, err := pool.Exec(ctx, ` + CREATE OR REPLACE FUNCTION reconcile_oldest_cursor(job_id INT, config JSONB) + RETURNS VOID LANGUAGE plpgsql AS $$ + DECLARE + actual_min INTEGER; + stored INTEGER; + BEGIN + SELECT ledger_number INTO actual_min FROM transactions + ORDER BY ledger_created_at ASC, to_id ASC LIMIT 1; + IF actual_min IS NULL THEN RETURN; END IF; + SELECT value::integer INTO stored FROM ingest_store WHERE key = config->>'cursor_name'; + IF stored IS NULL OR actual_min <= stored THEN RETURN; END IF; + UPDATE ingest_store SET value = actual_min::text WHERE key = config->>'cursor_name'; + RAISE LOG 'reconcile_oldest_cursor: advanced % from % to %', config->>'cursor_name', stored, actual_min; + END $$; + `); err != nil { + return fmt.Errorf("creating reconcile_oldest_cursor function: %w", err) + } + + var jobID int + err := pool.QueryRow(ctx, + "SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'", + ).Scan(&jobID) + switch { + case errors.Is(err, pgx.ErrNoRows): + // Runs every 1 hour: cheap enough (oldest chunk metadata + 1 row from + // ingest_store) to not need coordination with the retention job's own + // schedule, and idempotent — a no-op once the cursor is already correct. + if _, err = pool.Exec(ctx, ` + SELECT add_job( + 'reconcile_oldest_cursor', + '1 hour', + fixed_schedule => true, + config => $1::jsonb)`, + fmt.Sprintf(`{"cursor_name":"%s"}`, oldestCursorName), + ); err != nil { + return fmt.Errorf("scheduling reconciliation job: %w", err) + } + log.Ctx(ctx).Infof("Scheduled reconcile_oldest_cursor job (1h fixed interval) for cursor %q", oldestCursorName) + return nil + case err != nil: + return fmt.Errorf("looking up existing reconciliation job: %w", err) + } + + var matches bool + if err := pool.QueryRow(ctx, + `SELECT config->>'cursor_name' = $2 FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, oldestCursorName, + ).Scan(&matches); err != nil { + return fmt.Errorf("comparing existing reconciliation job (job %d): %w", jobID, err) + } + if matches { + return nil + } + + if _, err := pool.Exec(ctx, + "SELECT alter_job($1, config => jsonb_build_object('cursor_name', $2::text))", + jobID, oldestCursorName, + ); err != nil { + return fmt.Errorf("altering reconciliation job (job %d): %w", jobID, err) + } + log.Ctx(ctx).Infof("Updated reconcile_oldest_cursor cursor to %q (job %d)", oldestCursorName, jobID) + return nil +} diff --git a/internal/ingest/timescaledb_test.go b/internal/ingest/timescaledb_test.go index f582b7cdc..0cdd61f13 100644 --- a/internal/ingest/timescaledb_test.go +++ b/internal/ingest/timescaledb_test.go @@ -114,6 +114,89 @@ func TestConfigureHypertableSettings(t *testing.T) { } }) + t.Run("retention_policy_job_id_stable_across_calls", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDsBefore := make(map[string]int) + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + jobIDsBefore[table] = jobID + } + + // Re-applying the same retention period must not delete/recreate the job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + assert.Equal(t, jobIDsBefore[table], jobID, "retention job_id should be stable across identical re-application for %s", table) + } + }) + + t.Run("retention_policy_altered_not_recreated", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDsBefore := make(map[string]int) + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + jobIDsBefore[table] = jobID + } + + // Changing the retention period must alter the existing job in place, + // not delete and recreate it. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + for _, table := range hypertables { + jobID, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs + WHERE proc_name = 'policy_retention' AND hypertable_name = $1`, + table, + ) + require.NoError(t, err, "querying retention job_id for %s", table) + assert.Equal(t, jobIDsBefore[table], jobID, "retention job_id should be stable when the period changes for %s", table) + + dropAfterMatches, err := db.QueryOne[bool](ctx, dbConnectionPool, + `SELECT (config->>'drop_after')::interval = '90 days'::interval + FROM timescaledb_information.jobs WHERE job_id = $1`, + jobID, + ) + require.NoError(t, err, "querying drop_after for %s", table) + assert.True(t, dropAfterMatches, "drop_after should be updated to 90 days for %s", table) + } + }) + t.Run("reconciliation_job_created", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -160,6 +243,69 @@ func TestConfigureHypertableSettings(t *testing.T) { assert.Equal(t, 1, count, "expected exactly 1 reconciliation job after re-application") }) + t.Run("reconciliation_job_id_stable_across_calls", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDBefore, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + + // Re-applying with a different retention period but the same cursor name + // must not delete/recreate the reconciliation job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "90 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDAfter, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, jobIDBefore, jobIDAfter, "reconciliation job_id should be stable across re-application") + }) + + t.Run("reconciliation_job_altered_not_recreated_when_cursor_changes", func(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "oldest_ledger_cursor", "", "", 0) + require.NoError(t, err) + + jobIDBefore, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + + // A changed cursor name must alter the existing job's config in place, + // not delete and recreate the job. + err = configureHypertableSettings(ctx, dbConnectionPool, "1 day", "30 days", "renamed_cursor", "", "", 0) + require.NoError(t, err) + + jobIDAfter, err := db.QueryOne[int](ctx, dbConnectionPool, + `SELECT job_id FROM timescaledb_information.jobs WHERE proc_name = 'reconcile_oldest_cursor'`, + ) + require.NoError(t, err) + assert.Equal(t, jobIDBefore, jobIDAfter, "reconciliation job_id should be stable when the cursor name changes") + + cursorName, err := db.QueryOne[string](ctx, dbConnectionPool, + `SELECT config->>'cursor_name' FROM timescaledb_information.jobs WHERE job_id = $1`, + jobIDAfter, + ) + require.NoError(t, err) + assert.Equal(t, "renamed_cursor", cursorName, "cursor_name should be updated in the reconciliation job's config") + }) + t.Run("no_reconciliation_job_without_retention", func(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() From 01fa42e9fd9b2e4c2cb59364eea4f4eb9702d672 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 00:09:18 -0400 Subject: [PATCH 02/14] perf(data): time-pinned lateral probes for batch queries; query hygiene MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The five dataloader batch queries (BatchGetByStateChangeIDs on transactions/operations, BatchGetByToIDs and BatchGetByOperationIDs on operations/state_changes) carried no ledger_created_at predicate, so every call hash-joined or band-joined against a full decompression of all chunks — 26-40s per batch on 9 days of mainnet data, growing linearly with retention. They are now UNNEST + CROSS JOIN LATERAL per-key probes with the partition column pinned from the parent's ledger time, projecting only the requested columns; the ORDER BY/LIMIT laterals sit behind an OFFSET 0 fence so the chunk-selecting scan keeps runtime chunk exclusion (measured on the dev replica: 2.5-18ms for 100-key batches with all 9 chunks excluded, constant with retention). Tuple IN-lists and interpolated limits become bound array/limit parameters. TransactionColumnsKey and the resolvers now thread the parent ledger time into the loader keys. Also: root GetAll queries fail closed on non-positive limits (a previously overflowable path silently dropped the LIMIT clause on state_changes); GetLedgerGaps takes the backfill window so gap detection scans only the requested ledger range via chunk-skipping, with open trailing gaps clipped to the window edge; the stateChanges txHash filter tolerates duplicate hashes (IN instead of a scalar subquery); dead ProtocolsModel.GetClassified removed. --- internal/data/ingest_store.go | 22 ++- internal/data/ingest_store_test.go | 62 ++++++- internal/data/mocks.go | 8 - internal/data/operations.go | 127 ++++++++----- internal/data/operations_test.go | 76 +++++++- internal/data/protocols.go | 21 --- internal/data/query_utils.go | 11 ++ internal/data/statechanges.go | 174 +++++++++++------- internal/data/statechanges_test.go | 138 +++++++++++++- internal/data/transactions.go | 40 ++-- internal/data/transactions_test.go | 16 +- .../graphql/dataloaders/operation_loaders.go | 10 +- .../dataloaders/statechange_loaders.go | 12 +- .../dataloaders/transaction_loaders.go | 14 +- .../graphql/resolvers/operation.resolvers.go | 11 +- .../resolvers/operation_resolvers_test.go | 6 +- internal/serve/graphql/resolvers/resolver.go | 15 +- .../resolvers/statechange.resolvers.go | 36 ++-- .../resolvers/statechange_resolvers_test.go | 2 + .../serve/graphql/resolvers/test_utils.go | 6 + .../resolvers/transaction.resolvers.go | 22 ++- .../resolvers/transaction_resolvers_test.go | 4 +- internal/services/ingest_backfill.go | 6 +- 23 files changed, 603 insertions(+), 236 deletions(-) diff --git a/internal/data/ingest_store.go b/internal/data/ingest_store.go index 7c022cc58..e64467538 100644 --- a/internal/data/ingest_store.go +++ b/internal/data/ingest_store.go @@ -169,19 +169,33 @@ func (m *IngestStoreModel) UpdateMin(ctx context.Context, dbTx pgx.Tx, cursorNam return nil } -func (m *IngestStoreModel) GetLedgerGaps(ctx context.Context) ([]LedgerRange, error) { +// GetLedgerGaps returns gaps between consecutive present ledger_number values within +// [startLedger, endLedger]. Windowing the DISTINCT scan on ledger_number (which has +// chunk-skipping stats enabled) makes the cost proportional to the window instead of a full +// hypertable scan. +// +// Edge semantics: callers must pass a startLedger that is itself a present ledger_number (e.g. +// the oldest ingested ledger) — the left edge is NOT synthesized. If startLedger fell strictly +// inside an already-open gap, this function would not report the portion before the first +// present row in-window; this mirrors the original unwindowed behavior, which never reported a +// gap before the very first row in the whole table. The right edge IS handled: COALESCE falls +// back to endLedger+1 when LEAD finds no next row within the window (the next present ledger +// lies beyond endLedger, or doesn't exist yet), so a gap still open at the window's boundary is +// reported clipped to endLedger instead of silently dropped (plain LEAD would return NULL there, +// failing the gap_start <= gap_end filter and losing the trailing partial gap). +func (m *IngestStoreModel) GetLedgerGaps(ctx context.Context, startLedger, endLedger uint32) ([]LedgerRange, error) { const query = ` SELECT gap_start, gap_end FROM ( SELECT ledger_number + 1 AS gap_start, - LEAD(ledger_number) OVER (ORDER BY ledger_number) - 1 AS gap_end - FROM (SELECT DISTINCT ledger_number FROM transactions) t + COALESCE(LEAD(ledger_number) OVER (ORDER BY ledger_number), $2 + 1) - 1 AS gap_end + FROM (SELECT DISTINCT ledger_number FROM transactions WHERE ledger_number BETWEEN $1 AND $2) t ) gaps WHERE gap_start <= gap_end ORDER BY gap_start ` start := time.Now() - ledgerGaps, err := db.QueryMany[LedgerRange](ctx, m.DB, query) + ledgerGaps, err := db.QueryMany[LedgerRange](ctx, m.DB, query, int32(startLedger), int32(endLedger)) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("GetLedgerGaps", "transactions").Observe(duration) m.Metrics.QueriesTotal.WithLabelValues("GetLedgerGaps", "transactions").Inc() diff --git a/internal/data/ingest_store_test.go b/internal/data/ingest_store_test.go index d2f4485e4..5fb8ed802 100644 --- a/internal/data/ingest_store_test.go +++ b/internal/data/ingest_store_test.go @@ -401,15 +401,21 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { testCases := []struct { name string + startLedger uint32 + endLedger uint32 setupDB func(t *testing.T) expectedGaps []LedgerRange }{ { name: "returns_empty_when_no_transactions", + startLedger: 1, + endLedger: 100, expectedGaps: []LedgerRange{}, }, { - name: "returns_empty_when_no_gaps", + name: "returns_empty_when_no_gaps", + startLedger: 100, + endLedger: 102, setupDB: func(t *testing.T) { // Insert consecutive ledgers: 100, 101, 102 for i, ledger := range []uint32{100, 101, 102} { @@ -423,7 +429,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { expectedGaps: []LedgerRange{}, }, { - name: "returns_single_gap", + name: "returns_single_gap", + startLedger: 100, + endLedger: 105, setupDB: func(t *testing.T) { // Insert ledgers 100 and 105, creating gap 101-104 for i, ledger := range []uint32{100, 105} { @@ -439,7 +447,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { }, }, { - name: "returns_multiple_gaps", + name: "returns_multiple_gaps", + startLedger: 100, + endLedger: 110, setupDB: func(t *testing.T) { // Insert ledgers 100, 105, 110, creating gaps 101-104 and 106-109 for i, ledger := range []uint32{100, 105, 110} { @@ -456,7 +466,9 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { }, }, { - name: "handles_single_ledger_gap", + name: "handles_single_ledger_gap", + startLedger: 100, + endLedger: 102, setupDB: func(t *testing.T) { // Insert ledgers 100 and 102, creating gap of just 101 for i, ledger := range []uint32{100, 102} { @@ -471,6 +483,46 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { {GapStart: 101, GapEnd: 101}, }, }, + { + // SQL-05: the real next ledger (200) lies beyond the requested window (150). The + // trailing gap must be reported clipped to endLedger, not dropped just because LEAD + // finds no next row inside the window. + name: "clips_trailing_gap_still_open_at_window_boundary", + startLedger: 100, + endLedger: 150, + setupDB: func(t *testing.T) { + for i, ledger := range []uint32{100, 200} { + _, err := dbConnectionPool.Exec(ctx, + `INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 150}, + }, + }, + { + // Ledgers outside [startLedger, endLedger] must not affect the result: a gap beyond + // endLedger (106-109) is excluded entirely, proving the query is bounded rather than + // scanning the whole table. + name: "ignores_gaps_entirely_outside_the_window", + startLedger: 100, + endLedger: 104, + setupDB: func(t *testing.T) { + for i, ledger := range []uint32{100, 105, 110} { + _, err := dbConnectionPool.Exec(ctx, + `INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES ($1, $2, 100, 'TransactionResultCodeTxSuccess', $3, NOW())`, + fmt.Sprintf("hash%d", i), i+1, ledger) + require.NoError(t, err) + } + }, + expectedGaps: []LedgerRange{ + {GapStart: 101, GapEnd: 104}, + }, + }, } for _, tc := range testCases { @@ -490,7 +542,7 @@ func Test_IngestStoreModel_GetLedgerGaps(t *testing.T) { tc.setupDB(t) } - gaps, err := m.GetLedgerGaps(ctx) + gaps, err := m.GetLedgerGaps(ctx, tc.startLedger, tc.endLedger) require.NoError(t, err) assert.Equal(t, tc.expectedGaps, gaps) }) diff --git a/internal/data/mocks.go b/internal/data/mocks.go index 05e0ca567..83a3aa96f 100644 --- a/internal/data/mocks.go +++ b/internal/data/mocks.go @@ -309,14 +309,6 @@ func (m *ProtocolsModelMock) GetByIDs(ctx context.Context, protocolIDs []string) return args.Get(0).([]Protocols), args.Error(1) } -func (m *ProtocolsModelMock) GetClassified(ctx context.Context) ([]Protocols, error) { - args := m.Called(ctx) - if args.Get(0) == nil { - return nil, args.Error(1) - } - return args.Get(0).([]Protocols), args.Error(1) -} - func (m *ProtocolsModelMock) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error { args := m.Called(ctx, dbTx, protocolID) return args.Error(0) diff --git a/internal/data/operations.go b/internal/data/operations.go index 7409d10c8..4a1787b70 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -38,6 +38,9 @@ func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) } func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") queryBuilder := strings.Builder{} var args []interface{} @@ -112,44 +115,59 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 // and the next transaction (index 4096 = 0x1000). // // See SEP-35: https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0035.md -func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") - queryBuilder := strings.Builder{} - // This CTE query implements per-transaction pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all operations from just a few - // transactions, we use ROW_NUMBER() with PARTITION BY to limit results per transaction. - // This guarantees that each transaction gets at most 'limit' operations, providing - // more balanced and predictable pagination across multiple transactions. - // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. - query := ` - WITH - inputs (to_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_operations_per_to_id AS ( - SELECT - o.*, - i.to_id as tx_to_id, - ROW_NUMBER() OVER (PARTITION BY i.to_id ORDER BY o.id %s) AS rn - FROM - operations o - JOIN - inputs i ON o.id > i.to_id AND o.id < i.to_id + 4096 - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM ranked_operations_per_to_id - ` - fmt.Fprintf(&queryBuilder, query, sortOrder, columns) +func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { + if len(toIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + + // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding + // (see the package doc above). Each key carries its parent transaction's ledger_created_at, + // so the LATERAL probe pins both the TOID band and the partition column, giving per-key + // runtime chunk exclusion instead of a Parallel Append across the whole hypertable. The + // ORDER BY + LIMIT live inside the LATERAL (a per-key top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-transaction cap; a subquery + // containing LIMIT cannot be flattened into a join, so the planner must run this as one + // per-key probe (see BatchGetByAccountAddress). Only the requested columns are selected + // inside the LATERAL, avoiding decompressing full rows before projection — so + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST) rather than + // o.ledger_created_at, which the LATERAL may not have projected; the two are identical for + // any row that survives the WHERE match. o.id is safe to reference directly since + // prepareColumnsWithID always forces "id" into the projection. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column name ("id"), not "o.id" — there is no "o" in scope at that + // level, only the fenced derived table. + args := []interface{}{toIDs, ledgerCreatedAts} + var queryBuilder strings.Builder + fmt.Fprintf(&queryBuilder, ` + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, o.id as cursor_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(to_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM operations o + WHERE o.id > k.to_id AND o.id < k.to_id + 4096 AND o.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY id %s`, columns, columns, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } - query = queryBuilder.String() + queryBuilder.WriteString(` + ) o + `) + + query := queryBuilder.String() if sortOrder == DESC { query = fmt.Sprintf(`SELECT * FROM (%s) AS operations ORDER BY operations.cursor_ledger_created_at ASC, operations.cursor_id ASC`, query) } start := time.Now() - operations, err := db.QueryManyPtrs[types.OperationWithCursor](ctx, m.DB, query, toIDs) + operations, err := db.QueryManyPtrs[types.OperationWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByToIDs", "operations").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByToIDs", "operations").Observe(float64(len(toIDs))) @@ -162,15 +180,16 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, col } // BatchGetByToID gets operations for a single transaction ToID with pagination support. -// Operations for a transaction are found using TOID range: (tx_to_id, tx_to_id + 4096). -func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { +// Operations for a transaction are found using TOID range: (tx_to_id, tx_to_id + 4096), pinned +// to the parent transaction's ledger_created_at for partition-column chunk exclusion. +func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") queryBuilder := strings.Builder{} // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. - fmt.Fprintf(&queryBuilder, `SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM operations WHERE id > $1 AND id < $1 + 4096`, columns) + fmt.Fprintf(&queryBuilder, `SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM operations WHERE id > $1 AND id < $1 + 4096 AND ledger_created_at = $2`, columns) - args := []interface{}{toID} - argIndex := 2 + args := []interface{}{toID, ledgerCreatedAt} + argIndex := 3 if cursor != nil { if sortOrder == DESC { @@ -348,26 +367,32 @@ func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAd } // BatchGetByStateChangeIDs gets the operations that are associated with the given state change IDs. -func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, stateChangeIDs []int64, columns string) ([]*types.OperationWithStateChangeID, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "operations", "id") - - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(stateChangeIDs)) - for i := range stateChangeIDs { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], stateChangeIDs[i]) +// Callers supply each key's parent ledger_created_at (the state change's ledger time, same ledger +// as its operation) so the LATERAL can pin the complete operations primary key (id, +// ledger_created_at) instead of joining through state_changes and hash-joining a full IN-list. +func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, stateChangeIDs []int64, ledgerCreatedAts []time.Time, columns string) ([]*types.OperationWithStateChangeID, error) { + if len(scOpIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("scOpIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scOpIDs), len(ledgerCreatedAts)) } + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + // ORDER BY reads k.ledger_created_at (from UNNEST), not o.ledger_created_at: the LATERAL + // only projects the caller-requested columns, which may not include ledger_created_at, but + // k.ledger_created_at is always in scope and identical to o.ledger_created_at for any row + // that survives the WHERE match. query := fmt.Sprintf(` - SELECT %s, CONCAT(state_changes.to_id, '-', state_changes.operation_id, '-', state_changes.state_change_id) AS state_change_id - FROM operations - INNER JOIN state_changes ON operations.id = state_changes.operation_id - WHERE (state_changes.to_id, state_changes.operation_id, state_changes.state_change_id) IN (%s) - ORDER BY operations.ledger_created_at DESC - `, columns, strings.Join(tuples, ", ")) + SELECT %s, CONCAT(k.to_id, '-', k.operation_id, '-', k.state_change_id) AS state_change_id + FROM UNNEST($1::bigint[], $2::bigint[], $3::bigint[], $4::timestamptz[]) AS k(to_id, operation_id, state_change_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT %s FROM operations o + WHERE o.id = k.operation_id AND o.ledger_created_at = k.ledger_created_at + LIMIT 1 + ) o + ORDER BY k.ledger_created_at DESC + `, columns, columns) start := time.Now() - operationsWithStateChanges, err := db.QueryManyPtrs[types.OperationWithStateChangeID](ctx, m.DB, query) + operationsWithStateChanges, err := db.QueryManyPtrs[types.OperationWithStateChangeID](ctx, m.DB, query, scToIDs, scOpIDs, stateChangeIDs, ledgerCreatedAts) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByStateChangeIDs", "operations").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByStateChangeIDs", "operations").Observe(float64(len(stateChangeIDs))) diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index 5e65be650..d2804deb4 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -392,7 +392,11 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) { Metrics: dbMetrics, } - operations, err := m.BatchGetByToIDs(ctx, tc.toIDs, "", tc.limit, tc.sortOrder) + ledgerCreatedAts := make([]time.Time, len(tc.toIDs)) + for i := range ledgerCreatedAts { + ledgerCreatedAts[i] = now + } + operations, err := m.BatchGetByToIDs(ctx, tc.toIDs, ledgerCreatedAts, "", tc.limit, tc.sortOrder) require.NoError(t, err) assert.Len(t, operations, tc.expectedCount) @@ -435,6 +439,44 @@ func TestOperationModel_BatchGetByToIDs(t *testing.T) { } }) } + + t.Run("wrong ledger_created_at for a key returns no operations for that key (time pin enforced)", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByToIDs(ctx, []int64{4096}, []time.Time{wrongTime}, "", nil, ASC) + require.NoError(t, err) + assert.Empty(t, operations) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + _, err := m.BatchGetByToIDs(ctx, []int64{4096, 8192}, []time.Time{now}, "", nil, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns) — a narrow column set that + // excludes ledgerCreatedAt must not break the query with "column does not exist". + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + operations, err := m.BatchGetByToIDs(ctx, []int64{4096}, []time.Time{now}, "operation_type", nil, ASC) + require.NoError(t, err) + require.Len(t, operations, 3) + for _, op := range operations { + assert.NotZero(t, op.CompositeCursor.LedgerCreatedAt) + assert.NotZero(t, op.CompositeCursor.ID) + } + }) } func int32Ptr(v int32) *int32 { @@ -486,11 +528,18 @@ func TestOperationModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) // Test BatchGetByToID - operations, err := m.BatchGetByToID(ctx, 4096, "", nil, nil, ASC) + operations, err := m.BatchGetByToID(ctx, 4096, now, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, operations, 2) assert.Equal(t, xdr1.String(), operations[0].OperationXDR.String()) assert.Equal(t, xdr3.String(), operations[1].OperationXDR.String()) + + t.Run("wrong ledger_created_at returns no operations (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByToID(ctx, 4096, wrongTime, "", nil, nil, ASC) + require.NoError(t, err) + assert.Empty(t, operations) + }) } func TestOperationModel_BatchGetByAccountAddresses(t *testing.T) { @@ -663,7 +712,8 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { require.NoError(t, err) // Test BatchGetByStateChangeID - operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192, 12288}, []int64{4097, 8193, 4097}, []int64{1, 1, 1}, "") + ledgerCreatedAts := []time.Time{now, now, now} + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192, 12288}, []int64{4097, 8193, 4097}, []int64{1, 1, 1}, ledgerCreatedAts, "") require.NoError(t, err) assert.Len(t, operations, 3) @@ -675,6 +725,26 @@ func TestOperationModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, int64(4097), stateChangeIDsFound["4096-4097-1"]) assert.Equal(t, int64(8193), stateChangeIDsFound["8192-8193-1"]) assert.Equal(t, int64(4097), stateChangeIDsFound["12288-4097-1"]) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096}, []int64{4097}, []int64{1}, []time.Time{wrongTime}, "") + require.NoError(t, err) + assert.Empty(t, operations) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096, 8192}, []int64{4097, 8193}, []int64{1, 1}, []time.Time{now}, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: ORDER BY must not read a column absent from the LATERAL's narrowed projection. + t.Run("narrow column selection excluding ledger_created_at does not break ORDER BY", func(t *testing.T) { + operations, err := m.BatchGetByStateChangeIDs(ctx, []int64{4096}, []int64{4097}, []int64{1}, []time.Time{now}, "operation_type") + require.NoError(t, err) + assert.Len(t, operations, 1) + }) } // BenchmarkOperationModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. diff --git a/internal/data/protocols.go b/internal/data/protocols.go index 2a22ba4ba..13410ee91 100644 --- a/internal/data/protocols.go +++ b/internal/data/protocols.go @@ -37,7 +37,6 @@ type ProtocolsModelInterface interface { UpdateHistoryMigrationStatus(ctx context.Context, dbTx pgx.Tx, protocolIDs []string, status string) error UpdateCurrentStateMigrationStatus(ctx context.Context, dbTx pgx.Tx, protocolIDs []string, status string) error GetByIDs(ctx context.Context, protocolIDs []string) ([]Protocols, error) - GetClassified(ctx context.Context) ([]Protocols, error) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error } @@ -145,26 +144,6 @@ func (m *ProtocolsModel) GetByIDs(ctx context.Context, protocolIDs []string) ([] return protocols, nil } -// GetClassified returns all protocols with classification_status = 'success'. -func (m *ProtocolsModel) GetClassified(ctx context.Context) ([]Protocols, error) { - const query = ` - SELECT id, classification_status, history_migration_status, current_state_migration_status, created_at, updated_at - FROM protocols - WHERE classification_status = 'success' - ` - - start := time.Now() - protocols, err := db.QueryMany[Protocols](ctx, m.DB, query) - duration := time.Since(start).Seconds() - m.Metrics.QueryDuration.WithLabelValues("GetClassified", "protocols").Observe(duration) - m.Metrics.QueriesTotal.WithLabelValues("GetClassified", "protocols").Inc() - if err != nil { - m.Metrics.QueryErrors.WithLabelValues("GetClassified", "protocols", utils.GetDBErrorType(err)).Inc() - return nil, fmt.Errorf("querying classified protocols: %w", err) - } - return protocols, nil -} - // InsertIfNotExists inserts a protocol if it doesn't already exist (idempotent). func (m *ProtocolsModel) InsertIfNotExists(ctx context.Context, dbTx pgx.Tx, protocolID string) error { const query = `INSERT INTO protocols (id) VALUES ($1) ON CONFLICT (id) DO NOTHING` diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 78fedbe5a..02a2befa5 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -49,6 +49,17 @@ const ( DESC SortOrder = "DESC" ) +// validatePositiveLimit rejects a non-nil limit that is zero or negative. A caller-supplied +// page size can go negative (e.g. an int32-overflowed `first` upstream), and silently dropping +// the LIMIT clause in that case turns a bounded page read into an unbounded full-table read. +// limit == nil still means "no limit" (used internally) and is left untouched. +func validatePositiveLimit(limit *int32) error { + if limit != nil && *limit <= 0 { + return fmt.Errorf("limit must be positive, got %d", *limit) + } + return nil +} + // pgtypeTextFromNullString converts sql.NullString to pgtype.Text for efficient binary COPY. func pgtypeTextFromNullString(ns sql.NullString) pgtype.Text { return pgtype.Text{String: ns.String, Valid: ns.Valid} diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 40fb21574..144b2a169 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -48,9 +48,11 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account // Time range filter: enables TimescaleDB chunk pruning on the state_changes hypertable args, argIndex = appendTimeRangeConditions(&queryBuilder, "ledger_created_at", timeRange, args, argIndex) - // Add transaction hash filter if provided (uses subquery to find to_id by hash) + // Add transaction hash filter if provided (uses subquery to find to_id(s) by hash). + // idx_transactions_hash is non-unique (TimescaleDB can't enforce unique(hash) on a + // hypertable), so this must tolerate more than one matching row instead of erroring. if txHash != nil { - fmt.Fprintf(&queryBuilder, " AND to_id = (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex) + fmt.Fprintf(&queryBuilder, " AND to_id IN (SELECT to_id FROM transactions WHERE hash = $%d)", argIndex) args = append(args, types.HashBytea(*txHash)) argIndex++ } @@ -125,6 +127,9 @@ func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, account } func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") var queryBuilder strings.Builder var args []interface{} @@ -156,7 +161,7 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in queryBuilder.WriteString(" ORDER BY ledger_created_at ASC, to_id ASC, operation_id ASC, state_change_id ASC") } - if limit != nil && *limit > 0 { + if limit != nil { fmt.Fprintf(&queryBuilder, " LIMIT $%d", argIndex) args = append(args, *limit) } @@ -331,18 +336,19 @@ func generateStateChangeID() (int64, error) { return int64(binary.BigEndian.Uint64(buf[:]) & 0x7FFFFFFFFFFFFFFF), nil } -// BatchGetByToID gets state changes for a single transaction with pagination support. -func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { +// BatchGetByToID gets state changes for a single transaction with pagination support, pinned to +// the parent transaction's ledger_created_at for partition-column chunk exclusion. +func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM state_changes - WHERE to_id = $1 + WHERE to_id = $1 AND ledger_created_at = $2 `, columns) - args := []interface{}{toID} - argIndex := 2 + args := []interface{}{toID, ledgerCreatedAt} + argIndex := 3 // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. @@ -390,35 +396,55 @@ func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, colum return stateChanges, nil } -// BatchGetByToIDs gets the state changes that are associated with the given to_ids. -func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// BatchGetByToIDs gets the state changes that are associated with the given to_ids. Callers +// supply each to_id's parent ledger_created_at so the LATERAL can pin the partition column, +// giving per-key runtime chunk exclusion instead of a hash join across the whole hypertable. +func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + if len(toIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id") + + // The ORDER BY + LIMIT live inside the LATERAL (a per-transaction top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-to_id cap; a subquery containing + // LIMIT cannot be flattened into a join, so the planner must run this as one per-key PK-band + // probe (see BatchGetByAccountAddress) instead of decompressing the whole hypertable. + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST), not sc.ledger_created_at: + // the LATERAL only projects the caller-requested columns, which may not include + // ledger_created_at, but k.ledger_created_at is always in scope and identical to + // sc.ledger_created_at for any row that survives the WHERE match. sc.to_id/operation_id/ + // state_change_id are safe to reference directly since prepareColumnsWithID always forces + // them into the projection. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column names, not "sc."-qualified ones — there is no "sc" in scope + // at that level, only the fenced derived table. ledger_created_at and to_id are dropped from + // the ORDER BY (not just unqualified): both are pinned to a single constant by the WHERE + // clause within one lateral probe, so they're no-op leading sort keys; operation_id and + // state_change_id are the only columns that actually vary among a probe's matched rows. + args := []interface{}{toIDs, ledgerCreatedAts} var queryBuilder strings.Builder - // This CTE query implements per-transaction pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all state changes from just a few - // transactions, we use ROW_NUMBER() with PARTITION BY to_id to limit results per transaction. - // This guarantees that each transaction gets at most 'limit' state changes, providing - // more balanced and predictable pagination across multiple transactions. fmt.Fprintf(&queryBuilder, ` - WITH - inputs (to_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_state_changes_per_to_id AS ( - SELECT - sc.*, - ROW_NUMBER() OVER (PARTITION BY sc.to_id ORDER BY sc.ledger_created_at %s, sc.to_id %s, sc.operation_id %s, sc.state_change_id %s) AS rn - FROM - state_changes sc - JOIN - inputs i ON sc.to_id = i.to_id - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM ranked_state_changes_per_to_id - `, sortOrder, sortOrder, sortOrder, sortOrder, columns) + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, sc.to_id as cursor_to_id, sc.operation_id as cursor_operation_id, sc.state_change_id as cursor_state_change_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(to_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM state_changes sc + WHERE sc.to_id = k.to_id AND sc.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY operation_id %s, state_change_id %s`, columns, columns, sortOrder, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } + queryBuilder.WriteString(` + ) sc + `) + query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. @@ -429,7 +455,7 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, c } start := time.Now() - stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, toIDs) + stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByToIDs", "state_changes").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByToIDs", "state_changes").Observe(float64(len(toIDs))) @@ -441,18 +467,19 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, c return stateChanges, nil } -// BatchGetByOperationID gets state changes for a single operation with pagination support. -func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { +// BatchGetByOperationID gets state changes for a single operation with pagination support, +// pinned to the parent operation's ledger_created_at for partition-column chunk exclusion. +func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM state_changes - WHERE operation_id = $1 + WHERE operation_id = $1 AND ledger_created_at = $2 `, columns) - args := []interface{}{operationID} - argIndex := 2 + args := []interface{}{operationID, ledgerCreatedAt} + argIndex := 3 // Decomposed cursor pagination: expands ROW() tuple comparison into OR clauses so // TimescaleDB ColumnarScan can push filters into vectorized batch processing. @@ -555,34 +582,53 @@ func (m *StateChangeModel) BatchGetAccountStateChangesByToIDs(ctx context.Contex } // BatchGetByOperationIDs gets the state changes that are associated with the given operation IDs. -func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") +// Callers supply each operation's parent ledger_created_at so the LATERAL can pin the partition +// column, giving per-key runtime chunk exclusion instead of a hash join across the whole +// hypertable. +func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, ledgerCreatedAts []time.Time, columns string, limit *int32, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { + if len(operationIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("operationIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(operationIDs), len(ledgerCreatedAts)) + } + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id") + + // The ORDER BY + LIMIT live inside the LATERAL (a per-operation top-N), replacing the old + // ROW_NUMBER()-over-everything CTE with an equivalent per-operation cap; a subquery + // containing LIMIT cannot be flattened into a join, so the planner must run this as one + // per-key probe (see BatchGetByAccountAddress) instead of decompressing the whole hypertable. + // cursor_ledger_created_at reads k.ledger_created_at (from UNNEST), not sc.ledger_created_at: + // the LATERAL only projects the caller-requested columns, which may not include + // ledger_created_at, but k.ledger_created_at is always in scope and identical to + // sc.ledger_created_at for any row that survives the WHERE match. + // + // OFFSET 0 is an optimization fence: it keeps the chunk-selecting scan (which gets runtime + // chunk exclusion from the ledger_created_at equality) planned separately from the ORDER + // BY/LIMIT, which would otherwise force a Merge Append probing every chunk's sparse index — + // O(chunk count) instead of O(1) with retention. After the fence, ORDER BY reads the + // subquery's bare output column names, not "sc."-qualified ones — there is no "sc" in scope + // at that level, only the fenced derived table. ledger_created_at and operation_id are + // dropped from the ORDER BY (not just unqualified): both are pinned to a single constant by + // the WHERE clause within one lateral probe, so they're no-op leading sort keys; to_id and + // state_change_id are the only columns that actually vary among a probe's matched rows. + args := []interface{}{operationIDs, ledgerCreatedAts} var queryBuilder strings.Builder - // This CTE query implements per-operation pagination to ensure balanced results. - // Instead of applying a global LIMIT that could return all state changes from just a few - // operations, we use ROW_NUMBER() with PARTITION BY operation_id to limit results per operation. - // This guarantees that each operation gets at most 'limit' state changes, providing - // more balanced and predictable pagination across multiple operations. fmt.Fprintf(&queryBuilder, ` - WITH - inputs (operation_id) AS ( - SELECT * FROM UNNEST($1::bigint[]) - ), - - ranked_state_changes_per_operation_id AS ( - SELECT - sc.*, - ROW_NUMBER() OVER (PARTITION BY sc.operation_id ORDER BY sc.ledger_created_at %s, sc.to_id %s, sc.operation_id %s, sc.state_change_id %s) AS rn - FROM - state_changes sc - JOIN - inputs i ON sc.operation_id = i.operation_id - ) - SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id FROM ranked_state_changes_per_operation_id - `, sortOrder, sortOrder, sortOrder, sortOrder, columns) + SELECT %s, k.ledger_created_at as cursor_ledger_created_at, sc.to_id as cursor_to_id, sc.operation_id as cursor_operation_id, sc.state_change_id as cursor_state_change_id + FROM UNNEST($1::bigint[], $2::timestamptz[]) AS k(operation_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT * FROM ( + SELECT %s FROM state_changes sc + WHERE sc.operation_id = k.operation_id AND sc.ledger_created_at = k.ledger_created_at + OFFSET 0 + ) sub + ORDER BY to_id %s, state_change_id %s`, columns, columns, sortOrder, sortOrder) if limit != nil { - fmt.Fprintf(&queryBuilder, " WHERE rn <= %d", *limit) + queryBuilder.WriteString(" LIMIT $3") + args = append(args, *limit) } + queryBuilder.WriteString(` + ) sc + `) + query := queryBuilder.String() // For backward pagination, wrap query to reverse the final order. @@ -593,7 +639,7 @@ func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operation } start := time.Now() - stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, operationIDs) + stateChanges, err := db.QueryManyPtrs[types.StateChangeWithCursor](ctx, m.DB, query, args...) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByOperationIDs", "state_changes").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByOperationIDs", "state_changes").Observe(float64(len(operationIDs))) diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 24f7156f9..1d83c6c36 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -494,6 +494,57 @@ func TestStateChangeModel_BatchGetByAccountAddress_WithFilters(t *testing.T) { }) } +// TestStateChangeModel_BatchGetByAccountAddress_DuplicateHashTolerated covers SQL-07: idx_transactions_hash +// is non-unique (TimescaleDB can't enforce unique(hash) on a hypertable), so a data bug could +// duplicate a hash across two to_ids. The txHash filter must tolerate this ("IN" rather than "=" / +// a scalar subquery) instead of erroring with "more than one row returned by a subquery". +func TestStateChangeModel_BatchGetByAccountAddress_DuplicateHashTolerated(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + now := time.Now() + address := keypair.MustRandom().Address() + dupHash := types.HashBytea("0000000000000000000000000000000000000000000000000000000000000009") + + // Two different transactions (different to_id, the hypertable's real uniqueness boundary) + // sharing the same hash. + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at) + VALUES + ($1, 1, 100, 'TransactionResultCodeTxSuccess', 1, $2), + ($1, 2, 100, 'TransactionResultCodeTxSuccess', 2, $2) + `, dupHash, now) + require.NoError(t, err) + + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO state_changes (to_id, state_change_id, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) + VALUES + (1, 1, 'BALANCE', 'CREDIT', $1, 1, $2, 123), + (2, 1, 'BALANCE', 'CREDIT', $1, 2, $2, 456) + `, now, types.AddressBytea(address)) + require.NoError(t, err) + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + txHash := dupHash.String() + stateChanges, err := m.BatchGetByAccountAddress(ctx, address, &txHash, nil, nil, nil, "", nil, nil, ASC, nil) + require.NoError(t, err, "duplicate hash must not error as 'more than one row returned by a subquery'") + assert.Len(t, stateChanges, 2, "state changes from both to_ids sharing the hash must be returned") + + toIDsFound := make(map[int64]bool) + for _, sc := range stateChanges { + toIDsFound[sc.StateChange.ToID] = true + } + assert.True(t, toIDsFound[1]) + assert.True(t, toIDsFound[2]) +} + func TestStateChangeModel_GetAll(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() @@ -671,7 +722,11 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { Metrics: dbMetrics, } - stateChanges, err := m.BatchGetByToIDs(ctx, tc.toIDs, "", tc.limit, tc.sortOrder) + ledgerCreatedAts := make([]time.Time, len(tc.toIDs)) + for i := range ledgerCreatedAts { + ledgerCreatedAts[i] = now + } + stateChanges, err := m.BatchGetByToIDs(ctx, tc.toIDs, ledgerCreatedAts, "", tc.limit, tc.sortOrder) require.NoError(t, err) assert.Len(t, stateChanges, tc.expectedCount) @@ -689,6 +744,43 @@ func TestStateChangeModel_BatchGetByToIDs(t *testing.T) { } }) } + + t.Run("wrong ledger_created_at for a key returns no state changes for that key (time pin enforced)", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByToIDs(ctx, []int64{1}, []time.Time{wrongTime}, "", nil, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + _, err := m.BatchGetByToIDs(ctx, []int64{1, 2}, []time.Time{now}, "", nil, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns) — a narrow column set that + // excludes ledgerCreatedAt must not break the query with "column does not exist". + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + stateChanges, err := m.BatchGetByToIDs(ctx, []int64{1}, []time.Time{now}, "state_change_category", nil, ASC) + require.NoError(t, err) + require.Len(t, stateChanges, 3) + for _, sc := range stateChanges { + assert.NotZero(t, sc.StateChangeCursor.LedgerCreatedAt) + } + }) } func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { @@ -736,7 +828,8 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { // Test BatchGetByOperationID limit := int32(10) - stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, "", &limit, ASC) + ledgerCreatedAts := []time.Time{now, now} + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, ledgerCreatedAts, "", &limit, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -747,6 +840,30 @@ func TestStateChangeModel_BatchGetByOperationIDs(t *testing.T) { } assert.Equal(t, 2, operationIDsFound[123]) assert.Equal(t, 1, operationIDsFound[456]) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123}, []time.Time{wrongTime}, "", &limit, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByOperationIDs(ctx, []int64{123, 456}, []time.Time{now}, "", &limit, ASC) + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) + + // Regression: the cursor's ledger_created_at must not be read off the LATERAL's own + // projection (which only carries the caller-requested columns). + t.Run("narrow column selection excluding ledger_created_at still resolves the cursor", func(t *testing.T) { + stateChanges, err := m.BatchGetByOperationIDs(ctx, []int64{123}, []time.Time{now}, "state_change_category", &limit, ASC) + require.NoError(t, err) + require.Len(t, stateChanges, 2) + for _, sc := range stateChanges { + assert.NotZero(t, sc.StateChangeCursor.LedgerCreatedAt) + } + }) } func TestStateChangeModel_BatchGetByToID(t *testing.T) { @@ -792,7 +909,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { require.NoError(t, err) t.Run("get all state changes for single to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", nil, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -809,7 +926,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with pagination - first", func(t *testing.T) { limit := int32(2) - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", &limit, nil, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -820,7 +937,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { t.Run("get state changes with cursor pagination", func(t *testing.T) { limit := int32(2) cursor := &types.StateChangeCursor{LedgerCreatedAt: now, ToID: 1, OperationID: 123, StateChangeID: 1} - stateChanges, err := m.BatchGetByToID(ctx, 1, "", &limit, cursor, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", &limit, cursor, ASC) require.NoError(t, err) assert.Len(t, stateChanges, 2) @@ -830,7 +947,7 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("get state changes with DESC ordering", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 1, "", nil, nil, DESC) + stateChanges, err := m.BatchGetByToID(ctx, 1, now, "", nil, nil, DESC) require.NoError(t, err) assert.Len(t, stateChanges, 3) @@ -841,7 +958,14 @@ func TestStateChangeModel_BatchGetByToID(t *testing.T) { }) t.Run("no state changes for non-existent to_id", func(t *testing.T) { - stateChanges, err := m.BatchGetByToID(ctx, 999, "", nil, nil, ASC) + stateChanges, err := m.BatchGetByToID(ctx, 999, now, "", nil, nil, ASC) + require.NoError(t, err) + assert.Empty(t, stateChanges) + }) + + t.Run("wrong ledger_created_at returns no state changes (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + stateChanges, err := m.BatchGetByToID(ctx, 1, wrongTime, "", nil, nil, ASC) require.NoError(t, err) assert.Empty(t, stateChanges) }) diff --git a/internal/data/transactions.go b/internal/data/transactions.go index 9ec07b9f3..cfdac954e 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -39,6 +39,9 @@ func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns s } func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *int32, cursor *types.CompositeCursor, sortOrder SortOrder) ([]*types.TransactionWithCursor, error) { + if err := validatePositiveLimit(limit); err != nil { + return nil, err + } columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") queryBuilder := strings.Builder{} var args []interface{} @@ -212,26 +215,33 @@ func (m *TransactionModel) BatchGetByOperationIDs(ctx context.Context, operation return transactions, nil } -// BatchGetByStateChangeIDs gets the transactions that are associated with the given state changes -func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, columns string) ([]*types.TransactionWithStateChangeID, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") - - // Build tuples for the IN clause. Since (to_id, operation_id, state_change_id) is the primary key of state_changes, - // it will be faster to search on this tuple. - tuples := make([]string, len(scOrders)) - for i := range scOrders { - tuples[i] = fmt.Sprintf("(%d, %d, %d)", scToIDs[i], scOpIDs[i], scOrders[i]) +// BatchGetByStateChangeIDs gets the transactions that are associated with the given state changes. +// The state-change key triple's to_id already identifies the parent transaction (to_id is the PK +// prefix of state_changes), so this probes transactions directly instead of joining through +// state_changes. Callers supply each key's parent ledger_created_at (the state change's ledger +// time, same ledger as its transaction) so the LATERAL can pin the complete transactions primary +// key (to_id, ledger_created_at). The LATERAL LIMIT 1 is load-bearing: a subquery containing LIMIT +// cannot be flattened into a join, so the planner must run this as one per-key PK probe (see +// BatchGetByAccountAddress) instead of decompressing the whole hypertable to hash-join a full +// IN-list of (to_id, operation_id, state_change_id) tuples. +func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs []int64, scOpIDs []int64, scOrders []int64, ledgerCreatedAts []time.Time, columns string) ([]*types.TransactionWithStateChangeID, error) { + if len(scToIDs) != len(ledgerCreatedAts) { + return nil, fmt.Errorf("scToIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scToIDs), len(ledgerCreatedAts)) } + columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id") query := fmt.Sprintf(` - SELECT %s, CONCAT(sc.to_id, '-', sc.operation_id, '-', sc.state_change_id) as state_change_id - FROM transactions - INNER JOIN state_changes sc ON transactions.to_id = sc.to_id - WHERE (sc.to_id, sc.operation_id, sc.state_change_id) IN (%s) - `, columns, strings.Join(tuples, ", ")) + SELECT %s, CONCAT(k.to_id, '-', k.operation_id, '-', k.state_change_id) AS state_change_id + FROM UNNEST($1::bigint[], $2::bigint[], $3::bigint[], $4::timestamptz[]) AS k(to_id, operation_id, state_change_id, ledger_created_at) + CROSS JOIN LATERAL ( + SELECT %s FROM transactions t + WHERE t.to_id = k.to_id AND t.ledger_created_at = k.ledger_created_at + LIMIT 1 + ) t + `, columns, columns) start := time.Now() - transactions, err := db.QueryManyPtrs[types.TransactionWithStateChangeID](ctx, m.DB, query) + transactions, err := db.QueryManyPtrs[types.TransactionWithStateChangeID](ctx, m.DB, query, scToIDs, scOpIDs, scOrders, ledgerCreatedAts) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("BatchGetByStateChangeIDs", "transactions").Observe(duration) m.Metrics.BatchSize.WithLabelValues("BatchGetByStateChangeIDs", "transactions").Observe(float64(len(scOrders))) diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index de94141ff..2ddf771f5 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -429,7 +429,8 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { require.NoError(t, err) // Test BatchGetByStateChangeID - transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2, 3}, []int64{1, 2, 3}, []int64{1, 1, 1}, "") + ledgerCreatedAts := []time.Time{now, now, now} + transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2, 3}, []int64{1, 2, 3}, []int64{1, 1, 1}, ledgerCreatedAts, "") require.NoError(t, err) assert.Len(t, transactions, 3) @@ -442,6 +443,19 @@ func TestTransactionModel_BatchGetByStateChangeIDs(t *testing.T) { assert.Equal(t, scTestHash1, stateChangeIDsFound["1-1-1"]) // to_id=1 -> scTestHash1 (to_id=1) assert.Equal(t, scTestHash2, stateChangeIDsFound["2-2-1"]) // to_id=2 -> scTestHash2 (to_id=2) assert.Equal(t, scTestHash3, stateChangeIDsFound["3-3-1"]) // to_id=3 -> scTestHash3 (to_id=3) + + t.Run("wrong ledger_created_at for a key excludes it (time pin enforced)", func(t *testing.T) { + wrongTime := now.Add(-24 * time.Hour) + transactions, err := m.BatchGetByStateChangeIDs(ctx, []int64{1}, []int64{1}, []int64{1}, []time.Time{wrongTime}, "") + require.NoError(t, err) + assert.Empty(t, transactions) + }) + + t.Run("mismatched array lengths error", func(t *testing.T) { + _, err := m.BatchGetByStateChangeIDs(ctx, []int64{1, 2}, []int64{1, 2}, []int64{1, 1}, []time.Time{now}, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "parallel arrays of equal length") + }) } // BenchmarkTransactionModel_BatchCopy benchmarks bulk insert using pgx's binary COPY protocol. diff --git a/internal/serve/graphql/dataloaders/operation_loaders.go b/internal/serve/graphql/dataloaders/operation_loaders.go index a99826461..3b0a51ea6 100644 --- a/internal/serve/graphql/dataloaders/operation_loaders.go +++ b/internal/serve/graphql/dataloaders/operation_loaders.go @@ -48,15 +48,17 @@ func operationsByToIDLoader(models *data.Models) *dataloadgen.Loader[OperationCo // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.Operations.BatchGetByToID(ctx, keys[0].ToID, columns, limit, keys[0].Cursor, sortOrder) + return models.Operations.BatchGetByToID(ctx, keys[0].ToID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } toIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) maxLimit := min(*limit, MaxOperationsPerBatch) for i, key := range keys { toIDs[i] = key.ToID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.Operations.BatchGetByToIDs(ctx, toIDs, columns, &maxLimit, sortOrder) + return models.Operations.BatchGetByToIDs(ctx, toIDs, ledgerCreatedAts, columns, &maxLimit, sortOrder) }, func(item *types.OperationWithCursor) int64 { // Derive tx_to_id from operation ID using TOID bit masking @@ -96,14 +98,16 @@ func operationByStateChangeIDLoader(models *data.Models) *dataloadgen.Loader[Ope func(ctx context.Context, keys []OperationColumnsKey) ([]*types.OperationWithStateChangeID, error) { columns := keys[0].Columns scIDs := make([]string, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { scIDs[i] = key.StateChangeID + ledgerCreatedAts[i] = key.LedgerCreatedAt } scToIDs, scOpIDs, scOrders, err := parseStateChangeIDs(scIDs) if err != nil { return nil, fmt.Errorf("parsing state change IDs: %w", err) } - return models.Operations.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, columns) + return models.Operations.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, ledgerCreatedAts, columns) }, func(item *types.OperationWithStateChangeID) string { return item.StateChangeID diff --git a/internal/serve/graphql/dataloaders/statechange_loaders.go b/internal/serve/graphql/dataloaders/statechange_loaders.go index ba26010fb..871881864 100644 --- a/internal/serve/graphql/dataloaders/statechange_loaders.go +++ b/internal/serve/graphql/dataloaders/statechange_loaders.go @@ -45,15 +45,17 @@ func stateChangesByToIDLoader(models *data.Models) *dataloadgen.Loader[StateChan // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.StateChanges.BatchGetByToID(ctx, keys[0].ToID, columns, limit, keys[0].Cursor, sortOrder) + return models.StateChanges.BatchGetByToID(ctx, keys[0].ToID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } toIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) maxLimit := min(*limit, MaxStateChangesPerBatch) for i, key := range keys { toIDs[i] = key.ToID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.StateChanges.BatchGetByToIDs(ctx, toIDs, columns, &maxLimit, sortOrder) + return models.StateChanges.BatchGetByToIDs(ctx, toIDs, ledgerCreatedAts, columns, &maxLimit, sortOrder) }, func(item *types.StateChangeWithCursor) int64 { return item.StateChange.ToID @@ -85,14 +87,16 @@ func stateChangesByOperationIDLoader(models *data.Models) *dataloadgen.Loader[St // If there is only one key, we can use a simpler query without resorting to the CTE expressions. // Also, when a single key is requested, we can allow using normal cursor based pagination. if len(keys) == 1 { - return models.StateChanges.BatchGetByOperationID(ctx, keys[0].OperationID, columns, limit, keys[0].Cursor, sortOrder) + return models.StateChanges.BatchGetByOperationID(ctx, keys[0].OperationID, keys[0].LedgerCreatedAt, columns, limit, keys[0].Cursor, sortOrder) } operationIDs := make([]int64, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { operationIDs[i] = key.OperationID + ledgerCreatedAts[i] = key.LedgerCreatedAt } - return models.StateChanges.BatchGetByOperationIDs(ctx, operationIDs, columns, limit, sortOrder) + return models.StateChanges.BatchGetByOperationIDs(ctx, operationIDs, ledgerCreatedAts, columns, limit, sortOrder) }, func(item *types.StateChangeWithCursor) int64 { return item.StateChange.OperationID diff --git a/internal/serve/graphql/dataloaders/transaction_loaders.go b/internal/serve/graphql/dataloaders/transaction_loaders.go index 36249761d..4c50a370a 100644 --- a/internal/serve/graphql/dataloaders/transaction_loaders.go +++ b/internal/serve/graphql/dataloaders/transaction_loaders.go @@ -3,6 +3,7 @@ package dataloaders import ( "context" "fmt" + "time" "github.com/vikstrous/dataloadgen" @@ -11,10 +12,11 @@ import ( ) type TransactionColumnsKey struct { - AccountID string - OperationID int64 - StateChangeID string - Columns string + AccountID string + OperationID int64 + StateChangeID string + Columns string + LedgerCreatedAt time.Time // parent state change's ledger time; pins the partition column for BatchGetByStateChangeIDs } // txByOperationIDLoader creates a dataloader for fetching transactions by operation ID @@ -50,14 +52,16 @@ func transactionByStateChangeIDLoader(models *data.Models) *dataloadgen.Loader[T func(ctx context.Context, keys []TransactionColumnsKey) ([]*types.TransactionWithStateChangeID, error) { columns := keys[0].Columns scIDs := make([]string, len(keys)) + ledgerCreatedAts := make([]time.Time, len(keys)) for i, key := range keys { scIDs[i] = key.StateChangeID + ledgerCreatedAts[i] = key.LedgerCreatedAt } scToIDs, scOpIDs, scOrders, err := parseStateChangeIDs(scIDs) if err != nil { return nil, fmt.Errorf("parsing state change IDs: %w", err) } - return models.Transactions.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, columns) + return models.Transactions.BatchGetByStateChangeIDs(ctx, scToIDs, scOpIDs, scOrders, ledgerCreatedAts, columns) }, func(item *types.TransactionWithStateChangeID) string { return item.StateChangeID diff --git a/internal/serve/graphql/resolvers/operation.resolvers.go b/internal/serve/graphql/resolvers/operation.resolvers.go index 1debac4a6..191dbecae 100644 --- a/internal/serve/graphql/resolvers/operation.resolvers.go +++ b/internal/serve/graphql/resolvers/operation.resolvers.go @@ -82,11 +82,12 @@ func (r *operationResolver) StateChanges(ctx context.Context, obj *types.Operati loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.StateChangeColumnsKey{ - OperationID: obj.ID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.StateChangeCursor, - SortOrder: params.SortOrder, + OperationID: obj.ID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.StateChangeCursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } stateChanges, err := loaders.StateChangesByOperationIDLoader.Load(ctx, loaderKey) diff --git a/internal/serve/graphql/resolvers/operation_resolvers_test.go b/internal/serve/graphql/resolvers/operation_resolvers_test.go index 9b6fd8c41..a42ec875a 100644 --- a/internal/serve/graphql/resolvers/operation_resolvers_test.go +++ b/internal/serve/graphql/resolvers/operation_resolvers_test.go @@ -28,7 +28,7 @@ func TestOperationResolver_Transaction(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -74,7 +74,7 @@ func TestOperationResolver_Accounts(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -124,7 +124,7 @@ func TestOperationResolver_StateChanges(t *testing.T) { }, }, }} - parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64()} + parentOperation := &types.Operation{ID: toid.New(1000, 1, 1).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} nonExistentOperation := &types.Operation{ID: 9999} t.Run("success without pagination", func(t *testing.T) { diff --git a/internal/serve/graphql/resolvers/resolver.go b/internal/serve/graphql/resolvers/resolver.go index 7ea29efa2..d3a79c634 100644 --- a/internal/serve/graphql/resolvers/resolver.go +++ b/internal/serve/graphql/resolvers/resolver.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "strings" + "time" "github.com/google/uuid" @@ -128,14 +129,15 @@ func (r *Resolver) resolveStateChangeAccount(accountID types.AddressBytea) (*typ // resolveStateChangeOperation resolves the operation field for any state change type // Reuses the existing logic from the original StateChange resolver -func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, operationID int64, stateChangeID int64) (*types.Operation, error) { +func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, operationID int64, stateChangeID int64, ledgerCreatedAt time.Time) (*types.Operation, error) { loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) dbColumns := GetDBColumnsForFields(ctx, types.Operation{}) stateChangeKey := fmt.Sprintf("%d-%d-%d", toID, operationID, stateChangeID) loaderKey := dataloaders.OperationColumnsKey{ - StateChangeID: stateChangeKey, - Columns: strings.Join(dbColumns, ", "), + StateChangeID: stateChangeKey, + Columns: strings.Join(dbColumns, ", "), + LedgerCreatedAt: ledgerCreatedAt, } operations, err := loaders.OperationByStateChangeIDLoader.Load(ctx, loaderKey) if err != nil { @@ -146,14 +148,15 @@ func (r *Resolver) resolveStateChangeOperation(ctx context.Context, toID int64, // resolveStateChangeTransaction resolves the transaction field for any state change type // Reuses the existing logic from the original StateChange resolver -func (r *Resolver) resolveStateChangeTransaction(ctx context.Context, toID int64, operationID int64, stateChangeID int64) (*types.Transaction, error) { +func (r *Resolver) resolveStateChangeTransaction(ctx context.Context, toID int64, operationID int64, stateChangeID int64, ledgerCreatedAt time.Time) (*types.Transaction, error) { loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) dbColumns := GetDBColumnsForFields(ctx, types.Transaction{}) stateChangeKey := fmt.Sprintf("%d-%d-%d", toID, operationID, stateChangeID) loaderKey := dataloaders.TransactionColumnsKey{ - StateChangeID: stateChangeKey, - Columns: strings.Join(dbColumns, ", "), + StateChangeID: stateChangeKey, + Columns: strings.Join(dbColumns, ", "), + LedgerCreatedAt: ledgerCreatedAt, } transaction, err := loaders.TransactionByStateChangeIDLoader.Load(ctx, loaderKey) if err != nil { diff --git a/internal/serve/graphql/resolvers/statechange.resolvers.go b/internal/serve/graphql/resolvers/statechange.resolvers.go index 6c7562368..d4e8bc153 100644 --- a/internal/serve/graphql/resolvers/statechange.resolvers.go +++ b/internal/serve/graphql/resolvers/statechange.resolvers.go @@ -30,12 +30,12 @@ func (r *accountChangeResolver) Account(ctx context.Context, obj *types.AccountS // Operation is the resolver for the operation field. func (r *accountChangeResolver) Operation(ctx context.Context, obj *types.AccountStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *accountChangeResolver) Transaction(ctx context.Context, obj *types.AccountStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // FunderAddress is the resolver for the funderAddress field. @@ -70,12 +70,12 @@ func (r *balanceAuthorizationChangeResolver) Account(ctx context.Context, obj *t // Operation is the resolver for the operation field. func (r *balanceAuthorizationChangeResolver) Operation(ctx context.Context, obj *types.BalanceAuthorizationStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *balanceAuthorizationChangeResolver) Transaction(ctx context.Context, obj *types.BalanceAuthorizationStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. @@ -114,12 +114,12 @@ func (r *flagsChangeResolver) Account(ctx context.Context, obj *types.FlagsState // Operation is the resolver for the operation field. func (r *flagsChangeResolver) Operation(ctx context.Context, obj *types.FlagsStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *flagsChangeResolver) Transaction(ctx context.Context, obj *types.FlagsStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Flags is the resolver for the flags field. @@ -148,12 +148,12 @@ func (r *metadataChangeResolver) Account(ctx context.Context, obj *types.Metadat // Operation is the resolver for the operation field. func (r *metadataChangeResolver) Operation(ctx context.Context, obj *types.MetadataStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *metadataChangeResolver) Transaction(ctx context.Context, obj *types.MetadataStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // KeyValue is the resolver for the keyValue field. @@ -178,12 +178,12 @@ func (r *reservesChangeResolver) Account(ctx context.Context, obj *types.Reserve // Operation is the resolver for the operation field. func (r *reservesChangeResolver) Operation(ctx context.Context, obj *types.ReservesStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *reservesChangeResolver) Transaction(ctx context.Context, obj *types.ReservesStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // SponsoredAddress is the resolver for the sponsoredAddress field. @@ -233,12 +233,12 @@ func (r *signerChangeResolver) Account(ctx context.Context, obj *types.SignerSta // Operation is the resolver for the operation field. func (r *signerChangeResolver) Operation(ctx context.Context, obj *types.SignerStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *signerChangeResolver) Transaction(ctx context.Context, obj *types.SignerStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // SignerAddress is the resolver for the signerAddress field. @@ -286,12 +286,12 @@ func (r *signerThresholdsChangeResolver) Account(ctx context.Context, obj *types // Operation is the resolver for the operation field. func (r *signerThresholdsChangeResolver) Operation(ctx context.Context, obj *types.SignerThresholdsStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *signerThresholdsChangeResolver) Transaction(ctx context.Context, obj *types.SignerThresholdsStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Thresholds is the resolver for the thresholds field. @@ -331,12 +331,12 @@ func (r *standardBalanceChangeResolver) Account(ctx context.Context, obj *types. // Operation is the resolver for the operation field. func (r *standardBalanceChangeResolver) Operation(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *standardBalanceChangeResolver) Transaction(ctx context.Context, obj *types.StandardBalanceStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. @@ -375,12 +375,12 @@ func (r *trustlineChangeResolver) Account(ctx context.Context, obj *types.Trustl // Operation is the resolver for the operation field. func (r *trustlineChangeResolver) Operation(ctx context.Context, obj *types.TrustlineStateChangeModel) (*types.Operation, error) { - return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeOperation(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // Transaction is the resolver for the transaction field. func (r *trustlineChangeResolver) Transaction(ctx context.Context, obj *types.TrustlineStateChangeModel) (*types.Transaction, error) { - return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID) + return r.resolveStateChangeTransaction(ctx, obj.ToID, obj.OperationID, obj.StateChangeID, obj.LedgerCreatedAt) } // TokenID is the resolver for the tokenId field. diff --git a/internal/serve/graphql/resolvers/statechange_resolvers_test.go b/internal/serve/graphql/resolvers/statechange_resolvers_test.go index a9304bc16..78cab3617 100644 --- a/internal/serve/graphql/resolvers/statechange_resolvers_test.go +++ b/internal/serve/graphql/resolvers/statechange_resolvers_test.go @@ -338,6 +338,7 @@ func TestStateChangeResolver_Operation(t *testing.T) { OperationID: opID, StateChangeID: 1, StateChangeCategory: types.StateChangeCategoryBalance, + LedgerCreatedAt: sharedTestLedgerCreatedAt, }, } @@ -394,6 +395,7 @@ func TestStateChangeResolver_Transaction(t *testing.T) { ToID: toid.New(1000, 1, 0).ToInt64(), StateChangeID: 1, StateChangeCategory: types.StateChangeCategoryBalance, + LedgerCreatedAt: sharedTestLedgerCreatedAt, }, } diff --git a/internal/serve/graphql/resolvers/test_utils.go b/internal/serve/graphql/resolvers/test_utils.go index 365fcf782..0aa36cf76 100644 --- a/internal/serve/graphql/resolvers/test_utils.go +++ b/internal/serve/graphql/resolvers/test_utils.go @@ -51,6 +51,11 @@ var sharedTestAccountAddress = keypair.MustRandom().Address() // sharedNonExistentAccountAddress is a valid Stellar address that doesn't exist in the test DB. var sharedNonExistentAccountAddress = keypair.MustRandom().Address() +// sharedTestLedgerCreatedAt is the ledger_created_at set on every row setupDB inserts. Data-layer +// batch lookups pin this partition column, so any test building a parent node (Transaction, +// Operation, StateChange) that references setupDB's fixture must carry this exact value. +var sharedTestLedgerCreatedAt time.Time + // Test transaction hashes used by setupDB (64-char hex strings for BYTEA storage) // These must match the pattern in setupDB: fmt.Sprintf("...487%x", i) where i = 0,1,2,3 var ( @@ -63,6 +68,7 @@ var ( func setupDB(ctx context.Context, t *testing.T, dbConnectionPool *pgxpool.Pool) { testLedger := int32(1000) now := time.Now() + sharedTestLedgerCreatedAt = now parentAccount := &types.Account{StellarAddress: types.AddressBytea(sharedTestAccountAddress)} txns := make([]*types.Transaction, 0, 4) ops := make([]*types.Operation, 0, 8) diff --git a/internal/serve/graphql/resolvers/transaction.resolvers.go b/internal/serve/graphql/resolvers/transaction.resolvers.go index 6ffafc610..38794b909 100644 --- a/internal/serve/graphql/resolvers/transaction.resolvers.go +++ b/internal/serve/graphql/resolvers/transaction.resolvers.go @@ -34,11 +34,12 @@ func (r *transactionResolver) Operations(ctx context.Context, obj *types.Transac loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.OperationColumnsKey{ - ToID: obj.ToID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.Cursor, - SortOrder: params.SortOrder, + ToID: obj.ToID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.Cursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } operations, err := loaders.OperationsByToIDLoader.Load(ctx, loaderKey) @@ -98,11 +99,12 @@ func (r *transactionResolver) StateChanges(ctx context.Context, obj *types.Trans loaders := ctx.Value(middleware.LoadersKey).(*dataloaders.Dataloaders) loaderKey := dataloaders.StateChangeColumnsKey{ - ToID: obj.ToID, - Columns: strings.Join(dbColumns, ", "), - Limit: &queryLimit, - Cursor: params.StateChangeCursor, - SortOrder: params.SortOrder, + ToID: obj.ToID, + Columns: strings.Join(dbColumns, ", "), + Limit: &queryLimit, + Cursor: params.StateChangeCursor, + SortOrder: params.SortOrder, + LedgerCreatedAt: obj.LedgerCreatedAt, } stateChanges, err := loaders.StateChangesByToIDLoader.Load(ctx, loaderKey) diff --git a/internal/serve/graphql/resolvers/transaction_resolvers_test.go b/internal/serve/graphql/resolvers/transaction_resolvers_test.go index 70e7d5ff8..00cfcd180 100644 --- a/internal/serve/graphql/resolvers/transaction_resolvers_test.go +++ b/internal/serve/graphql/resolvers/transaction_resolvers_test.go @@ -37,7 +37,7 @@ func TestTransactionResolver_Operations(t *testing.T) { }, }} // ToID=toid.New(1000, 1, 0) matches the test data setup in test_utils.go (testLedger=1000, i=0) - parentTx := &types.Transaction{Hash: types.HashBytea(testTxHash1), ToID: toid.New(1000, 1, 0).ToInt64()} + parentTx := &types.Transaction{Hash: types.HashBytea(testTxHash1), ToID: toid.New(1000, 1, 0).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} t.Run("success", func(t *testing.T) { loaders := dataloaders.NewDataloaders(resolver.models) @@ -219,7 +219,7 @@ func TestTransactionResolver_StateChanges(t *testing.T) { }, }, }} - parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64()} + parentTx := &types.Transaction{Hash: "1376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: toid.New(1000, 1, 0).ToInt64(), LedgerCreatedAt: sharedTestLedgerCreatedAt} nonExistentTx := &types.Transaction{Hash: "2376b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", ToID: 0} t.Run("success without pagination", func(t *testing.T) { diff --git a/internal/services/ingest_backfill.go b/internal/services/ingest_backfill.go index a632225b0..d98036b65 100644 --- a/internal/services/ingest_backfill.go +++ b/internal/services/ingest_backfill.go @@ -105,7 +105,11 @@ func (m *ingestService) calculateBackfillGaps(ctx context.Context, startLedger, return nil, fmt.Errorf("getting oldest ingest ledger: %w", err) } - currentGaps, err := m.models.IngestStore.GetLedgerGaps(ctx) + // Window at oldestIngestedLedger (guaranteed to be a present ledger_number, never inside a + // gap) rather than startLedger: this lets GetLedgerGaps's chunk-skipping-friendly WHERE + // clause bound the scan without duplicating the explicit "before history began" gap that + // case 2 below already appends, and without ever needing to synthesize a leading gap. + currentGaps, err := m.models.IngestStore.GetLedgerGaps(ctx, oldestIngestedLedger, endLedger) if err != nil { return nil, fmt.Errorf("calculating gaps in ledger range: %w", err) } From 8e01c7365b4d1bb9644d1f238635ab457ccc141c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 00:31:34 -0400 Subject: [PATCH 03/14] feat(data): NUMERIC amount columns; deterministic namespaced state_change IDs Numeric amounts, rates, and prices previously stored as TEXT (sac/sep41 balances, sep41 allowance amounts, the blend position/reserve/backstop/ emission/oracle columns) become unconstrained NUMERIC: writes cast once at the ingestion boundary, accumulation SQL operates natively (no more text->numeric->text round-trips per write), binary COPY encodes via pgtype.Numeric (malformed decoder output now fails at write time), and readers cast back to text so Go structs and the GraphQL String! contract stay byte-identical. state_change_id switches from crypto/rand to a deterministic ordinal per (to_id, operation_id) assigned at emission in slice order, namespaced per emitter (indexer base 0, SEP-41 1<<40, Blend 2<<40) because all three write state changes for the same operations in independently-committed transactions. Reprocessing a ledger now yields byte-identical rows, an accidental duplicate insert fails loudly on the PK instead of silently duplicating, and cursor ordering within an operation reflects emission order. --- internal/data/sac_balances.go | 17 ++++- internal/data/sep41/allowances.go | 8 ++- internal/data/sep41/balances.go | 15 ++-- internal/data/statechanges.go | 30 +++----- internal/data/statechanges_test.go | 65 +++++++++++++++++ .../migrations/2026-01-16.0-sac-balances.sql | 2 +- .../2026-04-17.0-sep41_balances.sql | 2 +- .../2026-04-17.1-sep41_allowances.sql | 2 +- internal/indexer/indexer.go | 12 +++- .../processors/contracts_test_utils.go | 4 +- .../processors/state_change_builder.go | 5 +- internal/indexer/types/types.go | 37 ++++++++++ internal/indexer/types/types_test.go | 72 +++++++++++++++++++ internal/services/sep41/processor.go | 4 ++ 14 files changed, 234 insertions(+), 41 deletions(-) diff --git a/internal/data/sac_balances.go b/internal/data/sac_balances.go index c9f936a2a..3ee2df725 100644 --- a/internal/data/sac_balances.go +++ b/internal/data/sac_balances.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgxpool" "github.com/stellar/wallet-backend/internal/db" @@ -66,7 +67,7 @@ func (m *SACBalanceModel) GetByAccount(ctx context.Context, accountAddress strin } query := ` - SELECT asb.account_id, asb.contract_id, asb.balance, asb.is_authorized, + SELECT asb.account_id, asb.contract_id, asb.balance::text, asb.is_authorized, asb.is_clawback_enabled, asb.last_modified_ledger, ct.contract_id AS token_id, ct.code, ct.issuer, ct.decimals FROM sac_balances asb @@ -148,7 +149,9 @@ func (m *SACBalanceModel) BatchUpsert(ctx context.Context, dbTx pgx.Tx, upserts INSERT INTO sac_balances ( account_id, contract_id, balance, is_authorized, is_clawback_enabled, last_modified_ledger ) - SELECT * FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::boolean[], $5::boolean[], $6::int[]) + SELECT u.account_id, u.contract_id, u.balance::numeric, u.is_authorized, u.is_clawback_enabled, u.last_modified_ledger + FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::boolean[], $5::boolean[], $6::int[]) + AS u(account_id, contract_id, balance, is_authorized, is_clawback_enabled, last_modified_ledger) ON CONFLICT (account_id, contract_id) DO UPDATE SET balance = EXCLUDED.balance, is_authorized = EXCLUDED.is_authorized, @@ -224,10 +227,18 @@ func (m *SACBalanceModel) BatchCopy(ctx context.Context, dbTx pgx.Tx, balances [ if err != nil { return nil, fmt.Errorf("converting account address to bytes: %w", err) } + // CopyFrom uses the binary protocol, which requires a pgtype value for + // NUMERIC rather than a bare string. Scan also validates the balance is + // well-formed decimal text, catching malformed decoder output at the + // write boundary instead of storing it silently. + var balanceNumeric pgtype.Numeric + if err := balanceNumeric.Scan(bal.Balance); err != nil { + return nil, fmt.Errorf("parsing SAC balance %q for account %s: %w", bal.Balance, bal.AccountID, err) + } return []any{ accountIDBytes, bal.ContractID, - bal.Balance, + balanceNumeric, bal.IsAuthorized, bal.IsClawbackEnabled, bal.LedgerNumber, diff --git a/internal/data/sep41/allowances.go b/internal/data/sep41/allowances.go index d2f9db795..d4c55ac98 100644 --- a/internal/data/sep41/allowances.go +++ b/internal/data/sep41/allowances.go @@ -82,7 +82,7 @@ func (m *AllowanceModel) GetByOwner(ctx context.Context, ownerAddress string, li query := ` SELECT - a.spender_id, a.contract_id, a.amount, a.expiration_ledger, a.last_modified_ledger, + a.spender_id, a.contract_id, a.amount::text, a.expiration_ledger, a.last_modified_ledger, ct.contract_id AS token_id FROM sep41_allowances a INNER JOIN contract_tokens ct ON ct.id = a.contract_id @@ -196,10 +196,12 @@ func (m *AllowanceModel) BatchUpsert(ctx context.Context, dbTx pgx.Tx, upserts [ owner_id, spender_id, contract_id, amount, expiration_ledger, last_modified_ledger ) - SELECT * FROM UNNEST( + SELECT u.owner_id, u.spender_id, u.contract_id, + u.amount::numeric, u.expiration_ledger, u.last_modified_ledger + FROM UNNEST( $1::bytea[], $2::bytea[], $3::uuid[], $4::text[], $5::integer[], $6::integer[] - ) + ) AS u(owner_id, spender_id, contract_id, amount, expiration_ledger, last_modified_ledger) ON CONFLICT (owner_id, spender_id, contract_id) DO UPDATE SET amount = EXCLUDED.amount, expiration_ledger = EXCLUDED.expiration_ledger, diff --git a/internal/data/sep41/balances.go b/internal/data/sep41/balances.go index de3fe67ba..e89b10684 100644 --- a/internal/data/sep41/balances.go +++ b/internal/data/sep41/balances.go @@ -73,7 +73,7 @@ func (m *BalanceModel) GetByAccount(ctx context.Context, accountAddress string, query := ` SELECT - b.contract_id, b.balance, b.last_modified_ledger, + b.contract_id, b.balance::text, b.last_modified_ledger, ct.contract_id AS token_id, ct.name, ct.symbol, ct.decimals FROM sep41_balances b INNER JOIN contract_tokens ct ON ct.id = b.contract_id @@ -171,13 +171,16 @@ func (m *BalanceModel) BatchApplyDeltas(ctx context.Context, dbTx pgx.Tx, deltas ledgers[i] = int32(d.LedgerNumber) } - // Sum in SQL: existing + delta. The TEXT columns are cast to numeric for arithmetic - // and back to text for storage so the column type stays TEXT (preserves full i128 range). + // Sum in SQL: existing + delta. balance is stored as NUMERIC so this is native + // arithmetic; the delta arrives as text (i128 decimal string) and is cast once at + // the boundary, since Postgres has no implicit text->numeric cast. const upsertQuery = ` INSERT INTO sep41_balances (account_id, contract_id, balance, last_modified_ledger) - SELECT * FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::integer[]) + SELECT u.account_id, u.contract_id, u.balance::numeric, u.last_modified_ledger + FROM UNNEST($1::bytea[], $2::uuid[], $3::text[], $4::integer[]) + AS u(account_id, contract_id, balance, last_modified_ledger) ON CONFLICT (account_id, contract_id) DO UPDATE SET - balance = (sep41_balances.balance::numeric + EXCLUDED.balance::numeric)::text, + balance = sep41_balances.balance + EXCLUDED.balance, last_modified_ledger = EXCLUDED.last_modified_ledger` if _, err := dbTx.Exec(ctx, upsertQuery, accountIDs, contractIDs, balances, ledgers); err != nil { m.Metrics.QueryErrors.WithLabelValues("BatchApplyDeltas", "sep41_balances", utils.GetDBErrorType(err)).Inc() @@ -186,7 +189,7 @@ func (m *BalanceModel) BatchApplyDeltas(ctx context.Context, dbTx pgx.Tx, deltas const deleteZeroesQuery = ` DELETE FROM sep41_balances - WHERE balance::numeric = 0 + WHERE balance = 0 AND (account_id, contract_id) IN ( SELECT * FROM UNNEST($1::bytea[], $2::uuid[]) )` diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 144b2a169..1ba38401d 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -2,8 +2,6 @@ package data import ( "context" - "crypto/rand" - "encoding/binary" "fmt" "strings" "time" @@ -190,9 +188,16 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in // BatchCopy inserts state changes using pgx's binary COPY protocol. // Uses native pgtype types for optimal performance (see https://github.com/jackc/pgx/issues/763). // +// Each StateChange must already carry its final, deterministic StateChangeID +// (see types.AssignStateChangeOrdinals) — BatchCopy writes it as-is and never +// generates one. +// // IMPORTANT: BatchCopy will FAIL if any duplicate records exist. The PostgreSQL COPY // protocol does not support conflict handling. Callers must ensure no duplicates exist -// before calling this method, or handle the unique constraint violation error appropriately. +// before calling this method, or handle the unique constraint violation error +// appropriately. Because state_change_id is now deterministic, this is also the +// idempotency backstop: re-copying an already-committed ledger's rows collides on the +// primary key and fails loudly instead of inserting duplicates. func (m *StateChangeModel) BatchCopy( ctx context.Context, pgxTx pgx.Tx, @@ -221,13 +226,6 @@ func (m *StateChangeModel) BatchCopy( pgx.CopyFromSlice(len(stateChanges), func(i int) ([]any, error) { sc := stateChanges[i] - // Generate a fresh random ID at insertion time so retries on PK collision - // produce new IDs automatically. - stateChangeID, err := generateStateChangeID() - if err != nil { - return nil, err - } - // Convert account_id to BYTEA (required field) accountBytes, err := sc.AccountID.Value() if err != nil { @@ -270,7 +268,7 @@ func (m *StateChangeModel) BatchCopy( return []any{ pgtype.Int8{Int64: sc.ToID, Valid: true}, - pgtype.Int8{Int64: stateChangeID, Valid: true}, + pgtype.Int8{Int64: sc.StateChangeID, Valid: true}, pgtype.Text{String: string(sc.StateChangeCategory), Valid: true}, pgtypeTextFromReason(sc.StateChangeReason), pgtype.Timestamptz{Time: sc.LedgerCreatedAt, Valid: true}, @@ -326,16 +324,6 @@ func (m *StateChangeModel) BatchCopy( return len(stateChanges), nil } -// generateStateChangeID produces a random positive int64 from crypto/rand. -// Full 63 bits of entropy, masked to ensure a positive value. -func generateStateChangeID() (int64, error) { - var buf [8]byte - if _, err := rand.Read(buf[:]); err != nil { - return 0, fmt.Errorf("generating state change ID: %w", err) - } - return int64(binary.BigEndian.Uint64(buf[:]) & 0x7FFFFFFFFFFFFFFF), nil -} - // BatchGetByToID gets state changes for a single transaction with pagination support, pinned to // the parent transaction's ledger_created_at for partition-column chunk exclusion. func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 1d83c6c36..99a79b80b 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -215,6 +215,71 @@ func TestStateChangeModel_BatchCopy(t *testing.T) { } } +// TestStateChangeModel_BatchCopy_DuplicateFailsOnPK verifies the SQL-06 +// idempotency backstop: since state_change_id is now a deterministic ordinal +// (assigned by types.AssignStateChangeOrdinals before BatchCopy is called, +// not randomly generated inside BatchCopy), re-copying an already-committed +// ledger's rows collides on the state_changes primary key and fails loudly +// instead of silently inserting duplicate rows. +func TestStateChangeModel_BatchCopy_DuplicateFailsOnPK(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + now := time.Now() + kp := keypair.MustRandom() + + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, "f176b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4877", int64(1), 100, "TransactionResultCodeTxSuccess", uint32(1), now, false) + require.NoError(t, err) + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &StateChangeModel{DB: dbConnectionPool, Metrics: dbMetrics} + + conn, err := pgx.Connect(ctx, dbt.DSN) + require.NoError(t, err) + defer conn.Close(ctx) + + // Same (to_id, operation_id, state_change_id, ledger_created_at) as a + // deterministic emitter would recompute for the same ledger on a re-ingest. + sc := types.StateChange{ + ToID: 1, + StateChangeID: 1, + StateChangeCategory: types.StateChangeCategoryBalance, + StateChangeReason: types.StateChangeReasonAdd, + LedgerCreatedAt: now, + LedgerNumber: 1, + AccountID: types.AddressBytea(kp.Address()), + OperationID: 123, + } + + pgxTx, err := conn.Begin(ctx) + require.NoError(t, err) + gotCount, err := m.BatchCopy(ctx, pgxTx, []types.StateChange{sc}) + require.NoError(t, err) + require.NoError(t, pgxTx.Commit(ctx)) + assert.Equal(t, 1, gotCount) + + // Re-copying the identical row (simulating a re-ingest of the same ledger) + // must fail on the primary key rather than inserting a duplicate. + pgxTx2, err := conn.Begin(ctx) + require.NoError(t, err) + _, err = m.BatchCopy(ctx, pgxTx2, []types.StateChange{sc}) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate key value violates unique constraint") + pgxTx2.Rollback(ctx) + + dbInsertedIDs, err := db.QueryMany[string](ctx, dbConnectionPool, "SELECT CONCAT(to_id, '-', operation_id, '-', state_change_id) FROM state_changes") + require.NoError(t, err) + assert.Len(t, dbInsertedIDs, 1, "the failed re-copy must not have left duplicate rows") +} + func TestStateChangeModel_BatchGetByAccountAddress(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() diff --git a/internal/db/migrations/2026-01-16.0-sac-balances.sql b/internal/db/migrations/2026-01-16.0-sac-balances.sql index a62dd0ff8..8fdf1bd14 100644 --- a/internal/db/migrations/2026-01-16.0-sac-balances.sql +++ b/internal/db/migrations/2026-01-16.0-sac-balances.sql @@ -9,7 +9,7 @@ CREATE TABLE sac_balances ( account_id BYTEA NOT NULL, contract_id UUID NOT NULL, - balance TEXT NOT NULL DEFAULT '0', + balance NUMERIC NOT NULL DEFAULT 0, is_authorized BOOLEAN NOT NULL DEFAULT true, is_clawback_enabled BOOLEAN NOT NULL DEFAULT false, last_modified_ledger INTEGER NOT NULL DEFAULT 0, diff --git a/internal/db/migrations/2026-04-17.0-sep41_balances.sql b/internal/db/migrations/2026-04-17.0-sep41_balances.sql index b49b80d09..866c31c36 100644 --- a/internal/db/migrations/2026-04-17.0-sep41_balances.sql +++ b/internal/db/migrations/2026-04-17.0-sep41_balances.sql @@ -7,7 +7,7 @@ CREATE TABLE sep41_balances ( account_id BYTEA NOT NULL, contract_id UUID NOT NULL, - balance TEXT NOT NULL DEFAULT '0', + balance NUMERIC NOT NULL DEFAULT 0, last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (account_id, contract_id), CONSTRAINT fk_sep41_contract_token diff --git a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql index 2194f82d7..9465524e5 100644 --- a/internal/db/migrations/2026-04-17.1-sep41_allowances.sql +++ b/internal/db/migrations/2026-04-17.1-sep41_allowances.sql @@ -7,7 +7,7 @@ CREATE TABLE sep41_allowances ( owner_id BYTEA NOT NULL, spender_id BYTEA NOT NULL, contract_id UUID NOT NULL, - amount TEXT NOT NULL DEFAULT '0', + amount NUMERIC NOT NULL DEFAULT 0, expiration_ledger INTEGER NOT NULL DEFAULT 0, last_modified_ledger INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (owner_id, spender_id, contract_id), diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 6f67a7146..6a7a77feb 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -342,13 +342,21 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa } } - // Insert state changes + // Drop state changes with no account to associate with, then assign each + // retained one a deterministic state_change_id: an ordinal numbered 1..N in + // emission order within its (to_id, operation_id) group. Filtering first + // keeps the ordinals gap-free for what's actually persisted. + retainedStateChanges := stateChanges[:0] for _, stateChange := range stateChanges { - // Skip empty state changes (no account to associate with) if stateChange.AccountID == "" { continue } + retainedStateChanges = append(retainedStateChanges, stateChange) + } + types.AssignStateChangeOrdinals(retainedStateChanges, types.StateChangeOrdinalBaseIndexer) + // Insert state changes + for _, stateChange := range retainedStateChanges { // Get the correct operation for this state change var operation types.Operation if stateChange.OperationID != 0 { diff --git a/internal/indexer/processors/contracts_test_utils.go b/internal/indexer/processors/contracts_test_utils.go index ad62e8058..d89b97465 100644 --- a/internal/indexer/processors/contracts_test_utils.go +++ b/internal/indexer/processors/contracts_test_utils.go @@ -152,7 +152,9 @@ func makeFeeBumpOp(feeBumpSourceAccount string, baseOp *TransactionOperationWrap } // assertStateChangeEqual compares two state changes and fails if they are not equal. -// It normalizes IngestedAt (non-deterministic wall clock) and StateChangeID (random). +// It normalizes IngestedAt (non-deterministic wall clock). StateChangeID is left at its +// Build()-time zero value on both sides — ProcessOperation output is compared before the +// emitter assigns ordinals (see types.AssignStateChangeOrdinals), so no normalization is needed. func assertStateChangeEqual(t *testing.T, want types.StateChange, got types.StateChange) { t.Helper() diff --git a/internal/indexer/processors/state_change_builder.go b/internal/indexer/processors/state_change_builder.go index 660ed022e..7c7cbc0db 100644 --- a/internal/indexer/processors/state_change_builder.go +++ b/internal/indexer/processors/state_change_builder.go @@ -178,8 +178,9 @@ func (b *StateChangeBuilder) WithSponsoredData(dataName string) *StateChangeBuil } // Build returns the constructed state change. -// Note: StateChangeID is assigned at insertion time by BatchCopy, not here. -// This ensures retries on PK collision generate fresh IDs automatically. +// Note: StateChangeID is left zero here — the emitter assigns it afterward via +// types.AssignStateChangeOrdinals, numbering each transaction's or window's +// state changes in deterministic emission order before persisting. func (b *StateChangeBuilder) Build() types.StateChange { return b.base } diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index be64f677a..151617de2 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -687,6 +687,43 @@ type StateChangeCursorGetter interface { GetCursor() StateChangeCursor } +// State change ID namespace registry. +// +// state_changes rows are written by more than one independent emitter: the +// main indexer (effects, token transfers, etc.) and protocol processors +// (SEP-41, Blend) that persist their own history inside a separately +// CAS-guarded transaction. Two emitters can legitimately target the same +// (to_id, operation_id) — e.g. a SEP-41 or Blend contract invocation is one +// operation the main indexer also sees. Each emitter numbers its own state +// changes 1..N in deterministic emission order within an (to_id, operation_id) +// group (see AssignStateChangeOrdinals) and adds its base below, so the +// resulting state_change_id values can never collide across emitters even +// when they share an operation. Bases are spaced far apart (int64 has ample +// room) and must never change once rows exist with them. +const ( + StateChangeOrdinalBaseIndexer int64 = 0 + StateChangeOrdinalBaseSEP41 int64 = 1 << 40 + StateChangeOrdinalBaseBlend int64 = 2 << 40 +) + +// AssignStateChangeOrdinals assigns each state change in changes a +// deterministic state_change_id: an ordinal numbered 1..N in slice order +// within each distinct OperationID group, plus base. Processing the same +// input twice (e.g. a re-ingested ledger) yields byte-identical IDs, so a +// duplicate BatchCopy fails loudly on the state_changes primary key instead of +// inserting duplicate rows. +// +// Callers must pass the final, filtered slice — the one actually handed to +// BatchCopy — so ordinals come out contiguous (1..N, no gaps) per group. +func AssignStateChangeOrdinals(changes []StateChange, base int64) { + next := make(map[int64]int64, len(changes)) + for i := range changes { + opID := changes[i].OperationID + next[opID]++ + changes[i].StateChangeID = base + next[opID] + } +} + type NullableJSONB map[string]any var _ sql.Scanner = (*NullableJSONB)(nil) diff --git a/internal/indexer/types/types_test.go b/internal/indexer/types/types_test.go index c8422f890..18919e6f0 100644 --- a/internal/indexer/types/types_test.go +++ b/internal/indexer/types/types_test.go @@ -379,3 +379,75 @@ func TestHashBytea_String(t *testing.T) { }) } } + +// buildOrdinalTestInput returns a fixed slice of state changes spanning two +// operations, interleaved in emission order: op 10 gets three, op 20 gets two. +func buildOrdinalTestInput() []StateChange { + return []StateChange{ + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 20, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 10, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 1, OperationID: 20, StateChangeCategory: StateChangeCategoryBalance}, + } +} + +func TestAssignStateChangeOrdinals_OrdinalsPerOperationGroup(t *testing.T) { + changes := buildOrdinalTestInput() + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseIndexer) + + // op 10's three changes are numbered 1..3 in the order they appear; op 20's + // two changes are numbered 1..2 independently, interleaved with op 10's. + assert.Equal(t, []int64{1, 1, 2, 3, 2}, []int64{ + changes[0].StateChangeID, changes[1].StateChangeID, changes[2].StateChangeID, + changes[3].StateChangeID, changes[4].StateChangeID, + }) +} + +func TestAssignStateChangeOrdinals_NamespaceBaseIsAdded(t *testing.T) { + changes := buildOrdinalTestInput() + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseBlend) + + for _, sc := range changes { + assert.GreaterOrEqual(t, sc.StateChangeID, StateChangeOrdinalBaseBlend) + assert.Less(t, sc.StateChangeID, StateChangeOrdinalBaseBlend+1000) + } + // Same relative ordinals as the unnamespaced case, offset by the base. + assert.Equal(t, StateChangeOrdinalBaseBlend+1, changes[0].StateChangeID) + assert.Equal(t, StateChangeOrdinalBaseBlend+3, changes[3].StateChangeID) +} + +func TestAssignStateChangeOrdinals_Deterministic(t *testing.T) { + first := buildOrdinalTestInput() + second := buildOrdinalTestInput() + + AssignStateChangeOrdinals(first, StateChangeOrdinalBaseSEP41) + AssignStateChangeOrdinals(second, StateChangeOrdinalBaseSEP41) + + // Processing the same input twice (e.g. a re-ingested ledger) must produce + // byte-identical state_change_ids, not just equal-length results. + require.Len(t, second, len(first)) + for i := range first { + assert.Equal(t, first[i].StateChangeID, second[i].StateChangeID, "index %d", i) + } +} + +func TestAssignStateChangeOrdinals_DifferentEmittersNeverCollide(t *testing.T) { + indexerChanges := []StateChange{{ToID: 1, OperationID: 42}} + sep41Changes := []StateChange{{ToID: 1, OperationID: 42}} + blendChanges := []StateChange{{ToID: 1, OperationID: 42}} + + AssignStateChangeOrdinals(indexerChanges, StateChangeOrdinalBaseIndexer) + AssignStateChangeOrdinals(sep41Changes, StateChangeOrdinalBaseSEP41) + AssignStateChangeOrdinals(blendChanges, StateChangeOrdinalBaseBlend) + + // Same (to_id, operation_id) from three independent emitters: without + // namespacing these would all be state_change_id=1 and collide on the + // state_changes primary key. + ids := map[int64]bool{ + indexerChanges[0].StateChangeID: true, + sep41Changes[0].StateChangeID: true, + blendChanges[0].StateChangeID: true, + } + assert.Len(t, ids, 3, "each emitter's ID for the same operation must be distinct") +} diff --git a/internal/services/sep41/processor.go b/internal/services/sep41/processor.go index 25faa4595..c422be22e 100644 --- a/internal/services/sep41/processor.go +++ b/internal/services/sep41/processor.go @@ -317,6 +317,10 @@ func (p *processor) PersistHistory(ctx context.Context, dbTx pgx.Tx) error { if len(p.stagedStateChanges) == 0 { return nil } + // Assign deterministic state_change_ids in emission order, namespaced under + // the SEP-41 base so they can never collide with the main indexer's or + // Blend's IDs for the same operation (see types.AssignStateChangeOrdinals). + types.AssignStateChangeOrdinals(p.stagedStateChanges, types.StateChangeOrdinalBaseSEP41) if _, err := p.stateChanges.BatchCopy(ctx, dbTx, p.stagedStateChanges); err != nil { return fmt.Errorf("persisting %d SEP-41 state changes for ledger %d: %w", len(p.stagedStateChanges), p.ledgerNumber, err) From bdce3ea1b4c723e29b05a52e5c8e2a6c3496b495 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 08:58:05 -0400 Subject: [PATCH 04/14] feat(db): chunk skipping on transactions.ledger_number for windowed gap detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit EXPLAIN verification of the windowed GetLedgerGaps showed the window was bounded only by the columnstore's batch min/max metadata — ledger_number carried no chunk-skipping ranges, so no chunks were excluded at plan time and every chunk paid at least a metadata pass. ledger_number rises monotonically with the partition column, making it an ideal skipping column: enabling it gives ledger-windowed queries plan-time chunk exclusion (ranges are recorded as chunks compress). The GetLedgerGaps doc comment now describes both bounding mechanisms accurately. --- internal/data/ingest_store.go | 8 +++++--- internal/db/migrations/2025-06-10.2-transactions.sql | 4 ++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/data/ingest_store.go b/internal/data/ingest_store.go index e64467538..642b924c4 100644 --- a/internal/data/ingest_store.go +++ b/internal/data/ingest_store.go @@ -170,9 +170,11 @@ func (m *IngestStoreModel) UpdateMin(ctx context.Context, dbTx pgx.Tx, cursorNam } // GetLedgerGaps returns gaps between consecutive present ledger_number values within -// [startLedger, endLedger]. Windowing the DISTINCT scan on ledger_number (which has -// chunk-skipping stats enabled) makes the cost proportional to the window instead of a full -// hypertable scan. +// [startLedger, endLedger]. The window bounds the cost two ways: chunks whose recorded +// ledger_number range (enable_chunk_skipping on transactions.ledger_number; ranges are +// captured when a chunk is compressed) doesn't overlap the window are excluded at plan +// time, and any remaining out-of-window batches are dropped by the columnstore's batch +// min/max metadata without decompression. // // Edge semantics: callers must pass a startLedger that is itself a present ledger_number (e.g. // the oldest ingested ledger) — the left edge is NOT synthesized. If startLedger fell strictly diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index c51dd9268..a7b43ce48 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -20,6 +20,10 @@ CREATE TABLE transactions ( ); SELECT enable_chunk_skipping('transactions', 'to_id'); +-- ledger_number rises monotonically with the partition column, so per-chunk min/max +-- ranges let ledger_number-windowed queries (e.g. backfill gap detection) exclude +-- non-overlapping chunks at plan time. +SELECT enable_chunk_skipping('transactions', 'ledger_number'); -- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB -- auto-creates on the hypertable's partition column. The root GraphQL connection From d1ff8d85d5d0c5bf9c7e93ccf60334e44ae85cbd Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 12:45:08 -0400 Subject: [PATCH 05/14] perf(db): consolidate hypertable indexes into query-ordered primary keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three hypertables carried a secondary index over the same column set as their primary key in a different order, and the two _accounts tables kept TimescaleDB's auto-created single-column time index that no query path uses. The PKs now use the column order their consumers need — the _accounts tables as (account_id, ledger_created_at, id) so one index serves both uniqueness and the account-history walk (B-tree scans run backward for DESC pages), state_changes as (ledger_created_at, to_id, operation_id, state_change_id) matching the root connection sort and decomposed cursor — and the redundant secondaries and dead time indexes are gone. Every remaining index has a nameable consumer. The uncompressed hot-day index working set drops from ~14GB to ~9.5GB and the two highest-insert- rate tables halve their random-B-tree maintenance per row. Verified by an EXPLAIN sweep of all sixteen consumer queries against the new layout on a seeded multi-chunk database: every account walk and root page is an index (only) scan with no sort or seq scan; reverse-lookup paths unchanged. The operations doc gains the hypertable index-usage measurement caveat: parent pg_stat_user_indexes always reads zero — usage accrues on chunk indexes in _timescaledb_internal and must be aggregated from there. --- docs/operations.md | 14 ++++++++++++++ .../db/migrations/2025-06-10.2-transactions.sql | 13 +++++++++++-- .../db/migrations/2025-06-10.3-operations.sql | 15 +++++++++++++-- .../db/migrations/2025-06-10.4-statechanges.sql | 16 ++++++++-------- 4 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 docs/operations.md diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 000000000..bd65403cb --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,14 @@ +# Operations + +**Hypertable index usage:** `pg_stat_user_indexes` on the `public` schema always reports `idx_scan = 0` for hypertable indexes — scans accrue on the per-chunk indexes in `_timescaledb_internal`, and the parent's index entry is only a template. To measure real usage (e.g. for an unused-index audit), aggregate chunk-index stats: + +```sql +SELECT ht.table_name AS hypertable, regexp_replace(sui.indexrelname, '^_hyper_\d+_\d+_chunk_', '') AS index_name, + sum(sui.idx_scan) AS scans +FROM pg_stat_user_indexes sui +JOIN _timescaledb_catalog.chunk c ON c.schema_name = sui.schemaname AND c.table_name = sui.relname +JOIN _timescaledb_catalog.hypertable ht ON ht.id = c.hypertable_id +GROUP BY 1, 2 ORDER BY 1, 3 DESC; +``` + +Every index in the schema must have a nameable consumer query; secondary indexes duplicating a primary key's column set are not kept (the PK's column order is chosen to serve its consumers directly — B-tree scans run forward or backward, so ASC/DESC variants of the same key are one index, not two). diff --git a/internal/db/migrations/2025-06-10.2-transactions.sql b/internal/db/migrations/2025-06-10.2-transactions.sql index a7b43ce48..af0bbd882 100644 --- a/internal/db/migrations/2025-06-10.2-transactions.sql +++ b/internal/db/migrations/2025-06-10.2-transactions.sql @@ -41,7 +41,12 @@ CREATE TABLE transactions_accounts ( tx_to_id BIGINT NOT NULL, account_id BYTEA NOT NULL, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, tx_to_id, ledger_created_at) + -- Column order matches BatchGetByAccountAddress's access shape: account_id is pinned by + -- equality, then the walk sorts by (ledger_created_at, tx_to_id). With account_id fixed, + -- the planner serves that sort via an index (only) scan over this PK -- forward or + -- backward -- so the dedicated (account_id, ledger_created_at DESC, tx_to_id DESC) index + -- once needed alongside it is redundant. + PRIMARY KEY (account_id, ledger_created_at, tx_to_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -52,8 +57,12 @@ CREATE TABLE transactions_accounts ( SELECT enable_chunk_skipping('transactions_accounts', 'tx_to_id'); +-- TimescaleDB's default single-column index on the partition column. Retention drops chunks +-- by range metadata, and no query path filters this table by bare ledger_created_at, so it +-- has zero consumers. +DROP INDEX IF EXISTS transactions_accounts_ledger_created_at_idx; + CREATE INDEX idx_transactions_accounts_tx_to_id ON transactions_accounts(tx_to_id); -CREATE INDEX idx_transactions_accounts_account_id ON transactions_accounts(account_id, ledger_created_at DESC, tx_to_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.3-operations.sql b/internal/db/migrations/2025-06-10.3-operations.sql index 7f4804068..9b0abd0df 100644 --- a/internal/db/migrations/2025-06-10.3-operations.sql +++ b/internal/db/migrations/2025-06-10.3-operations.sql @@ -48,7 +48,14 @@ CREATE TABLE operations_accounts ( operation_id BIGINT NOT NULL, account_id BYTEA NOT NULL, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (account_id, operation_id, ledger_created_at) + -- Column order (account_id, ledger_created_at, operation_id) serves both consumers as a + -- single index: BatchGetByAccountAddress pins account_id by equality and sorts by + -- (ledger_created_at, operation_id), which the planner walks as an index (only) scan over + -- this PK -- forward or backward; BatchGetAccountOperationsByToIDs probes equality on + -- account_id, equality on ledger_created_at, and a range on operation_id, which fits this + -- key as a perfect prefix. The dedicated (account_id, ledger_created_at DESC, operation_id + -- DESC) index once needed for the former is now redundant. + PRIMARY KEY (account_id, ledger_created_at, operation_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -59,8 +66,12 @@ CREATE TABLE operations_accounts ( SELECT enable_chunk_skipping('operations_accounts', 'operation_id'); +-- TimescaleDB's default single-column index on the partition column. Retention drops chunks +-- by range metadata, and no query path filters this table by bare ledger_created_at, so it +-- has zero consumers. +DROP INDEX IF EXISTS operations_accounts_ledger_created_at_idx; + CREATE INDEX idx_operations_accounts_operation_id ON operations_accounts(operation_id); -CREATE INDEX idx_operations_accounts_account_id ON operations_accounts(account_id, ledger_created_at DESC, operation_id DESC); -- +migrate Down diff --git a/internal/db/migrations/2025-06-10.4-statechanges.sql b/internal/db/migrations/2025-06-10.4-statechanges.sql index 67cf2a7a9..2d7e426e2 100644 --- a/internal/db/migrations/2025-06-10.4-statechanges.sql +++ b/internal/db/migrations/2025-06-10.4-statechanges.sql @@ -45,7 +45,12 @@ CREATE TABLE state_changes ( key_value JSONB, to_muxed_id TEXT, ledger_created_at TIMESTAMPTZ NOT NULL, - PRIMARY KEY (to_id, operation_id, state_change_id, ledger_created_at) + -- The root GraphQL connection sorts by (ledger_created_at DESC, to_id DESC, operation_id + -- DESC, state_change_id DESC). This PK's column order matches that sort key directly, so + -- a top-N first page is served by scanning the PK itself -- forward or backward, no + -- heapsort -- which also makes it this table's replacement for the default single-column + -- index TimescaleDB would otherwise auto-create on the partition column. + PRIMARY KEY (ledger_created_at, to_id, operation_id, state_change_id) ) WITH ( tsdb.hypertable, tsdb.partition_column = 'ledger_created_at', @@ -58,14 +63,9 @@ CREATE TABLE state_changes ( SELECT enable_chunk_skipping('state_changes', 'to_id'); SELECT enable_chunk_skipping('state_changes', 'operation_id'); --- Replaces the default single-column (ledger_created_at DESC) index TimescaleDB --- auto-creates on the hypertable's partition column. The root GraphQL connection --- sorts by (ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC), --- which the default index can't serve for a top-N first page (it still needs a --- heapsort within each timestamp). This composite index matches the API sort key --- directly and makes the auto-created one redundant. +-- TimescaleDB's default single-column index on the partition column; the reordered PK above +-- is now a superset of it (ledger_created_at is its leading column), so it's redundant. DROP INDEX IF EXISTS state_changes_ledger_created_at_idx; -CREATE INDEX idx_state_changes_created_at_sort ON state_changes (ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); CREATE INDEX idx_state_changes_operation_id ON state_changes(operation_id); CREATE INDEX idx_state_changes_account_category ON state_changes(account_id, state_change_category, state_change_reason, ledger_created_at DESC, to_id DESC, operation_id DESC, state_change_id DESC); From 57a3ca64067136549e3aeb7bac093108d80f54a7 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 16 Jul 2026 13:51:35 -0400 Subject: [PATCH 06/14] chore(deps): pin TimescaleDB 2.28.2 --- .github/workflows/build-cnpg-timescaledb-dev.yml | 4 ++-- .github/workflows/build-cnpg-timescaledb-stg.yml | 4 ++-- .github/workflows/go.yaml | 2 +- Dockerfile-timescale-cnpg | 4 ++-- docker-compose.yaml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-cnpg-timescaledb-dev.yml b/.github/workflows/build-cnpg-timescaledb-dev.yml index d20db6e21..d795c995f 100644 --- a/.github/workflows/build-cnpg-timescaledb-dev.yml +++ b/.github/workflows/build-cnpg-timescaledb-dev.yml @@ -8,8 +8,8 @@ on: default: '17' required: true tsdb_version: - description: TimescaleDB version (semver, e.g. 2.25.0) - default: '2.25.0' + description: TimescaleDB version (semver, e.g. 2.28.2) + default: '2.28.2' required: true permissions: diff --git a/.github/workflows/build-cnpg-timescaledb-stg.yml b/.github/workflows/build-cnpg-timescaledb-stg.yml index cef0d001e..6450c3a6d 100644 --- a/.github/workflows/build-cnpg-timescaledb-stg.yml +++ b/.github/workflows/build-cnpg-timescaledb-stg.yml @@ -8,8 +8,8 @@ on: default: '17' required: true tsdb_version: - description: TimescaleDB version (semver, e.g. 2.25.0) - default: '2.25.0' + description: TimescaleDB version (semver, e.g. 2.28.2) + default: '2.28.2' required: true permissions: diff --git a/.github/workflows/go.yaml b/.github/workflows/go.yaml index e659730e3..d762a6569 100644 --- a/.github/workflows/go.yaml +++ b/.github/workflows/go.yaml @@ -82,7 +82,7 @@ jobs: runs-on: ubuntu-latest services: postgres: - image: timescale/timescaledb:2.25.0-pg17 + image: timescale/timescaledb:2.28.2-pg17 env: POSTGRES_USER: postgres POSTGRES_DB: postgres diff --git a/Dockerfile-timescale-cnpg b/Dockerfile-timescale-cnpg index 5ca5b2d87..51cbf80e4 100644 --- a/Dockerfile-timescale-cnpg +++ b/Dockerfile-timescale-cnpg @@ -5,7 +5,7 @@ # TimescaleDB from the apt repo and layers it onto the CNPG base image. # # Usage: -# docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.25.0 -t cnpg-timescaledb . +# docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.28.2 -t cnpg-timescaledb . ARG PG_MAJOR=17 @@ -15,7 +15,7 @@ ARG PG_MAJOR=17 FROM ghcr.io/cloudnative-pg/postgresql:${PG_MAJOR}-bookworm AS timescaledb-builder ARG PG_MAJOR -ARG TSDB_VERSION=2.25.0 +ARG TSDB_VERSION=2.28.2 USER root diff --git a/docker-compose.yaml b/docker-compose.yaml index bf402fe09..23ce273c8 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: db: container_name: db - image: timescale/timescaledb:2.25.0-pg17 + image: timescale/timescaledb:2.28.2-pg17 command: ["postgres", "-c", "timescaledb.enable_chunk_skipping=on", "-c", "timescaledb.enable_sparse_index_bloom=on"] healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"] From a214a2d82a158f9c16282f9add79a1c9ebbe6298 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 16 Jul 2026 15:52:32 -0400 Subject: [PATCH 07/14] feat(data): per-processor sub-namespaces for state_change IDs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Indexer state_change_ids previously depended on the order emitting processors are registered and invoked in: all streams were merged into one slice and numbered together, so inserting or reordering a processor would shift every later processor's ordinals for operations where both emit. Replaying an already-persisted ledger with such a binary would produce different IDs and silently bypass the duplicate-insert PK backstop. Each emitting processor now numbers its own emissions independently and adds its slot from the sub-namespace registry (k<<28 inside the indexer's 1<<40-wide namespace: token transfers 0, effects 1, contract deploys 2, SAC events 3), declared via StateChangeSubBase on the processor itself so the slot travels with the processor rather than the registration site. Within one processor, emission order derives from the transaction meta — canonical on-chain data — so IDs stay reproducible across runs and code versions regardless of how the processor list evolves. Protocol emitters (SEP-41, ...) are single-stream and keep using their emitter base directly. Account-less state changes are now filtered per stream before ordinal assignment (previously after merging), which also stops them from being counted as a phantom "" participant in the participant metric. --- internal/indexer/indexer.go | 54 ++++++++++++------- internal/indexer/indexer_test.go | 51 ++++++++++++++++++ internal/indexer/mocks.go | 7 +++ .../indexer/processors/contract_deploy.go | 4 ++ internal/indexer/processors/contracts/sac.go | 4 ++ internal/indexer/processors/effects.go | 4 ++ internal/indexer/types/types.go | 44 ++++++++++++--- 7 files changed, 143 insertions(+), 25 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 6a7a77feb..bc6b38084 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -72,6 +72,10 @@ type ParticipantsProcessorInterface interface { type OperationProcessorInterface interface { ProcessOperation(ctx context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) Name() string + // StateChangeSubBase is this processor's slot in the indexer's state_change_id + // sub-namespace registry (see types.StateChangeSubBase*). It is part of the + // on-disk ID layout and must never change once rows exist with it. + StateChangeSubBase() int64 } // LedgerChangeProcessor is a generic interface for processors that extract data from ledger changes. @@ -342,21 +346,9 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa } } - // Drop state changes with no account to associate with, then assign each - // retained one a deterministic state_change_id: an ordinal numbered 1..N in - // emission order within its (to_id, operation_id) group. Filtering first - // keeps the ordinals gap-free for what's actually persisted. - retainedStateChanges := stateChanges[:0] + // Insert state changes (already filtered and ID-assigned per processor + // stream by getTransactionStateChanges) for _, stateChange := range stateChanges { - if stateChange.AccountID == "" { - continue - } - retainedStateChanges = append(retainedStateChanges, stateChange) - } - types.AssignStateChangeOrdinals(retainedStateChanges, types.StateChangeOrdinalBaseIndexer) - - // Insert state changes - for _, stateChange := range retainedStateChanges { // Get the correct operation for this state change var operation types.Operation if stateChange.OperationID != 0 { @@ -374,19 +366,23 @@ func (i *Indexer) processTransaction(ctx context.Context, tx ingest.LedgerTransa return allParticipants.Cardinality(), nil } -// getTransactionStateChanges processes operations of a transaction and calculates all state changes +// getTransactionStateChanges processes operations of a transaction and calculates all state +// changes. Each emitting processor's stream is filtered and given its deterministic +// state_change_ids independently, inside that processor's own sub-namespace (see the sub-base +// registry in types), so the returned changes are ready to persist and no processor's IDs +// depend on another processor's output or on registration order. func (i *Indexer) getTransactionStateChanges(ctx context.Context, transaction ingest.LedgerTransaction, opsParticipants map[int64]processors.OperationParticipants) ([]types.StateChange, error) { - stateChanges := []types.StateChange{} + streams := make([][]types.StateChange, len(i.processors)) // Process operations sequentially since there are only 3 processors per operation // Creating a worker pool here adds unnecessary overhead for _, opParticipants := range opsParticipants { - for _, processor := range i.processors { + for procIdx, processor := range i.processors { processorStateChanges, processorErr := processor.ProcessOperation(ctx, opParticipants.OpWrapper) if processorErr != nil && !errors.Is(processorErr, processors.ErrInvalidOpType) { return nil, fmt.Errorf("processing %s state changes: %w", processor.Name(), processorErr) } - stateChanges = append(stateChanges, processorStateChanges...) + streams[procIdx] = append(streams[procIdx], processorStateChanges...) } } @@ -395,11 +391,31 @@ func (i *Indexer) getTransactionStateChanges(ctx context.Context, transaction in if err != nil { return nil, fmt.Errorf("processing token transfer state changes: %w", err) } - stateChanges = append(stateChanges, tokenTransferStateChanges...) + var stateChanges []types.StateChange + for procIdx, processor := range i.processors { + stateChanges = assignStateChangeStream(stateChanges, streams[procIdx], processor.StateChangeSubBase()) + } + stateChanges = assignStateChangeStream(stateChanges, tokenTransferStateChanges, types.StateChangeSubBaseTokenTransfer) return stateChanges, nil } +// assignStateChangeStream drops stream entries with no account to associate with, assigns +// the retained ones their deterministic state_change_ids at the emitting processor's slot in +// the indexer namespace, and appends them to dst. Filtering before assignment keeps ordinals +// gap-free (1..N per (to_id, operation_id) group) for what's actually persisted. +func assignStateChangeStream(dst, stream []types.StateChange, subBase int64) []types.StateChange { + retained := stream[:0] + for _, stateChange := range stream { + if stateChange.AccountID == "" { + continue + } + retained = append(retained, stateChange) + } + types.AssignStateChangeOrdinals(retained, types.StateChangeOrdinalBaseIndexer+subBase) + return append(dst, retained...) +} + // GetLedgerTransactions extracts transactions from ledger close meta. func GetLedgerTransactions(ctx context.Context, networkPassphrase string, ledgerMeta xdr.LedgerCloseMeta) ([]ingest.LedgerTransaction, error) { ledgerTxReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, ledgerMeta) diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index 11ef00f7a..a3fc8acd1 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -1355,3 +1355,54 @@ func TestExtractContractEventsForLedger_CorpusCoversWrapperVersions(t *testing.T require.True(t, wrapperWithEvents[1], "corpus must include a wrapper-V1 (Protocol 20-22) ledger with extracted contract events") require.True(t, wrapperWithEvents[2], "corpus must include a wrapper-V2 (Protocol 23+) ledger with extracted contract events") } + +func TestAssignStateChangeStream_SubNamespaces(t *testing.T) { + effects := []types.StateChange{ + {OperationID: 42, AccountID: "GA"}, + {OperationID: 42, AccountID: "GB"}, + {OperationID: 43, AccountID: "GC"}, + } + tokenTransfers := []types.StateChange{ + {OperationID: 42, AccountID: "GD"}, + {OperationID: 42, AccountID: ""}, // no account — dropped before assignment + {OperationID: 42, AccountID: "GE"}, + } + + var merged []types.StateChange + merged = assignStateChangeStream(merged, effects, types.StateChangeSubBaseEffects) + merged = assignStateChangeStream(merged, tokenTransfers, types.StateChangeSubBaseTokenTransfer) + + // Each stream numbers its own emissions 1..N per operation inside its own + // sub-namespace; the account-less entry is filtered out first so ordinals + // stay gap-free. + require.Len(t, merged, 5) + assert.Equal(t, types.StateChangeSubBaseEffects+1, merged[0].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseEffects+2, merged[1].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseEffects+1, merged[2].StateChangeID) // op 43: its own group + assert.Equal(t, types.StateChangeSubBaseTokenTransfer+1, merged[3].StateChangeID) + assert.Equal(t, types.StateChangeSubBaseTokenTransfer+2, merged[4].StateChangeID) + + // The IDs a stream gets are a function of that stream alone: assigning the + // same effects emissions with no other stream present yields identical IDs. + // This is the property that makes IDs immune to processors being added, + // removed, or reordered around this one. + alone := assignStateChangeStream(nil, []types.StateChange{ + {OperationID: 42, AccountID: "GA"}, + {OperationID: 42, AccountID: "GB"}, + {OperationID: 43, AccountID: "GC"}, + }, types.StateChangeSubBaseEffects) + require.Len(t, alone, 3) + for i := range alone { + assert.Equal(t, merged[i].StateChangeID, alone[i].StateChangeID) + } + + // All IDs across streams sharing operation 42 are distinct. + seen := map[int64]bool{} + for _, sc := range merged { + if sc.OperationID != 42 { + continue + } + assert.False(t, seen[sc.StateChangeID], "duplicate state_change_id %d", sc.StateChangeID) + seen[sc.StateChangeID] = true + } +} diff --git a/internal/indexer/mocks.go b/internal/indexer/mocks.go index c1951e78f..30351f9dc 100644 --- a/internal/indexer/mocks.go +++ b/internal/indexer/mocks.go @@ -38,6 +38,13 @@ func (m *MockTokenTransferProcessor) ProcessTransaction(ctx context.Context, tx type MockOperationProcessor struct { mock.Mock + // SubBase is returned by StateChangeSubBase as a plain field so tests can + // place a mock processor in a sub-namespace without mock.On plumbing. + SubBase int64 +} + +func (m *MockOperationProcessor) StateChangeSubBase() int64 { + return m.SubBase } func (m *MockOperationProcessor) ProcessOperation(ctx context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) { diff --git a/internal/indexer/processors/contract_deploy.go b/internal/indexer/processors/contract_deploy.go index 0293901f0..9f9717d2f 100644 --- a/internal/indexer/processors/contract_deploy.go +++ b/internal/indexer/processors/contract_deploy.go @@ -28,6 +28,10 @@ func (p *ContractDeployProcessor) Name() string { return "contract_deploy" } +func (p *ContractDeployProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseContractDeploy +} + // ProcessOperation emits a state change for each contract deployment (including subinvocations). func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *TransactionOperationWrapper) ([]types.StateChange, error) { startTime := time.Now() diff --git a/internal/indexer/processors/contracts/sac.go b/internal/indexer/processors/contracts/sac.go index df7a7e1cb..e0ebd2a82 100644 --- a/internal/indexer/processors/contracts/sac.go +++ b/internal/indexer/processors/contracts/sac.go @@ -49,6 +49,10 @@ func (p *SACEventsProcessor) Name() string { return "sac" } +func (p *SACEventsProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseSACEvents +} + // ProcessOperation processes contract events and converts them into state changes. func (p *SACEventsProcessor) ProcessOperation(_ context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) { startTime := time.Now() diff --git a/internal/indexer/processors/effects.go b/internal/indexer/processors/effects.go index b6fb18bed..66f17ad9c 100644 --- a/internal/indexer/processors/effects.go +++ b/internal/indexer/processors/effects.go @@ -81,6 +81,10 @@ func (p *EffectsProcessor) Name() string { return "effects" } +func (p *EffectsProcessor) StateChangeSubBase() int64 { + return types.StateChangeSubBaseEffects +} + // ProcessOperation extracts effects from a Stellar operation and converts them into state changes. // It processes account state changes like signer modifications, threshold updates, flag changes, // home domain updates, data entry changes, and sponsorship relationship modifications. diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index 151617de2..4c0a3ea51 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -700,21 +700,53 @@ type StateChangeCursorGetter interface { // resulting state_change_id values can never collide across emitters even // when they share an operation. Bases are spaced far apart (int64 has ample // room) and must never change once rows exist with them. +// +// The indexer subdivides its namespace further, one sub-namespace per +// emitting processor — see the sub-base registry below. const ( StateChangeOrdinalBaseIndexer int64 = 0 StateChangeOrdinalBaseSEP41 int64 = 1 << 40 StateChangeOrdinalBaseBlend int64 = 2 << 40 ) +// Sub-namespace registry within the indexer's emitter namespace. +// +// The indexer is itself multi-stream: several processors (token transfers, +// effects, contract deploys, SAC events) can emit state changes for the same +// operation. Each processor numbers its own emissions independently (see +// Indexer.getTransactionStateChanges) and adds StateChangeOrdinalBaseIndexer +// plus its sub-base below, so IDs never depend on the order processors are +// registered or invoked in — adding or reordering processors cannot shift +// another processor's IDs. Within one processor, emission order derives from +// the transaction meta (canonical on-chain data), which is what makes the +// scheme reproducible across runs. +// +// Sub-bases are spaced 1<<28 apart: 4096 sub-namespaces fit inside the +// indexer's 1<<40-wide namespace, each with room for ~268M state changes per +// (to_id, operation_id) — both far beyond what transaction meta size limits +// allow. Protocol emitters (SEP-41, Blend) are single-stream and use their +// base directly; a future multi-stream emitter subdivides its own namespace +// the same way. Like the emitter bases, sub-bases must never change once +// rows exist with them; new processors take the next unused slot. +const ( + StateChangeSubBaseTokenTransfer int64 = 0 << 28 + StateChangeSubBaseEffects int64 = 1 << 28 + StateChangeSubBaseContractDeploy int64 = 2 << 28 + StateChangeSubBaseSACEvents int64 = 3 << 28 +) + // AssignStateChangeOrdinals assigns each state change in changes a // deterministic state_change_id: an ordinal numbered 1..N in slice order -// within each distinct OperationID group, plus base. Processing the same -// input twice (e.g. a re-ingested ledger) yields byte-identical IDs, so a -// duplicate BatchCopy fails loudly on the state_changes primary key instead of -// inserting duplicate rows. +// within each distinct OperationID group, plus base. base is the calling +// emitter's namespace base — plus its processor's sub-base for multi-stream +// emitters like the indexer. Processing the same input twice (e.g. a +// re-ingested ledger) yields byte-identical IDs, so a duplicate BatchCopy +// fails loudly on the state_changes primary key instead of inserting +// duplicate rows. // -// Callers must pass the final, filtered slice — the one actually handed to -// BatchCopy — so ordinals come out contiguous (1..N, no gaps) per group. +// Callers must pass one emission stream at a time — the final, filtered slice +// actually handed to BatchCopy — so ordinals come out contiguous (1..N, no +// gaps) per group within the stream's namespace. func AssignStateChangeOrdinals(changes []StateChange, base int64) { next := make(map[int64]int64, len(changes)) for i := range changes { From b6fe8ac8a9b792e82aa568985e4bbee83b9bca49 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 16 Jul 2026 16:41:13 -0400 Subject: [PATCH 08/14] fix(data): force ledger_created_at into every tx/op/state-change projection The time-pinned relationship lookups key on the parent row's ledger_created_at. Projections built from client field selections only carried it when the client happened to request that scalar (only BatchGetByAccountAddress forced it), so e.g. `transaction { operations }` without ledgerCreatedAt selected pinned the child lookup to the zero time and returned an empty connection. Every transaction/operation/state-change projection now forces the partition column, exactly like the key columns already were; regression tests cover the minimal-projection paths and the pinned child lookup end to end. --- internal/data/operations.go | 14 ++++----- internal/data/operations_test.go | 37 ++++++++++++++++++++++++ internal/data/query_utils.go | 6 ++++ internal/data/statechanges.go | 14 ++++----- internal/data/statechanges_test.go | 34 ++++++++++++++++++++++ internal/data/transactions.go | 8 +++--- internal/data/transactions_test.go | 46 ++++++++++++++++++++++++++++++ 7 files changed, 141 insertions(+), 18 deletions(-) diff --git a/internal/data/operations.go b/internal/data/operations.go index 4a1787b70..140d17d5a 100644 --- a/internal/data/operations.go +++ b/internal/data/operations.go @@ -23,7 +23,7 @@ type OperationModel struct { } func (m *OperationModel) GetByID(ctx context.Context, id int64, columns string) (*types.Operation, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") query := fmt.Sprintf(`SELECT %s FROM operations WHERE id = $1`, columns) start := time.Now() operation, err := db.QueryOne[types.Operation](ctx, m.DB, query, id) @@ -41,7 +41,7 @@ func (m *OperationModel) GetAll(ctx context.Context, columns string, limit *int3 if err := validatePositiveLimit(limit); err != nil { return nil, err } - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") queryBuilder := strings.Builder{} var args []interface{} argIndex := 1 @@ -119,7 +119,7 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, led if len(toIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding // (see the package doc above). Each key carries its parent transaction's ledger_created_at, @@ -183,7 +183,7 @@ func (m *OperationModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, led // Operations for a transaction are found using TOID range: (tx_to_id, tx_to_id + 4096), pinned // to the parent transaction's ledger_created_at for partition-column chunk exclusion. func (m *OperationModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *int64, sortOrder SortOrder) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "", "id", "ledger_created_at") queryBuilder := strings.Builder{} // Operations for a tx_to_id are in range (tx_to_id, tx_to_id + 4096) based on TOID encoding. fmt.Fprintf(&queryBuilder, `SELECT %s, ledger_created_at as cursor_ledger_created_at, id as cursor_id FROM operations WHERE id > $1 AND id < $1 + 4096 AND ledger_created_at = $2`, columns) @@ -247,7 +247,7 @@ func (m *OperationModel) BatchGetAccountOperationsByToIDs(ctx context.Context, a if len(toIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s FROM UNNEST($2::bigint[], $3::timestamptz[]) AS i(to_id, ledger_created_at) @@ -284,7 +284,7 @@ func (m *OperationModel) BatchGetAccountOperationsByToIDs(ctx context.Context, a // Uses a MATERIALIZED CTE + LATERAL join pattern to allow TimescaleDB ChunkAppend optimization // on the operations_accounts hypertable by ordering on ledger_created_at first. func (m *OperationModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, columns string, limit *int32, cursor *types.CompositeCursor, orderBy SortOrder, timeRange *TimeRange) ([]*types.OperationWithCursor, error) { - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} @@ -374,7 +374,7 @@ func (m *OperationModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs [ if len(scOpIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("scOpIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scOpIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id") + columns = prepareColumnsWithID(columns, types.Operation{}, "o", "id", "ledger_created_at") // ORDER BY reads k.ledger_created_at (from UNNEST), not o.ledger_created_at: the LATERAL // only projects the caller-requested columns, which may not include ledger_created_at, but diff --git a/internal/data/operations_test.go b/internal/data/operations_test.go index d2804deb4..91bd653a4 100644 --- a/internal/data/operations_test.go +++ b/internal/data/operations_test.go @@ -903,3 +903,40 @@ func TestOperationModel_BatchGetAccountOperationsByToIDs(t *testing.T) { assert.Equal(t, int64(4097), ops[3].ID) }) } + +// A client selecting no time fields must still get operations whose LedgerCreatedAt is +// hydrated: the state-change and transaction relationship resolvers pin their lookups on the +// parent operation's partition timestamp, so every projection forces ledger_created_at. +func TestOperationModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + m := &OperationModel{DB: dbConnectionPool, Metrics: metrics.NewMetrics(reg).DB} + + now := time.Now().UTC().Truncate(time.Microsecond) + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) + VALUES (4098, 'PAYMENT', 'xdr', 'op_success', true, 1, $1) + `, now) + require.NoError(t, err) + + op, err := m.GetByID(ctx, 4098, "id") + require.NoError(t, err) + assert.True(t, now.Equal(op.LedgerCreatedAt), "GetByID with minimal projection must hydrate ledger_created_at") + + limit := int32(10) + ops, err := m.GetAll(ctx, "id", &limit, nil, DESC) + require.NoError(t, err) + require.Len(t, ops, 1) + assert.True(t, now.Equal(ops[0].Operation.LedgerCreatedAt), "GetAll with minimal projection must hydrate ledger_created_at") + + batched, err := m.BatchGetByToID(ctx, 4096, now, "id", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, batched, 1) + assert.True(t, now.Equal(batched[0].Operation.LedgerCreatedAt), "BatchGetByToID with minimal projection must hydrate ledger_created_at") +} diff --git a/internal/data/query_utils.go b/internal/data/query_utils.go index 02a2befa5..7d5ba698c 100644 --- a/internal/data/query_utils.go +++ b/internal/data/query_utils.go @@ -171,6 +171,12 @@ func getDBColumns(model any) set.Set[string] { // prepareColumnsWithID ensures that all specified identifier columns are always included in the column list. // It appends whatever idColumns are passed in, which is useful for tables (such as state_changes) that use composite keys. +// +// Every transaction/operation/state-change projection also forces ledger_created_at — the +// hypertable partition column — even when the client didn't request that field: relationship +// resolvers pin their child lookups on the parent row's ledger_created_at (see the dataloader +// keys), so a row fetched without it would silently time-pin those lookups to the zero value +// and return empty relationships. func prepareColumnsWithID(columns string, model any, prefix string, idColumns ...string) string { var dbColumns set.Set[string] if columns == "" { diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 1ba38401d..2f55a2e18 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -32,7 +32,7 @@ var _ StateChangeWriter = (*StateChangeModel)(nil) // BatchGetByAccountAddress gets the state changes that are associated with the given account address. // Optional filters: txHash, operationID, category, and reason can be used to further filter results. func (m *StateChangeModel) BatchGetByAccountAddress(ctx context.Context, accountAddress string, txHash *string, operationID *int64, category *string, reason *string, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder, timeRange *TimeRange) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder args := []interface{}{types.AddressBytea(accountAddress)} argIndex := 2 @@ -128,7 +128,7 @@ func (m *StateChangeModel) GetAll(ctx context.Context, columns string, limit *in if err := validatePositiveLimit(limit); err != nil { return nil, err } - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder var args []interface{} argIndex := 1 @@ -327,7 +327,7 @@ func (m *StateChangeModel) BatchCopy( // BatchGetByToID gets state changes for a single transaction with pagination support, pinned to // the parent transaction's ledger_created_at for partition-column chunk exclusion. func (m *StateChangeModel) BatchGetByToID(ctx context.Context, toID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id @@ -391,7 +391,7 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, l if len(toIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("toIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(toIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") // The ORDER BY + LIMIT live inside the LATERAL (a per-transaction top-N), replacing the old // ROW_NUMBER()-over-everything CTE with an equivalent per-to_id cap; a subquery containing @@ -458,7 +458,7 @@ func (m *StateChangeModel) BatchGetByToIDs(ctx context.Context, toIDs []int64, l // BatchGetByOperationID gets state changes for a single operation with pagination support, // pinned to the parent operation's ledger_created_at for partition-column chunk exclusion. func (m *StateChangeModel) BatchGetByOperationID(ctx context.Context, operationID int64, ledgerCreatedAt time.Time, columns string, limit *int32, cursor *types.StateChangeCursor, sortOrder SortOrder) ([]*types.StateChangeWithCursor, error) { - columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") var queryBuilder strings.Builder fmt.Fprintf(&queryBuilder, ` SELECT %s, ledger_created_at as cursor_ledger_created_at, to_id as cursor_to_id, operation_id as cursor_operation_id, state_change_id as cursor_state_change_id @@ -545,7 +545,7 @@ func (m *StateChangeModel) BatchGetAccountStateChangesByToIDs(ctx context.Contex hi = t } } - columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s FROM state_changes sc @@ -577,7 +577,7 @@ func (m *StateChangeModel) BatchGetByOperationIDs(ctx context.Context, operation if len(operationIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("operationIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(operationIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id") + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") // The ORDER BY + LIMIT live inside the LATERAL (a per-operation top-N), replacing the old // ROW_NUMBER()-over-everything CTE with an equivalent per-operation cap; a subquery diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 99a79b80b..618e19f01 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -1198,3 +1198,37 @@ func TestStateChangeModel_BatchGetAccountStateChangesByToIDs(t *testing.T) { assert.Equal(t, int64(1), scs[2].StateChangeID) }) } + +// A client selecting no time fields must still get state changes whose LedgerCreatedAt is +// hydrated: the operation and transaction relationship resolvers pin their lookups on the +// parent state change's partition timestamp, so every projection forces ledger_created_at. +func TestStateChangeModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + m := &StateChangeModel{DB: dbConnectionPool, Metrics: metrics.NewMetrics(reg).DB} + + now := time.Now().UTC().Truncate(time.Microsecond) + address := keypair.MustRandom().Address() + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO state_changes (to_id, state_change_id, state_change_category, state_change_reason, ledger_created_at, ledger_number, account_id, operation_id) + VALUES (4096, 1, 'BALANCE', 'CREDIT', $1, 1, $2, 4098) + `, now, types.AddressBytea(address)) + require.NoError(t, err) + + limit := int32(10) + byToID, err := m.BatchGetByToID(ctx, 4096, now, "state_change_category", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, byToID, 1) + assert.True(t, now.Equal(byToID[0].StateChange.LedgerCreatedAt), "BatchGetByToID with minimal projection must hydrate ledger_created_at") + + byOpID, err := m.BatchGetByOperationID(ctx, 4098, now, "state_change_category", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, byOpID, 1) + assert.True(t, now.Equal(byOpID[0].StateChange.LedgerCreatedAt), "BatchGetByOperationID with minimal projection must hydrate ledger_created_at") +} diff --git a/internal/data/transactions.go b/internal/data/transactions.go index cfdac954e..c547f3191 100644 --- a/internal/data/transactions.go +++ b/internal/data/transactions.go @@ -23,7 +23,7 @@ type TransactionModel struct { } func (m *TransactionModel) GetByHash(ctx context.Context, hash string, columns string) (*types.Transaction, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id", "ledger_created_at") query := fmt.Sprintf(`SELECT %s FROM transactions WHERE hash = $1`, columns) start := time.Now() hashBytea := types.HashBytea(hash) @@ -42,7 +42,7 @@ func (m *TransactionModel) GetAll(ctx context.Context, columns string, limit *in if err := validatePositiveLimit(limit); err != nil { return nil, err } - columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "", "to_id", "ledger_created_at") queryBuilder := strings.Builder{} var args []interface{} argIndex := 1 @@ -191,7 +191,7 @@ func (m *TransactionModel) BatchGetByAccountAddress(ctx context.Context, account // // See SEP-35: https://github.com/stellar/stellar-protocol/blob/master/ecosystem/sep-0035.md func (m *TransactionModel) BatchGetByOperationIDs(ctx context.Context, operationIDs []int64, columns string) ([]*types.TransactionWithOperationID, error) { - columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "transactions", "to_id", "ledger_created_at") // Join operations to transactions using TOID encoding: // An operation ID's lower 12 bits encode the operation index within the transaction. // Masking these bits (id &~ 0xFFF) gives the transaction's to_id. @@ -228,7 +228,7 @@ func (m *TransactionModel) BatchGetByStateChangeIDs(ctx context.Context, scToIDs if len(scToIDs) != len(ledgerCreatedAts) { return nil, fmt.Errorf("scToIDs and ledgerCreatedAts must be parallel arrays of equal length, got %d and %d", len(scToIDs), len(ledgerCreatedAts)) } - columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id") + columns = prepareColumnsWithID(columns, types.Transaction{}, "t", "to_id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s, CONCAT(k.to_id, '-', k.operation_id, '-', k.state_change_id) AS state_change_id diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 2ddf771f5..0ae2e5a28 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -523,3 +523,49 @@ func BenchmarkTransactionModel_BatchCopy(b *testing.B) { }) } } + +// A client selecting no time fields must still get rows whose LedgerCreatedAt is hydrated: +// relationship resolvers pin their child lookups (operations, state changes) on the parent +// transaction's partition timestamp, so every projection forces ledger_created_at. +func TestTransactionModel_MinimalProjectionHydratesLedgerCreatedAt(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + reg := prometheus.NewRegistry() + dbMetrics := metrics.NewMetrics(reg).DB + m := &TransactionModel{DB: dbConnectionPool, Metrics: dbMetrics} + opModel := &OperationModel{DB: dbConnectionPool, Metrics: dbMetrics} + + now := time.Now().UTC().Truncate(time.Microsecond) + txHash := types.HashBytea("0076b7b0133690fbfb2de8fa9ca2273cb4f2e29447e0cf0e14a5f82d0daa4876") + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO transactions (hash, to_id, fee_charged, result_code, ledger_number, ledger_created_at, is_fee_bump) + VALUES ($1, 4096, 100, 'TransactionResultCodeTxSuccess', 1, $2, false) + `, txHash, now) + require.NoError(t, err) + _, err = dbConnectionPool.Exec(ctx, ` + INSERT INTO operations (id, operation_type, operation_xdr, result_code, successful, ledger_number, ledger_created_at) + VALUES (4098, 'PAYMENT', 'xdr', 'op_success', true, 1, $1) + `, now) + require.NoError(t, err) + + tx, err := m.GetByHash(ctx, txHash.String(), "hash") + require.NoError(t, err) + assert.True(t, now.Equal(tx.LedgerCreatedAt), "GetByHash with minimal projection must hydrate ledger_created_at") + + limit := int32(10) + txs, err := m.GetAll(ctx, "hash", &limit, nil, DESC) + require.NoError(t, err) + require.Len(t, txs, 1) + assert.True(t, now.Equal(txs[0].Transaction.LedgerCreatedAt), "GetAll with minimal projection must hydrate ledger_created_at") + + // End-to-end: the hydrated timestamp pins the child operations lookup. + ops, err := opModel.BatchGetByToID(ctx, tx.ToID, tx.LedgerCreatedAt, "id", &limit, nil, ASC) + require.NoError(t, err) + require.Len(t, ops, 1, "time-pinned child lookup must find the operation via the hydrated timestamp") + assert.Equal(t, int64(4098), ops[0].Operation.ID) +} From 150ec416f74b4e790923f94b06cada0c3be51d73 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 20 Jul 2026 15:12:13 -0400 Subject: [PATCH 09/14] fix(indexer): group state-change ordinals by (to_id, operation_id) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AssignStateChangeOrdinals keyed its counter map on operation_id alone, but the namespace-registry contract numbers ordinals within an (to_id, operation_id) group. Current emitters are safe only incidentally (the indexer streams per transaction, so to_id is constant; SEP-41's operation_id embeds the transaction index and never spans to_ids). A window-scoped emitter staging transaction-level changes with operation_id=0 across several transactions would share one counter bucket, making a row's ordinal depend on the window's contents — re-ingesting the same ledger under different bounds would then mint different state_change_ids for the same rows and silently bypass the primary key. Key the map on the (to_id, operation_id) pair so ordinals are window-independent and match the documented contract. --- internal/indexer/types/types.go | 11 ++++++----- internal/indexer/types/types_test.go | 23 +++++++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index 4c0a3ea51..f71a406d0 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -737,7 +737,7 @@ const ( // AssignStateChangeOrdinals assigns each state change in changes a // deterministic state_change_id: an ordinal numbered 1..N in slice order -// within each distinct OperationID group, plus base. base is the calling +// within each distinct (to_id, operation_id) group, plus base. base is the calling // emitter's namespace base — plus its processor's sub-base for multi-stream // emitters like the indexer. Processing the same input twice (e.g. a // re-ingested ledger) yields byte-identical IDs, so a duplicate BatchCopy @@ -748,11 +748,12 @@ const ( // actually handed to BatchCopy — so ordinals come out contiguous (1..N, no // gaps) per group within the stream's namespace. func AssignStateChangeOrdinals(changes []StateChange, base int64) { - next := make(map[int64]int64, len(changes)) + type ordinalKey struct{ toID, opID int64 } + next := make(map[ordinalKey]int64, len(changes)) for i := range changes { - opID := changes[i].OperationID - next[opID]++ - changes[i].StateChangeID = base + next[opID] + k := ordinalKey{changes[i].ToID, changes[i].OperationID} + next[k]++ + changes[i].StateChangeID = base + next[k] } } diff --git a/internal/indexer/types/types_test.go b/internal/indexer/types/types_test.go index 18919e6f0..a0662f4ec 100644 --- a/internal/indexer/types/types_test.go +++ b/internal/indexer/types/types_test.go @@ -451,3 +451,26 @@ func TestAssignStateChangeOrdinals_DifferentEmittersNeverCollide(t *testing.T) { } assert.Len(t, ids, 3, "each emitter's ID for the same operation must be distinct") } + +func TestAssignStateChangeOrdinals_GroupsByToIDAndOperationID(t *testing.T) { + // A window-scoped emitter can stage transaction-level changes with + // OperationID=0 for several transactions (distinct ToIDs) in one call. + // Ordinals restart per (to_id, operation_id) pair, not per operation_id + // alone: otherwise a row's ID would depend on which other transactions + // happen to share the window, and re-running the same ledger under + // different window bounds would produce different IDs for the same logical + // row — silently bypassing the state_changes primary key on re-ingest. + changes := []StateChange{ + {ToID: 100, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 100, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 200, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + {ToID: 200, OperationID: 0, StateChangeCategory: StateChangeCategoryBalance}, + } + AssignStateChangeOrdinals(changes, StateChangeOrdinalBaseIndexer) + + // Keying on operation_id alone would yield 1,2,3,4 (one shared bucket). + assert.Equal(t, []int64{1, 2, 1, 2}, []int64{ + changes[0].StateChangeID, changes[1].StateChangeID, + changes[2].StateChangeID, changes[3].StateChangeID, + }) +} From c0124fe4bd3c3831a310bd100975a05fc0773983 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 20 Jul 2026 15:12:22 -0400 Subject: [PATCH 10/14] fix(data): hydrate loader-key columns in account state-change batch query BatchGetAccountStateChangesByToIDs forced only to_id and ledger_created_at into its projection, while its sibling BatchGetByAccountAddress forces the full key set. The state-change relationship resolvers build their dataloader key from (to_id, operation_id, state_change_id), so on a minimal account-scoped selection that maps to none of those scalars the rows returned with operation_id=0 and state_change_id=0: the key collapsed to "-0-0", nested operation/transaction resolved to null, and the dataloader deduped every state change of the transaction into one shared result. Force operation_id, state_change_id and account_id alongside to_id and ledger_created_at, matching the sibling. --- internal/data/statechanges.go | 2 +- internal/data/statechanges_test.go | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/internal/data/statechanges.go b/internal/data/statechanges.go index 2f55a2e18..7da6bcdbd 100644 --- a/internal/data/statechanges.go +++ b/internal/data/statechanges.go @@ -545,7 +545,7 @@ func (m *StateChangeModel) BatchGetAccountStateChangesByToIDs(ctx context.Contex hi = t } } - columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "ledger_created_at") + columns = prepareColumnsWithID(columns, types.StateChange{}, "sc", "to_id", "operation_id", "state_change_id", "account_id", "ledger_created_at") query := fmt.Sprintf(` SELECT %s FROM state_changes sc diff --git a/internal/data/statechanges_test.go b/internal/data/statechanges_test.go index 618e19f01..9b691ab19 100644 --- a/internal/data/statechanges_test.go +++ b/internal/data/statechanges_test.go @@ -1165,6 +1165,24 @@ func TestStateChangeModel_BatchGetAccountStateChangesByToIDs(t *testing.T) { assert.Equal(t, int64(1), scs[1].StateChangeID) }) + t.Run("minimal projection still hydrates the loader-key columns", func(t *testing.T) { + // The operation/transaction relationship resolvers build their dataloader + // key from (to_id, operation_id, state_change_id). A client selection that + // maps to none of those scalars must still come back with them hydrated, + // or the key collapses to "-0-0": nested operation/transaction + // resolve to null and the loader dedups every row into one. + scs, err := m.BatchGetAccountStateChangesByToIDs(ctx, acct, []int64{4096}, []time.Time{now}, "state_change_category") + require.NoError(t, err) + require.Len(t, scs, 2) + for _, sc := range scs { + assert.Equal(t, int64(4096), sc.ToID, "to_id must be hydrated") + assert.Equal(t, int64(4097), sc.OperationID, "operation_id must be hydrated for the loader key") + assert.NotZero(t, sc.StateChangeID, "state_change_id must be hydrated for the loader key") + assert.Equal(t, acct, sc.AccountID.String(), "account_id must be hydrated") + assert.True(t, now.Equal(sc.LedgerCreatedAt), "ledger_created_at must be hydrated") + } + }) + t.Run("empty when account has no state changes in the transaction", func(t *testing.T) { none := keypair.MustRandom().Address() scs, err := m.BatchGetAccountStateChangesByToIDs(ctx, none, []int64{4096}, []time.Time{now}, "") From 1ef14505106ae47f58250642ec05a82bbd7d96fd Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 20 Jul 2026 15:12:33 -0400 Subject: [PATCH 11/14] fix(ingest): restrict hypertable policy config to live mode Retention and compression policies live in shared database state, so configuring them on every ingest start (any mode) let ad-hoc backfills impose their default flags on the live deployment's policies: a backfill without --retention-period removed the retention policy and reconciliation job; a backfill with retention dropped the very history it was writing; and --compression-max-chunks ratcheted live compression jobs down permanently, since the >0 write is never reset by live's default of 0. Gate the call on live mode: a backfill no longer touches shared policy state. --- internal/ingest/ingest.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 2add68370..a6515e3c0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -134,12 +134,14 @@ func setupDeps(cfg Configs) (services.IngestService, error) { return nil, fmt.Errorf("connecting to the database: %w", err) } - // Configured regardless of ingestion mode: a backfill-first bootstrap must not - // run under TimescaleDB's auto-created columnstore policy defaults (unlimited - // maxchunks, no retention). All settings applied here are idempotent, so - // running this on every start (live or backfill) is safe. - if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { - return nil, fmt.Errorf("configuring hypertable settings: %w", err) + // Retention and compression policies live in shared database state, so only + // the live deployment configures them. A backfill runs with ad-hoc flags and + // must not mutate or remove the policies the live pods rely on; and an active + // retention policy would drop the very history a backfill is writing. + if cfg.IngestionMode == services.IngestionModeLive { + if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { + return nil, fmt.Errorf("configuring hypertable settings: %w", err) + } } m := metrics.NewMetrics(prometheus.NewRegistry()) From fc317f3f74092964c2e355865ebbc281d5040dae Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 20 Jul 2026 16:06:38 -0400 Subject: [PATCH 12/14] fix(indexer): deterministic emission order for multi-change state-change groups AssignStateChangeOrdinals freezes each state change's ID from its slice position within a (to_id, operation_id) group, so intra-group order must be reproducible across re-ingests. Two processors emitted multi-change groups in Go map-iteration order: - effects parseThresholds: a SetOptions op updating several thresholds now emits them in fixed low -> medium -> high order instead of thresholdToReasonMap iteration order. - contract_deploy: an InvokeHostFunction op deploying several contracts now emits them in canonical XDR walk order (root host function, then auth invocations depth-first) with first-wins dedup by contract ID, instead of draining a map keyed by contract ID. Exact-order regression tests pin both orders. --- .../indexer/processors/contract_deploy.go | 13 +-- .../processors/contract_deploy_test.go | 87 +++++++++++++++++++ internal/indexer/processors/effects.go | 9 +- internal/indexer/processors/effects_test.go | 26 ++++++ 4 files changed, 127 insertions(+), 8 deletions(-) diff --git a/internal/indexer/processors/contract_deploy.go b/internal/indexer/processors/contract_deploy.go index 9f9717d2f..8c1aaa2e2 100644 --- a/internal/indexer/processors/contract_deploy.go +++ b/internal/indexer/processors/contract_deploy.go @@ -53,19 +53,24 @@ func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *Transa WithCategory(types.StateChangeCategoryAccount). WithReason(types.StateChangeReasonCreate) - deployedContractsMap := map[string]types.StateChange{} + var stateChanges []types.StateChange + seen := map[string]struct{}{} processCreate := func(fromAddr xdr.ContractIdPreimageFromAddress) error { contractID, err := calculateContractID(p.networkPassphrase, fromAddr) if err != nil { return fmt.Errorf("calculating contract ID: %w", err) } + if _, ok := seen[contractID]; ok { + return nil + } deployerAddr, err := fromAddr.Address.String() if err != nil { return fmt.Errorf("deployer address to string: %w", err) } - deployedContractsMap[contractID] = builder.Clone().WithAccount(contractID).WithDeployer(deployerAddr).Build() + seen[contractID] = struct{}{} + stateChanges = append(stateChanges, builder.Clone().WithAccount(contractID).WithDeployer(deployerAddr).Build()) return nil } @@ -123,9 +128,5 @@ func (p *ContractDeployProcessor) ProcessOperation(_ context.Context, op *Transa } } - stateChanges := make([]types.StateChange, 0, len(deployedContractsMap)) - for _, sc := range deployedContractsMap { - stateChanges = append(stateChanges, sc) - } return stateChanges, nil } diff --git a/internal/indexer/processors/contract_deploy_test.go b/internal/indexer/processors/contract_deploy_test.go index be42a420e..a97892aa4 100644 --- a/internal/indexer/processors/contract_deploy_test.go +++ b/internal/indexer/processors/contract_deploy_test.go @@ -146,6 +146,93 @@ func Test_ContractDeployProcessor_Process_createContract(t *testing.T) { } } +// Test_ContractDeployProcessor_Process_multipleContractsDeterministicOrder pins the emission +// order for an operation that deploys several contracts across its root host function and its +// auth invocation tree. AssignStateChangeOrdinals (internal/indexer/types/types.go) derives each +// state change's ID from its slice position within the operation, so re-ingesting the same ledger +// must yield the same slice order every time. This test asserts the exact order (not +// ElementsMatch) and also exercises dedup: the auth tree redeclares the same contract creation +// that a sibling subinvocation already produced, and it must not be emitted twice. +func Test_ContractDeployProcessor_Process_multipleContractsDeterministicOrder(t *testing.T) { + const ( + rootDeployer = "GCQIH6MRLCJREVE76LVTKKEZXRIT6KSX7KU65HPDDBYFKFYHIYSJE57R" + subDeployerB = "GDG2KKXC62BINMUZNBTLG235323N6BOIR33JBF4ELTOUKUG5BDE6HJZT" + subDeployerC = "GAGWN4445WLODCXT7RUZXJLQK5XWX4GICXDOAAZZGK2N3BR67RIIVWJ7" + + contractA = "CA7UGIYR2H63C2ETN2VE4WDQ6YX5XNEWNWC2DP7A64B2ZR7VJJWF3SBF" // rootDeployer + TestSalt, root host function + ) + saltB := xdr.Uint256{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} + saltC := xdr.Uint256{2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2} + + preimageB := xdr.ContractIdPreimageFromAddress{Address: makeScAddress(subDeployerB), Salt: saltB} + preimageC := xdr.ContractIdPreimageFromAddress{Address: makeScAddress(subDeployerC), Salt: saltC} + + contractB, err := calculateContractID(network.TestNetworkPassphrase, preimageB) + require.NoError(t, err) + contractC, err := calculateContractID(network.TestNetworkPassphrase, preimageC) + require.NoError(t, err) + + ctx := context.Background() + + op := makeBasicSorobanOp() + setFromAddress(op, xdr.HostFunctionTypeHostFunctionTypeCreateContract, rootDeployer) + op.Operation.Body.InvokeHostFunctionOp.Auth = []xdr.SorobanAuthorizationEntry{ + { + RootInvocation: xdr.SorobanAuthorizedInvocation{ + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeContractFn, + }, + SubInvocations: []xdr.SorobanAuthorizedInvocation{ + { + // Deploys contractB, then walks depth-first into its own SubInvocations + // (deploying contractC) before the loop moves on to the next sibling. + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractHostFn, + CreateContractHostFn: &xdr.CreateContractArgs{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageB}}, + }, + SubInvocations: []xdr.SorobanAuthorizedInvocation{ + { + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractV2HostFn, + CreateContractV2HostFn: &xdr.CreateContractArgsV2{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageC}}, + }, + }, + }, + }, + { + // Redeclares the same contractB creation as the sibling above; must be + // deduped and not appear a second time in the output. + Function: xdr.SorobanAuthorizedFunction{ + Type: xdr.SorobanAuthorizedFunctionTypeSorobanAuthorizedFunctionTypeCreateContractHostFn, + CreateContractHostFn: &xdr.CreateContractArgs{ContractIdPreimage: xdr.ContractIdPreimage{Type: xdr.ContractIdPreimageTypeContractIdPreimageFromAddress, FromAddress: &preimageB}}, + }, + }, + }, + }, + }, + } + + proc := NewContractDeployProcessor(network.TestNetworkPassphrase, nil) + stateChanges, err := proc.ProcessOperation(ctx, op) + require.NoError(t, err) + + builder := NewStateChangeBuilder(12345, closeTime.Unix(), 53021371269120, nil). + WithOperationID(53021371269121). + WithReason(types.StateChangeReasonCreate). + WithCategory(types.StateChangeCategoryAccount) + + wantOrder := []types.StateChange{ + builder.Clone().WithDeployer(rootDeployer).WithAccount(contractA).Build(), + builder.Clone().WithDeployer(subDeployerB).WithAccount(contractB).Build(), + builder.Clone().WithDeployer(subDeployerC).WithAccount(contractC).Build(), + } + + require.Len(t, stateChanges, len(wantOrder)) + for i := range wantOrder { + assertStateChangeEqual(t, wantOrder[i], stateChanges[i]) + } +} + func Test_ContractDeployProcessor_Process_invokeContract(t *testing.T) { const ( opSourceAccount = "GBZURSTQQRSU3XB66CHJ3SH2ZWLG663V5SWM6HF3FL72BOMYHDT4QTUF" diff --git a/internal/indexer/processors/effects.go b/internal/indexer/processors/effects.go index 66f17ad9c..16297477a 100644 --- a/internal/indexer/processors/effects.go +++ b/internal/indexer/processors/effects.go @@ -40,6 +40,9 @@ var ( "med_threshold": types.StateChangeReasonMedium, "high_threshold": types.StateChangeReasonHigh, } + // thresholdOrder fixes the emission order of threshold state changes so + // ordinals within a (to_id, operation_id) group are deterministic across runs. + thresholdOrder = []string{"low_threshold", "med_threshold", "high_threshold"} // signerEffectToReasonMap maps Stellar signer effect types to our internal state change reasons // for tracking when signers are added, removed, or updated on accounts signerEffectToReasonMap = map[int32]types.StateChangeReason{ @@ -723,9 +726,11 @@ func (p *EffectsProcessor) parseThresholds(changeBuilder *StateChangeBuilder, ef // Find the account entry change to get old threshold values prevLedgerEntryState := p.getPrevLedgerEntryState(effect, xdr.LedgerEntryTypeAccount, changes) - // Create a separate state change for each threshold that was modified + // Create a separate state change for each threshold that was modified, in a fixed + // order so ordinal assignment is deterministic when multiple thresholds change together. var thresholdChanges []types.StateChange - for threshold, reason := range thresholdToReasonMap { + for _, threshold := range thresholdOrder { + reason := thresholdToReasonMap[threshold] if value, ok := effect.Details[threshold]; ok { // Convert XDR threshold value to int16 for storage newValue := int16(value.(xdr.Uint32)) diff --git a/internal/indexer/processors/effects_test.go b/internal/indexer/processors/effects_test.go index dbbf770ad..8941e8405 100644 --- a/internal/indexer/processors/effects_test.go +++ b/internal/indexer/processors/effects_test.go @@ -454,3 +454,29 @@ func TestEffects_ProcessTransaction(t *testing.T) { assert.Equal(t, strkey.MustEncode(strkey.VersionByteContract, assetContractID[:]), changes[0].TokenID.String()) }) } + +// TestEffects_ParseThresholds_DeterministicOrder pins the emission order of threshold state +// changes: when a single SetOptions effect updates low, medium, and high thresholds together, +// the resulting state changes must always come back in low -> medium -> high order so that +// ordinals assigned within a (to_id, operation_id) group are reproducible across re-ingests. +func TestEffects_ParseThresholds_DeterministicOrder(t *testing.T) { + processor := NewEffectsProcessor(networkPassphrase, nil) + changeBuilder := NewStateChangeBuilder(12345, 12345*100, toid.New(12345, 1, 1).ToInt64(), nil). + WithAccount("GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH"). + WithCategory(types.StateChangeCategorySignatureThreshold) + effect := &EffectOutput{ + Address: "GC4XF7RE3R4P77GY5XNGICM56IOKUURWAAANPXHFC7G5H6FCNQVVH3OH", + Details: map[string]interface{}{ + "low_threshold": xdr.Uint32(1), + "med_threshold": xdr.Uint32(2), + "high_threshold": xdr.Uint32(3), + }, + } + + thresholdChanges := processor.parseThresholds(changeBuilder, effect, nil) + + require.Len(t, thresholdChanges, 3) + assert.Equal(t, types.StateChangeReasonLow, thresholdChanges[0].StateChangeReason) + assert.Equal(t, types.StateChangeReasonMedium, thresholdChanges[1].StateChangeReason) + assert.Equal(t, types.StateChangeReasonHigh, thresholdChanges[2].StateChangeReason) +} From f1a477fd2d1bb1435fcff13e7abc1d171acf2dcf Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 20 Jul 2026 16:06:51 -0400 Subject: [PATCH 13/14] feat(services): enforce state_change ID namespace bases at processor construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ProtocolProcessor gains StateChangeOrdinalBase(); BuildProcessors validates every built processor's base is a positive multiple of the new types.StateChangeOrdinalNamespaceWidth (1<<40) and unique across the set, so a future protocol shipped with a stale or duplicated base fails fast at startup (both boot entry points funnel through BuildProcessors) instead of silently colliding state_change_ids on disk. SEP-41's PersistHistory assigns ordinals via its own StateChangeOrdinalBase() as the single source of truth. A registry-wide test builds all registered processors and re-checks the invariant in CI automatically as new protocols land. Docs: the ProtocolProcessor godoc now states the determinism contract — intra-(to_id, operation_id) emission order must be reproducible from canonical on-chain data; non-determinism across groups (parallel fan-out, group-disjoint map iteration) is explicitly allowed; processors that cannot emit a group in canonical order must sort it by a deterministic total key before AssignStateChangeOrdinals. The indexer's OperationProcessorInterface carries the same invariant for sub-streams, and the base/sub-base registry documents the bit-40/bit-28 split rationale (2^23 emitter namespaces x 2^40 ordinals per group; 4096 indexer sub-streams x 2^28 — every dimension orders of magnitude beyond transaction-meta size limits). --- internal/indexer/indexer.go | 7 +++ internal/indexer/types/types.go | 35 ++++++++--- internal/services/ingest_test.go | 4 ++ internal/services/mocks.go | 5 ++ internal/services/processor_registry.go | 23 +++++++ .../services/processor_registry_ext_test.go | 43 +++++++++++++ internal/services/processor_registry_test.go | 62 +++++++++++++++++++ internal/services/protocol_migrate_test.go | 5 ++ internal/services/protocol_processor.go | 30 +++++++++ internal/services/sep41/processor.go | 5 +- 10 files changed, 210 insertions(+), 9 deletions(-) create mode 100644 internal/services/processor_registry_ext_test.go diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index bc6b38084..7a877255c 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -70,6 +70,13 @@ type ParticipantsProcessorInterface interface { } type OperationProcessorInterface interface { + // ProcessOperation returns this operation's state changes. The slice must be + // in canonical, reproducible order — derived from the transaction meta / XDR + // walk order, never Go map iteration, goroutine completion, or channel + // arrival order — because types.AssignStateChangeOrdinals freezes each + // element's state_change_id from its position within the (to_id, operation_id) + // group. Cross-operation ordering is free: ordinals are keyed per group, so + // only order WITHIN one call's returned slice is load-bearing. ProcessOperation(ctx context.Context, opWrapper *processors.TransactionOperationWrapper) ([]types.StateChange, error) Name() string // StateChangeSubBase is this processor's slot in the indexer's state_change_id diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index f71a406d0..8e8c8112a 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -698,15 +698,31 @@ type StateChangeCursorGetter interface { // changes 1..N in deterministic emission order within an (to_id, operation_id) // group (see AssignStateChangeOrdinals) and adds its base below, so the // resulting state_change_id values can never collide across emitters even -// when they share an operation. Bases are spaced far apart (int64 has ample -// room) and must never change once rows exist with them. +// when they share an operation. Bases are consecutive multiples of +// StateChangeOrdinalNamespaceWidth and must never change once rows exist with +// them. +// +// Why the split at bit 40: an int64 has 63 usable positive bits. Reserving the +// low 40 bits per namespace carves the space into 2^23 (~8.4M) emitter +// namespaces, each holding 2^40 (~1.1e12) ordinals per (to_id, operation_id) +// group. Transaction meta size limits cap a single operation's emissions at a +// few thousand, and new emitters are added only at code-review speed, so both +// sides of the split sit orders of magnitude beyond their physical bounds — +// neither dimension can become the binding constraint, and frozen bases never +// face renumbering pressure. // // The indexer subdivides its namespace further, one sub-namespace per // emitting processor — see the sub-base registry below. const ( + // StateChangeOrdinalNamespaceWidth is the span of state_change_id values + // reserved for each emitter namespace; the bases below are consecutive + // multiples of it. Changing it renumbers every base and is forbidden once + // rows exist. + StateChangeOrdinalNamespaceWidth int64 = 1 << 40 + StateChangeOrdinalBaseIndexer int64 = 0 - StateChangeOrdinalBaseSEP41 int64 = 1 << 40 - StateChangeOrdinalBaseBlend int64 = 2 << 40 + StateChangeOrdinalBaseSEP41 int64 = 1 * StateChangeOrdinalNamespaceWidth + StateChangeOrdinalBaseBlend int64 = 2 * StateChangeOrdinalNamespaceWidth ) // Sub-namespace registry within the indexer's emitter namespace. @@ -721,10 +737,13 @@ const ( // the transaction meta (canonical on-chain data), which is what makes the // scheme reproducible across runs. // -// Sub-bases are spaced 1<<28 apart: 4096 sub-namespaces fit inside the -// indexer's 1<<40-wide namespace, each with room for ~268M state changes per -// (to_id, operation_id) — both far beyond what transaction meta size limits -// allow. Protocol emitters (SEP-41, Blend) are single-stream and use their +// Sub-bases split the indexer's 1<<40-wide namespace at bit 28: 2^12 = 4096 +// sub-streams fit inside it, each with 2^28 (~268M) ordinals per +// (to_id, operation_id) group. As with the bit-40 emitter split, both sides +// sit orders of magnitude beyond their physical bounds — transaction meta size +// limits cap per-operation emissions at a few thousand, and new sub-streams +// are added only at code-review speed — so neither dimension can become the +// binding constraint. Protocol emitters (SEP-41, Blend) are single-stream and use their // base directly; a future multi-stream emitter subdivides its own namespace // the same way. Like the emitter bases, sub-bases must never change once // rows exist with them; new processors take the next unused slot. diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 69b0e221d..470526da3 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1874,6 +1874,10 @@ type testProtocolProcessor struct { func (p *testProtocolProcessor) ProtocolID() string { return p.id } +func (p *testProtocolProcessor) StateChangeOrdinalBase() int64 { + return types.StateChangeOrdinalBaseSEP41 +} + func (p *testProtocolProcessor) Reset() { p.stagedLedgerCount = 0 } func (p *testProtocolProcessor) ProcessLedger(_ context.Context, input ProtocolProcessorInput) error { diff --git a/internal/services/mocks.go b/internal/services/mocks.go index 407679720..ea673acb6 100644 --- a/internal/services/mocks.go +++ b/internal/services/mocks.go @@ -264,6 +264,11 @@ func (m *ProtocolProcessorMock) ProtocolID() string { return args.String(0) } +func (m *ProtocolProcessorMock) StateChangeOrdinalBase() int64 { + args := m.Called() + return args.Get(0).(int64) +} + func (m *ProtocolProcessorMock) ProcessLedger(ctx context.Context, input ProtocolProcessorInput) error { args := m.Called(ctx, input) return args.Error(0) diff --git a/internal/services/processor_registry.go b/internal/services/processor_registry.go index 97683955b..dca9b7462 100644 --- a/internal/services/processor_registry.go +++ b/internal/services/processor_registry.go @@ -3,6 +3,8 @@ package services import ( "fmt" "sort" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) // processorRegistry holds factory functions keyed by protocol ID. Factories @@ -45,8 +47,17 @@ func GetAllProcessorIDs() []string { // BuildProcessors materializes processors for the given protocol IDs from the // registry. Returns an error if any ID has no registered factory. +// +// It also validates the state_change_id namespace bases of the built set: every +// base must be positive and a multiple of types.StateChangeOrdinalNamespaceWidth +// (base 0 is the main indexer's sub-namespaced region and is off-limits to +// protocol processors), and no two processors may share a base. Both boot entry +// points (internal/ingest and cmd/protocol_migrate) funnel through here, so a +// stale or duplicated base copied into a new processor fails fast at startup +// rather than silently colliding state_change_ids on-disk. func BuildProcessors(deps ProtocolDeps, protocolIDs []string) ([]ProtocolProcessor, error) { out := make([]ProtocolProcessor, 0, len(protocolIDs)) + protocolIDByBase := make(map[int64]string, len(protocolIDs)) for _, pid := range protocolIDs { factory, ok := GetProcessor(pid) if !ok { @@ -56,6 +67,18 @@ func BuildProcessors(deps ProtocolDeps, protocolIDs []string) ([]ProtocolProcess if p == nil { return nil, fmt.Errorf("processor factory for protocol %q returned nil", pid) } + + base := p.StateChangeOrdinalBase() + if base <= 0 || base%types.StateChangeOrdinalNamespaceWidth != 0 { + return nil, fmt.Errorf("protocol %q has invalid state_change_id base %d: "+ + "must be a positive multiple of %d", p.ProtocolID(), base, types.StateChangeOrdinalNamespaceWidth) + } + if other, dup := protocolIDByBase[base]; dup { + return nil, fmt.Errorf("protocols %q and %q share state_change_id base %d", + other, p.ProtocolID(), base) + } + protocolIDByBase[base] = p.ProtocolID() + out = append(out, p) } return out, nil diff --git a/internal/services/processor_registry_ext_test.go b/internal/services/processor_registry_ext_test.go new file mode 100644 index 000000000..90f76de07 --- /dev/null +++ b/internal/services/processor_registry_ext_test.go @@ -0,0 +1,43 @@ +package services_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/indexer/types" + "github.com/stellar/wallet-backend/internal/services" + + // Blank-import every protocol package so its init() registers with the + // framework. This test lives in the external services_test package to avoid + // the import cycle a protocol package (which imports services) would create + // with an in-package test. Add new protocol packages here as they land so + // BuildProcessors validates their state_change_id bases in CI automatically. + _ "github.com/stellar/wallet-backend/internal/services/sep41" +) + +// TestBuildProcessorsAllRegistered builds the full registered set with +// zero-value deps and asserts BuildProcessors accepts it: every base is a +// positive multiple of the namespace width and no two collide. Any future +// protocol whose factory tolerates zero-value deps is checked here for free. +func TestBuildProcessorsAllRegistered(t *testing.T) { + ids := services.GetAllProcessorIDs() + require.NotEmpty(t, ids, "expected at least one registered protocol processor") + + procs, err := services.BuildProcessors(services.ProtocolDeps{}, ids) + require.NoError(t, err) + assert.Len(t, procs, len(ids)) + + seen := make(map[int64]string, len(procs)) + for _, p := range procs { + base := p.StateChangeOrdinalBase() + assert.Positive(t, base, "protocol %q base must be positive", p.ProtocolID()) + assert.Zero(t, base%types.StateChangeOrdinalNamespaceWidth, + "protocol %q base %d must be a multiple of %d", p.ProtocolID(), base, types.StateChangeOrdinalNamespaceWidth) + if other, dup := seen[base]; dup { + t.Errorf("protocols %q and %q share base %d", other, p.ProtocolID(), base) + } + seen[base] = p.ProtocolID() + } +} diff --git a/internal/services/processor_registry_test.go b/internal/services/processor_registry_test.go index 3af5809ea..da3c97e37 100644 --- a/internal/services/processor_registry_test.go +++ b/internal/services/processor_registry_test.go @@ -5,8 +5,27 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/indexer/types" ) +// stubProcessor is a minimal ProtocolProcessor for BuildProcessors validation +// tests: it carries an ID and a state_change_id base and no-ops everything else. +type stubProcessor struct { + ProtocolProcessor // embed the interface; only the methods below are exercised + id string + base int64 +} + +func (s stubProcessor) ProtocolID() string { return s.id } +func (s stubProcessor) StateChangeOrdinalBase() int64 { return s.base } + +func registerStub(id string, base int64) { + RegisterProcessor(id, func(ProtocolDeps) ProtocolProcessor { + return stubProcessor{id: id, base: base} + }) +} + func withCleanProcessorRegistry(t *testing.T) { t.Helper() original := processorRegistry @@ -63,3 +82,46 @@ func TestRegisterProcessor(t *testing.T) { assert.Equal(t, []string{"A", "B"}, ids) }) } + +func TestBuildProcessorsBaseValidation(t *testing.T) { + w := types.StateChangeOrdinalNamespaceWidth + + t.Run("distinct valid bases succeed", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 1*w) + registerStub("B", 2*w) + + procs, err := BuildProcessors(ProtocolDeps{}, []string{"A", "B"}) + require.NoError(t, err) + assert.Len(t, procs, 2) + }) + + t.Run("duplicate bases error names both protocols", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 3*w) + registerStub("B", 3*w) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A", "B"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + assert.Contains(t, err.Error(), "B") + }) + + t.Run("zero base errors", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", 0) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + }) + + t.Run("non-multiple base errors", func(t *testing.T) { + withCleanProcessorRegistry(t) + registerStub("A", w+1) + + _, err := BuildProcessors(ProtocolDeps{}, []string{"A"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "A") + }) +} diff --git a/internal/services/protocol_migrate_test.go b/internal/services/protocol_migrate_test.go index c32949fdc..5d0e2c5e6 100644 --- a/internal/services/protocol_migrate_test.go +++ b/internal/services/protocol_migrate_test.go @@ -20,6 +20,7 @@ import ( "github.com/stellar/wallet-backend/internal/data" "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stellar/wallet-backend/internal/indexer/types" "github.com/stellar/wallet-backend/internal/metrics" "github.com/stellar/wallet-backend/internal/utils" ) @@ -184,6 +185,10 @@ type testRecordingProcessor struct { func (p *testRecordingProcessor) ProtocolID() string { return p.id } +func (p *testRecordingProcessor) StateChangeOrdinalBase() int64 { + return types.StateChangeOrdinalBaseSEP41 +} + func (p *testRecordingProcessor) Reset() { p.resetCount++ } func (p *testRecordingProcessor) ProcessLedger(_ context.Context, input ProtocolProcessorInput) error { diff --git a/internal/services/protocol_processor.go b/internal/services/protocol_processor.go index 0f880b712..6e0ccf450 100644 --- a/internal/services/protocol_processor.go +++ b/internal/services/protocol_processor.go @@ -28,8 +28,38 @@ func (m StagingMode) NeedsHistory() bool { return m != StagingModeCurrentState } func (m StagingMode) NeedsCurrentState() bool { return m != StagingModeHistory } // ProtocolProcessor produces and persists protocol-specific state for a ledger. +// +// Determinism contract for state changes: state_change_id values are frozen at +// persist and their base is never renumbered, so every emitted state change +// must land at a byte-identical ID on every re-run of the same input. This +// requires the relative order of all state changes within one +// (to_id, operation_id) group — the slice handed to +// types.AssignStateChangeOrdinals — to be reproducible from canonical on-chain +// data (event index in transaction meta, XDR structure walk order) at the +// moment ordinals are assigned. It must never depend on Go map iteration, +// goroutine completion, or channel arrival order, and one group's emissions +// must never straddle a map-iteration boundary. +// +// Non-determinism is explicitly allowed ACROSS groups: ordinals are keyed per +// (to_id, operation_id), so anything that only reorders whole groups — parallel +// fan-out across operations, transactions, or ledgers, or iterating a map in +// which each key maps to a distinct group (as SEP-41 does over ContractEvents) — +// cannot affect IDs. +// +// Escape hatch: a processor that cannot naturally emit a group in canonical +// order (e.g. it computes one operation's changes concurrently) must sort the +// group by a deterministic total key before calling AssignStateChangeOrdinals. +// IDs need not be semantically chronological — only byte-identical on re-run. +// The ordering step must happen before rows are written. type ProtocolProcessor interface { ProtocolID() string + // StateChangeOrdinalBase returns this protocol's state_change_id namespace + // base (see the registry in internal/indexer/types). It must be a positive + // multiple of types.StateChangeOrdinalNamespaceWidth, unique across all + // registered processors, and must never change once rows exist with it. + // PersistHistory must assign IDs via types.AssignStateChangeOrdinals using + // exactly this base. + StateChangeOrdinalBase() int64 // ProcessLedger accumulates ("folds") this ledger's protocol state into the // processor's staged sets. It does NOT reset between ledgers — the caller owns // reset via Reset(). Folding a window of ledgers then calling a Persist method diff --git a/internal/services/sep41/processor.go b/internal/services/sep41/processor.go index c422be22e..0f0086485 100644 --- a/internal/services/sep41/processor.go +++ b/internal/services/sep41/processor.go @@ -92,6 +92,9 @@ var _ services.ProtocolProcessor = (*processor)(nil) func (p *processor) ProtocolID() string { return ProtocolID } +// StateChangeOrdinalBase returns the SEP-41 state_change_id namespace base. +func (p *processor) StateChangeOrdinalBase() int64 { return types.StateChangeOrdinalBaseSEP41 } + // ProcessLedger consumes contract events that the indexer (or // ExtractContractEventsForLedger in the migration path) has already // extracted into the buffer. The processor never touches LedgerCloseMeta — @@ -320,7 +323,7 @@ func (p *processor) PersistHistory(ctx context.Context, dbTx pgx.Tx) error { // Assign deterministic state_change_ids in emission order, namespaced under // the SEP-41 base so they can never collide with the main indexer's or // Blend's IDs for the same operation (see types.AssignStateChangeOrdinals). - types.AssignStateChangeOrdinals(p.stagedStateChanges, types.StateChangeOrdinalBaseSEP41) + types.AssignStateChangeOrdinals(p.stagedStateChanges, p.StateChangeOrdinalBase()) if _, err := p.stateChanges.BatchCopy(ctx, dbTx, p.stagedStateChanges); err != nil { return fmt.Errorf("persisting %d SEP-41 state changes for ledger %d: %w", len(p.stagedStateChanges), p.ledgerNumber, err) From 918f38c3d784119c9c9f82867f3b5b147bce6079 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 21 Jul 2026 12:31:16 -0400 Subject: [PATCH 14/14] feat(indexer): enforce state_change sub-base registry at Indexer construction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit NewIndexer now validates each operation processor's StateChangeSubBase(): it must be a positive multiple of the new named types.StateChangeSubNamespaceWidth, fit inside the indexer's emitter namespace, and be unique across streams — with the token-transfer stream's slot (sub-base 0) reserved up front since it is emitted outside the processors slice. A stale or duplicated sub-base copied into a new processor now fails fast at startup instead of surfacing as a state_changes primary-key violation on the first ledger where two processors emit for the same operation, mirroring the protocol-base validation in services.BuildProcessors. --- internal/indexer/indexer.go | 38 +++++++++++++++++- internal/indexer/indexer_test.go | 69 ++++++++++++++++++++++++++++++-- internal/indexer/types/types.go | 14 +++++-- internal/loadtest/runner.go | 5 ++- internal/services/ingest.go | 7 +++- 5 files changed, 122 insertions(+), 11 deletions(-) diff --git a/internal/indexer/indexer.go b/internal/indexer/indexer.go index 7a877255c..5deb57b4b 100644 --- a/internal/indexer/indexer.go +++ b/internal/indexer/indexer.go @@ -121,8 +121,14 @@ type Indexer struct { // during ledger meta processing; classification is performed downstream // (per-batch) by services.DispatchClassification — this keeps the indexer // agnostic of any specific protocol or its validator shape. -func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metrics.IngestionMetrics) *Indexer { - return &Indexer{ +// +// It validates the state_change_id sub-base registry of the built processor +// set (see validateStateChangeSubBases), so a stale or duplicated sub-base +// copied into a new processor fails fast at startup rather than surfacing as +// a state_changes primary-key violation on the first ledger where two +// processors emit for the same operation. +func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metrics.IngestionMetrics) (*Indexer, error) { + indexer := &Indexer{ participantsProcessor: processors.NewParticipantsProcessor(networkPassphrase), tokenTransferProcessor: processors.NewTokenTransferProcessor(networkPassphrase, ingestionMetrics), sacBalancesProcessor: processors.NewSACBalancesProcessor(networkPassphrase, ingestionMetrics), @@ -142,6 +148,34 @@ func NewIndexer(networkPassphrase string, pool pond.Pool, ingestionMetrics *metr ingestionMetrics: ingestionMetrics, networkPassphrase: networkPassphrase, } + if err := validateStateChangeSubBases(indexer.processors); err != nil { + return nil, fmt.Errorf("validating state_change_id sub-bases: %w", err) + } + return indexer, nil +} + +// validateStateChangeSubBases validates the state_change_id sub-base registry +// of the given processor set (see types.StateChangeSubBase*): every sub-base +// must be a non-negative multiple of types.StateChangeSubNamespaceWidth that +// fits inside the indexer's emitter namespace, and no two streams may share +// one. The token-transfer stream's slot is reserved up front since it is +// emitted outside the processors slice (see getTransactionStateChanges). +func validateStateChangeSubBases(procs []OperationProcessorInterface) error { + streamBySubBase := map[int64]string{types.StateChangeSubBaseTokenTransfer: "token_transfer"} + for _, p := range procs { + subBase := p.StateChangeSubBase() + if subBase <= 0 || subBase%types.StateChangeSubNamespaceWidth != 0 || subBase >= types.StateChangeOrdinalNamespaceWidth { + return fmt.Errorf("processor %q has invalid state_change_id sub-base %d: "+ + "must be a positive multiple of %d below %d", + p.Name(), subBase, types.StateChangeSubNamespaceWidth, types.StateChangeOrdinalNamespaceWidth) + } + if other, dup := streamBySubBase[subBase]; dup { + return fmt.Errorf("indexer streams %q and %q share state_change_id sub-base %d", + other, p.Name(), subBase) + } + streamBySubBase[subBase] = p.Name() + } + return nil } // ProcessLedgerTransactions processes all transactions in a ledger in parallel. diff --git a/internal/indexer/indexer_test.go b/internal/indexer/indexer_test.go index a3fc8acd1..811c98e0c 100644 --- a/internal/indexer/indexer_test.go +++ b/internal/indexer/indexer_test.go @@ -145,8 +145,9 @@ func TestIndexer_NewIndexer(t *testing.T) { networkPassphrase := network.TestNetworkPassphrase pool := pond.NewPool(runtime.NumCPU()) - indexer := NewIndexer(networkPassphrase, pool, nil) + indexer, err := NewIndexer(networkPassphrase, pool, nil) + require.NoError(t, err) require.NotNil(t, indexer) assert.NotNil(t, indexer.participantsProcessor) assert.NotNil(t, indexer.tokenTransferProcessor) @@ -155,6 +156,67 @@ func TestIndexer_NewIndexer(t *testing.T) { assert.Len(t, indexer.processors, 3) // effects, contract deploy, SAC events } +func TestValidateStateChangeSubBases(t *testing.T) { + newProc := func(name string, subBase int64) *MockOperationProcessor { + p := &MockOperationProcessor{SubBase: subBase} + p.On("Name").Return(name).Maybe() + return p + } + + testCases := []struct { + name string + procs []OperationProcessorInterface + wantErrContains string + }{ + { + name: "🟢 distinct aligned sub-bases", + procs: []OperationProcessorInterface{ + newProc("effects", types.StateChangeSubBaseEffects), + newProc("contract_deploy", types.StateChangeSubBaseContractDeploy), + }, + }, + { + name: "🔴 duplicate sub-base between two processors", + procs: []OperationProcessorInterface{ + newProc("effects", types.StateChangeSubBaseEffects), + newProc("copy_pasted", types.StateChangeSubBaseEffects), + }, + wantErrContains: `indexer streams "effects" and "copy_pasted" share state_change_id sub-base`, + }, + { + name: "🔴 sub-base 0 is reserved for the token transfer stream", + procs: []OperationProcessorInterface{newProc("zero", 0)}, + wantErrContains: `processor "zero" has invalid state_change_id sub-base 0`, + }, + { + name: "🔴 sub-base not a multiple of the sub-namespace width", + procs: []OperationProcessorInterface{newProc("misaligned", types.StateChangeSubNamespaceWidth+1)}, + wantErrContains: "invalid state_change_id sub-base", + }, + { + name: "🔴 negative sub-base", + procs: []OperationProcessorInterface{newProc("negative", -types.StateChangeSubNamespaceWidth)}, + wantErrContains: "invalid state_change_id sub-base", + }, + { + name: "🔴 sub-base outside the indexer emitter namespace", + procs: []OperationProcessorInterface{newProc("oversized", types.StateChangeOrdinalNamespaceWidth)}, + wantErrContains: "invalid state_change_id sub-base", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateStateChangeSubBases(tc.procs) + if tc.wantErrContains == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tc.wantErrContains) + } + }) + } +} + func TestIndexer_ProcessLedgerTransactions(t *testing.T) { t.Run("🟢 single transaction with participants", func(t *testing.T) { // Create mocks @@ -764,9 +826,10 @@ func TestIndexer_ProcessLedgerTransactions_FeePhaseBalances(t *testing.T) { }, } - indexer := NewIndexer(network.TestNetworkPassphrase, pond.NewPool(runtime.NumCPU()), nil) + indexer, err := NewIndexer(network.TestNetworkPassphrase, pond.NewPool(runtime.NumCPU()), nil) + require.NoError(t, err) buffer := NewIndexerBuffer() - _, err := indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{feeTx}, buffer) + _, err = indexer.ProcessLedgerTransactions(context.Background(), []ingest.LedgerTransaction{feeTx}, buffer) require.NoError(t, err) accountChanges := buffer.GetAccountChanges() diff --git a/internal/indexer/types/types.go b/internal/indexer/types/types.go index 8e8c8112a..bb218c409 100644 --- a/internal/indexer/types/types.go +++ b/internal/indexer/types/types.go @@ -748,10 +748,16 @@ const ( // the same way. Like the emitter bases, sub-bases must never change once // rows exist with them; new processors take the next unused slot. const ( - StateChangeSubBaseTokenTransfer int64 = 0 << 28 - StateChangeSubBaseEffects int64 = 1 << 28 - StateChangeSubBaseContractDeploy int64 = 2 << 28 - StateChangeSubBaseSACEvents int64 = 3 << 28 + // StateChangeSubNamespaceWidth is the span of state_change_id values + // reserved for each indexer sub-stream; the sub-bases below are consecutive + // multiples of it. Changing it renumbers every sub-base and is forbidden + // once rows exist. + StateChangeSubNamespaceWidth int64 = 1 << 28 + + StateChangeSubBaseTokenTransfer int64 = 0 * StateChangeSubNamespaceWidth + StateChangeSubBaseEffects int64 = 1 * StateChangeSubNamespaceWidth + StateChangeSubBaseContractDeploy int64 = 2 * StateChangeSubNamespaceWidth + StateChangeSubBaseSACEvents int64 = 3 * StateChangeSubNamespaceWidth ) // AssignStateChangeOrdinals assigns each state change in changes a diff --git a/internal/loadtest/runner.go b/internal/loadtest/runner.go index 09a269a5c..7455edfe1 100644 --- a/internal/loadtest/runner.go +++ b/internal/loadtest/runner.go @@ -88,7 +88,10 @@ func Run(ctx context.Context, cfg RunConfig) error { metrics.RegisterPoolMetrics(m.Registry(), "loadtest_indexer", indexerPool) // Loadtest doesn't exercise classification; the indexer just buffers raw // WASM bytecode and the dispatcher is bypassed (no validators wired). - ledgerIndexer := indexer.NewIndexer(cfg.NetworkPassphrase, indexerPool, m.Ingestion) + ledgerIndexer, err := indexer.NewIndexer(cfg.NetworkPassphrase, indexerPool, m.Ingestion) + if err != nil { + return fmt.Errorf("creating ledger indexer: %w", err) + } // Create TokenIngestionService for token change processing tokenIngestionService := services.NewTokenIngestionService(services.TokenIngestionServiceConfig{ diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 99605a501..fce9fe6ed 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -162,6 +162,11 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { return nil, fmt.Errorf("building protocol processor map: %w", err) } + ledgerIndexer, err := indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion) + if err != nil { + return nil, fmt.Errorf("creating ledger indexer: %w", err) + } + return &ingestService{ ingestionMode: cfg.IngestionMode, models: cfg.Models, @@ -176,7 +181,7 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { appMetrics: cfg.Metrics, networkPassphrase: cfg.NetworkPassphrase, getLedgersLimit: cfg.GetLedgersLimit, - ledgerIndexer: indexer.NewIndexer(cfg.NetworkPassphrase, ledgerIndexerPool, cfg.Metrics.Ingestion), + ledgerIndexer: ledgerIndexer, contractMetadataService: cfg.ContractMetadataService, protocolValidators: cfg.ProtocolValidators, wasmSpecExtractor: cfg.WasmSpecExtractor,