From 6342b84421b1734c46463c59d3a27bd238d55ff6 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 14 Jul 2026 22:51:55 -0400 Subject: [PATCH 1/4] fix(ingest): graceful shutdown on SIGTERM with clean resource teardown The ingestion process ran on context.Background() and its signal handler only stopped the HTTP servers, so every SIGTERM (each K8s deploy) left the ingest loop running until the kubelet SIGKILLed it after the full grace period, and the wasm-extractor closer goroutine waited on a never-cancelled context so the wazero runtime was never released. Ingest() now derives its root context from signal.NotifyContext and threads it through setupDeps and Run. A shutdown-classified Run error exits 0 after an ordered synchronous teardown (ledger backend, wasm extractor, DB pool) via a cleanup func returned by setupDeps; genuine errors still fail the process. The HTTP servers shut down off the same root context, and the advisory-lock release defer detaches via context.WithoutCancel so the lock is actually freed when the context is already cancelled. --- internal/ingest/ingest.go | 98 ++++++++++++++++++--------- internal/ingest/ingest_test.go | 56 +++++++++++++++ internal/services/ingest_live.go | 8 ++- internal/services/ingest_live_test.go | 83 +++++++++++++++++++++++ 4 files changed, 212 insertions(+), 33 deletions(-) create mode 100644 internal/ingest/ingest_test.go create mode 100644 internal/services/ingest_live_test.go diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 2add68370..494fbaba0 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -2,6 +2,7 @@ package ingest import ( "context" + "errors" "fmt" "net/http" "net/http/pprof" @@ -29,6 +30,10 @@ import ( const ( ServerShutdownTimeout = 10 * time.Second + // wasmExtractorCloseTimeout bounds releasing the wazero runtime on shutdown. + // It runs on a fresh context (not the cancelled root ctx) since the close + // itself has nothing to do with the signal that triggered shutdown. + wasmExtractorCloseTimeout = 5 * time.Second ) // LedgerBackendType represents the type of ledger backend to use @@ -111,27 +116,48 @@ func (c Configs) BuildPoolConfig() db.PoolConfig { } func Ingest(cfg Configs) error { - ctx := context.Background() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() - ingestService, err := setupDeps(cfg) + ingestService, cleanup, err := setupDeps(ctx, cfg) if err != nil { - log.Ctx(ctx).Fatalf("Error setting up dependencies for ingest: %v", err) + return fmt.Errorf("setting up dependencies for ingest: %w", err) } + defer cleanup() - if err = ingestService.Run(ctx, uint32(cfg.StartLedger), uint32(cfg.EndLedger)); err != nil { - log.Ctx(ctx).Fatalf("running 'ingest' from %d to %d: %v", cfg.StartLedger, cfg.EndLedger, err) + runErr := ingestService.Run(ctx, uint32(cfg.StartLedger), uint32(cfg.EndLedger)) + if runErr != nil { + if isShutdownRequested(ctx, runErr) { + log.Ctx(ctx).Infof("shutdown requested; exiting cleanly: %v", runErr) + return nil + } + return fmt.Errorf("running 'ingest' from %d to %d: %w", cfg.StartLedger, cfg.EndLedger, runErr) } return nil } -func setupDeps(cfg Configs) (services.IngestService, error) { - ctx := context.Background() +// isShutdownRequested classifies a Run error as a clean-exit shutdown (root +// ctx cancelled by SIGINT/SIGTERM) versus a genuine failure. ctx is checked +// directly rather than just err, since a cancellation can surface through +// several different wrapped errors depending on where in the ingest loop it +// was observed. +func isShutdownRequested(ctx context.Context, err error) bool { + if ctx.Err() != nil { + return true + } + return errors.Is(err, context.Canceled) +} +// setupDeps builds the ingest service and its dependencies. The returned +// cleanup func releases resources owned at this layer (ledger backend, wasm +// spec extractor, DB pool) and must be called by the caller after Run +// returns. +func setupDeps(ctx context.Context, cfg Configs) (services.IngestService, func(), error) { poolCfg := cfg.BuildPoolConfig() dbConnectionPool, err := db.OpenDBConnectionPool(ctx, cfg.DatabaseURL, poolCfg) if err != nil { - return nil, fmt.Errorf("connecting to the database: %w", err) + return nil, nil, fmt.Errorf("connecting to the database: %w", err) } // Configured regardless of ingestion mode: a backfill-first bootstrap must not @@ -139,25 +165,25 @@ func setupDeps(cfg Configs) (services.IngestService, error) { // maxchunks, no retention). All settings applied here are idempotent, so // running this on every start (live or backfill) is safe. if err := configureHypertableSettings(ctx, dbConnectionPool, cfg.ChunkInterval, cfg.RetentionPeriod, cfg.OldestLedgerCursorName, cfg.CompressionScheduleInterval, cfg.CompressAfter, cfg.MaxChunksToCompress); err != nil { - return nil, fmt.Errorf("configuring hypertable settings: %w", err) + return nil, nil, fmt.Errorf("configuring hypertable settings: %w", err) } m := metrics.NewMetrics(prometheus.NewRegistry()) metrics.RegisterDBPoolMetrics(m.Registry(), dbConnectionPool) models, err := data.NewModels(dbConnectionPool, m.DB) if err != nil { - return nil, fmt.Errorf("creating models: %w", err) + return nil, nil, fmt.Errorf("creating models: %w", err) } httpClient := &http.Client{Timeout: 30 * time.Second} rpcService, err := services.NewRPCService(cfg.RPCURL, cfg.NetworkPassphrase, httpClient, m.RPC) if err != nil { - return nil, fmt.Errorf("instantiating rpc service: %w", err) + return nil, nil, fmt.Errorf("instantiating rpc service: %w", err) } - ledgerBackend, err := NewLedgerBackend(context.Background(), cfg) + ledgerBackend, err := NewLedgerBackend(ctx, cfg) if err != nil { - return nil, fmt.Errorf("creating ledger backend: %w", err) + return nil, nil, fmt.Errorf("creating ledger backend: %w", err) } // Create pond pool for contract metadata fetching @@ -167,7 +193,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { // Create ContractMetadataService for fetching and storing token metadata contractMetadataService, err := services.NewContractMetadataService(rpcService, models.Contract, contractMetadataPool) if err != nil { - return nil, fmt.Errorf("instantiating contract metadata service: %w", err) + return nil, nil, fmt.Errorf("instantiating contract metadata service: %w", err) } // Build a single ProtocolDeps to pass through both the validator and @@ -191,7 +217,7 @@ func setupDeps(cfg Configs) (services.IngestService, error) { }, ) if err != nil { - return nil, fmt.Errorf("connecting to history archive: %w", err) + return nil, nil, fmt.Errorf("connecting to history archive: %w", err) } tokenIngestionService := services.NewTokenIngestionService(services.TokenIngestionServiceConfig{ @@ -230,24 +256,19 @@ func setupDeps(cfg Configs) (services.IngestService, error) { allProtocolIDs := services.GetAllProcessorIDs() protocolProcessors, err := services.BuildProcessors(protocolDeps, allProtocolIDs) if err != nil { - return nil, fmt.Errorf("building protocol processors: %w", err) + return nil, nil, fmt.Errorf("building protocol processors: %w", err) } allValidatorIDs := services.GetAllValidatorIDs() protocolValidators, err := services.BuildValidators(protocolDeps, allValidatorIDs) if err != nil { - return nil, fmt.Errorf("building protocol validators: %w", err) + return nil, nil, fmt.Errorf("building protocol validators: %w", err) } // Spec extractor is owned by the ingest service so it lives for the - // process lifetime and is closed on shutdown. + // process lifetime; it is closed by the returned cleanup func, after + // Run has returned and no extraction can be in flight. wasmExtractor := services.NewWasmSpecExtractor() - go func() { - <-ctx.Done() - if err := wasmExtractor.Close(context.Background()); err != nil { - log.Ctx(ctx).Warnf("closing wasm spec extractor on shutdown: %v", err) - } - }() ingestService, err := services.NewIngestService(services.IngestServiceConfig{ IngestionMode: cfg.IngestionMode, @@ -273,31 +294,44 @@ func setupDeps(cfg Configs) (services.IngestService, error) { ContractMetadataService: contractMetadataService, }) if err != nil { - return nil, fmt.Errorf("instantiating ingest service: %w", err) + return nil, nil, fmt.Errorf("instantiating ingest service: %w", err) } // Start ingest server which serves metrics and health check endpoints. servers := startServers(cfg, models, rpcService, m) - // Wait for termination signal to gracefully shut down the servers. - quit := make(chan os.Signal, 1) - signal.Notify(quit, syscall.SIGINT, syscall.SIGTERM) + // Shut down the servers once the root context is cancelled (SIGINT/SIGTERM, + // registered by the caller via signal.NotifyContext). go func() { - <-quit + <-ctx.Done() log.Info("Shutting down servers...") - ctx, cancel := context.WithTimeout(context.Background(), ServerShutdownTimeout) + shutdownCtx, cancel := context.WithTimeout(context.Background(), ServerShutdownTimeout) defer cancel() for _, server := range servers { - if err := server.Shutdown(ctx); err != nil { + if err := server.Shutdown(shutdownCtx); err != nil { log.Errorf("Server forced to shutdown: %v", err) } } log.Info("Servers gracefully stopped") }() - return ingestService, nil + cleanup := func() { + if err := ledgerBackend.Close(); err != nil { + log.Ctx(ctx).Warnf("closing ledger backend: %v", err) + } + + closeCtx, cancel := context.WithTimeout(context.Background(), wasmExtractorCloseTimeout) + defer cancel() + if err := wasmExtractor.Close(closeCtx); err != nil { + log.Ctx(ctx).Warnf("closing wasm spec extractor: %v", err) + } + + dbConnectionPool.Close() + } + + return ingestService, cleanup, nil } // startServers initializes and starts the ingest server which serves metrics and health check endpoints. diff --git a/internal/ingest/ingest_test.go b/internal/ingest/ingest_test.go new file mode 100644 index 000000000..aa07ba50f --- /dev/null +++ b/internal/ingest/ingest_test.go @@ -0,0 +1,56 @@ +package ingest + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsShutdownRequested(t *testing.T) { + genuineErr := errors.New("genuine failure") + wrappedCanceled := fmt.Errorf("fetching ledger 5: %w", context.Canceled) + + cancelledCtx, cancel := context.WithCancel(context.Background()) + cancel() + + testCases := []struct { + name string + ctx context.Context + err error + want bool + }{ + { + name: "cancelled_ctx_with_genuine_error", + ctx: cancelledCtx, + err: genuineErr, + want: true, + }, + { + name: "live_ctx_with_wrapped_context_canceled", + ctx: context.Background(), + err: wrappedCanceled, + want: true, + }, + { + name: "live_ctx_with_genuine_error", + ctx: context.Background(), + err: genuineErr, + want: false, + }, + { + name: "cancelled_ctx_with_nil_error", + ctx: cancelledCtx, + err: nil, + want: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, isShutdownRequested(tc.ctx, tc.err)) + }) + } +} diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index 69645ef64..c0030f145 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -227,7 +227,13 @@ func (m *ingestService) startLiveIngestion(ctx context.Context) error { return errors.New("advisory lock not acquired") } defer func() { - if err := db.ReleaseAdvisoryLock(ctx, conn, m.advisoryLockID); err != nil { + // Detach from ctx: a shutdown signal cancels ctx before this defer + // runs, and pgx refuses to execute a query on an already-cancelled + // context, which would otherwise leak the lock (conn.Release above + // only returns the connection to the pool, it doesn't end the + // session, so the lock would stay held until the pool itself closes). + releaseCtx := context.WithoutCancel(ctx) + if err := db.ReleaseAdvisoryLock(releaseCtx, conn, m.advisoryLockID); err != nil { err = fmt.Errorf("releasing advisory lock: %w", err) log.Ctx(ctx).Error(err) } diff --git a/internal/services/ingest_live_test.go b/internal/services/ingest_live_test.go new file mode 100644 index 000000000..f2965b5e1 --- /dev/null +++ b/internal/services/ingest_live_test.go @@ -0,0 +1,83 @@ +package services + +import ( + "context" + "testing" + + "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go-stellar-sdk/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/apptracker" + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/db" + "github.com/stellar/wallet-backend/internal/db/dbtest" + "github.com/stellar/wallet-backend/internal/metrics" +) + +// Test_startLiveIngestion_ReleasesAdvisoryLockWhenContextCancelledMidStartup +// guards against ING-02: the advisory lock release used to run on the same +// (loop) context that PrepareRange/GetLedger use, so a shutdown signal +// arriving between lock acquisition and the ingest loop starting would cancel +// that context before the deferred release ran, and pgx refuses to execute a +// query on an already-cancelled context — silently leaking the lock. +// +// PrepareRange is used as the trigger point because it runs after the lock is +// acquired but before the ingest loop starts, standing in for a SIGTERM +// arriving during that narrow startup window. +func Test_startLiveIngestion_ReleasesAdvisoryLockWhenContextCancelledMidStartup(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + pool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer pool.Close() + + setupDBCursors(t, ctx, pool, 50, 40) + + m := metrics.NewMetrics(prometheus.NewRegistry()) + models, err := data.NewModels(pool, m.DB) + require.NoError(t, err) + + mockBackend := &LedgerBackendMock{} + mockBackend.On("PrepareRange", mock.Anything, mock.Anything). + Run(func(mock.Arguments) { cancel() }). + Return(nil) + + const testNetwork = "advisory-lock-release-test" + svc, err := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeLive, + Models: models, + OldestLedgerCursorName: "oldest_ledger_cursor", + AppTracker: &apptracker.MockAppTracker{}, + RPCService: &RPCServiceMock{}, + LedgerBackend: mockBackend, + Metrics: m, + GetLedgersLimit: defaultGetLedgersLimit, + Network: testNetwork, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: &HistoryArchiveMock{}, + }) + require.NoError(t, err) + + err = svc.Run(ctx, 0, 0) + require.Error(t, err, "run should surface the context cancellation") + + // Verify the lock was released using a second, independent pool: Postgres + // advisory locks are reentrant within the same session, so checking with + // the same connection that held the lock would pass even if the release + // silently failed. + verifyCtx := context.Background() + pool2, err := db.OpenDBConnectionPool(verifyCtx, dbt.DSN) + require.NoError(t, err) + defer pool2.Close() + + acquired, err := db.AcquireAdvisoryLock(verifyCtx, pool2, generateAdvisoryLockID(testNetwork)) + require.NoError(t, err) + assert.True(t, acquired, "advisory lock should have been released during shutdown despite the cancelled context") +} From 74ca68ecc9250b9ab825d17132896b961b600087 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 01:16:56 -0400 Subject: [PATCH 2/4] refactor(ingest): move classification and metadata RPC out of DB transactions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protocol classification previously ran inside the per-ledger persist transaction: a ledger deploying SEP-41/Blend contracts triggered simulate calls (batches of 20 with 2s inter-batch sleeps, 3x retries, 30s timeouts) while holding row locks on ingest_store cursors, contract_tokens, and blend_pools — an RPC outage could stall ingestion for minutes per ledger. Validators now split into Match (pure signature check), Prefetch (RPC only, before any transaction opens), and Apply (DB writes only — the signature carries no RPC service, making the guarantee compile-checked). The plan is computed once per ledger and reused verbatim across persist retries, so retries never re-issue RPC calls. protocol-setup shares the dispatcher and picks up the same fix. The checkpoint cold-start similarly fetched SAC metadata via RPC inside the single archive-load transaction; the load now commits with ledger-derived defaults and metadata enrichment runs in a short follow-up transaction whose failure logs rather than rolling back the completed load. --- internal/data/mocks.go | 5 +- internal/data/protocol_wasms.go | 14 +- internal/services/checkpoint.go | 66 ++++- internal/services/checkpoint_test.go | 144 ++++++++++ internal/services/ingest_live.go | 120 ++++---- internal/services/ingest_test.go | 28 +- internal/services/protocol_setup.go | 48 ++-- internal/services/protocol_setup_test.go | 23 +- .../services/protocol_validation_dispatch.go | 120 +++++--- .../protocol_validation_dispatch_test.go | 260 +++++++++++++----- internal/services/protocol_validator.go | 79 +++--- internal/services/sep41/validator.go | 138 +++++----- internal/services/sep41/validator_test.go | 43 ++- internal/services/validator_registry_test.go | 14 +- 14 files changed, 757 insertions(+), 345 deletions(-) diff --git a/internal/data/mocks.go b/internal/data/mocks.go index 83a3aa96f..2fa03a4d4 100644 --- a/internal/data/mocks.go +++ b/internal/data/mocks.go @@ -9,6 +9,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/mock" + "github.com/stellar/wallet-backend/internal/db" "github.com/stellar/wallet-backend/internal/indexer/types" ) @@ -252,8 +253,8 @@ func (m *ProtocolWasmsModelMock) GetUnclassified(ctx context.Context) ([]Protoco return args.Get(0).([]ProtocolWasms), args.Error(1) } -func (m *ProtocolWasmsModelMock) GetClassifiedByHashes(ctx context.Context, dbTx pgx.Tx, hashes []types.HashBytea) (map[types.HashBytea]string, error) { - args := m.Called(ctx, dbTx, hashes) +func (m *ProtocolWasmsModelMock) GetClassifiedByHashes(ctx context.Context, q db.Querier, hashes []types.HashBytea) (map[types.HashBytea]string, error) { + args := m.Called(ctx, q, hashes) if args.Get(0) == nil { return nil, args.Error(1) } diff --git a/internal/data/protocol_wasms.go b/internal/data/protocol_wasms.go index 2b7218f49..9a5959608 100644 --- a/internal/data/protocol_wasms.go +++ b/internal/data/protocol_wasms.go @@ -25,7 +25,7 @@ type ProtocolWasms struct { type ProtocolWasmsModelInterface interface { BatchInsert(ctx context.Context, dbTx pgx.Tx, wasms []ProtocolWasms) error GetUnclassified(ctx context.Context) ([]ProtocolWasms, error) - GetClassifiedByHashes(ctx context.Context, dbTx pgx.Tx, hashes []types.HashBytea) (map[types.HashBytea]string, error) + GetClassifiedByHashes(ctx context.Context, q db.Querier, hashes []types.HashBytea) (map[types.HashBytea]string, error) BatchUpdateProtocolID(ctx context.Context, dbTx pgx.Tx, wasmHashes []types.HashBytea, protocolID string) error } @@ -92,10 +92,12 @@ func (m *ProtocolWasmsModel) GetUnclassified(ctx context.Context) ([]ProtocolWas } // GetClassifiedByHashes returns wasm_hash → protocol_id for the supplied hashes -// where protocol_id IS NOT NULL. The read runs inside the supplied dbTx so -// callers on the live ingest path see classifications consistent with the -// in-flight transaction. -func (m *ProtocolWasmsModel) GetClassifiedByHashes(ctx context.Context, dbTx pgx.Tx, hashes []types.HashBytea) (map[types.HashBytea]string, error) { +// where protocol_id IS NOT NULL. q accepts either the connection pool or a +// pgx.Tx: this read is used both non-transactionally (classification +// prefetch, before any ledger transaction opens — staleness here is harmless +// since this-ledger uploads resolve from the buffer and prior rows are +// immutable) and, where a caller still holds an open tx, against that tx. +func (m *ProtocolWasmsModel) GetClassifiedByHashes(ctx context.Context, q db.Querier, hashes []types.HashBytea) (map[types.HashBytea]string, error) { if len(hashes) == 0 { return nil, nil } @@ -112,7 +114,7 @@ func (m *ProtocolWasmsModel) GetClassifiedByHashes(ctx context.Context, dbTx pgx const query = `SELECT wasm_hash, protocol_id, created_at FROM protocol_wasms WHERE wasm_hash = ANY($1::bytea[]) AND protocol_id IS NOT NULL` start := time.Now() - wasms, err := db.QueryMany[ProtocolWasms](ctx, dbTx, query, hashBytes) + wasms, err := db.QueryMany[ProtocolWasms](ctx, q, query, hashBytes) duration := time.Since(start).Seconds() m.Metrics.QueryDuration.WithLabelValues("GetClassifiedByHashes", "protocol_wasms").Observe(duration) m.Metrics.QueriesTotal.WithLabelValues("GetClassifiedByHashes", "protocol_wasms").Inc() diff --git a/internal/services/checkpoint.go b/internal/services/checkpoint.go index c5c706b0d..8b1a6fbd9 100644 --- a/internal/services/checkpoint.go +++ b/internal/services/checkpoint.go @@ -241,6 +241,12 @@ type checkpointProcessor struct { contractAddressesByWasmHash map[xdr.Hash][]xdr.Hash entries, trustlineCount, accountCount, batchCount int startTime time.Time + // pendingSACMetadata holds contract IDs for SAC contracts discovered + // during the load whose name/symbol/decimals weren't available from + // ledger data alone. finalize populates this without making any RPC + // call; PopulateFromCheckpoint fetches metadata for these IDs in a short + // follow-up transaction after the load commits. + pendingSACMetadata []string } // PopulateFromCheckpoint performs initial cache population from Stellar history archive. @@ -263,12 +269,13 @@ func (s *checkpointService) PopulateFromCheckpoint(ctx context.Context, checkpoi } }() + var proc *checkpointProcessor err = db.RunInTransaction(ctx, s.db, func(dbTx pgx.Tx) error { if _, txErr := dbTx.Exec(ctx, "SET LOCAL synchronous_commit = off"); txErr != nil { return fmt.Errorf("setting synchronous_commit=off: %w", txErr) } - proc := &checkpointProcessor{ + proc = &checkpointProcessor{ service: s, dbTx: dbTx, checkpointLedger: checkpointLedger, @@ -322,6 +329,44 @@ func (s *checkpointService) PopulateFromCheckpoint(ctx context.Context, checkpoi if err != nil { return fmt.Errorf("running db transaction for checkpoint population: %w", err) } + + // Enrich SAC metadata via RPC in a short follow-up transaction, after the + // load (including cursor initialization) has already committed. A fetch + // failure here is logged and leaves these rows at their ledger-derived + // defaults — it must not undo the completed load; a later classification + // pass can retry. + if len(proc.pendingSACMetadata) > 0 { + if enrichErr := s.enrichSACMetadata(ctx, proc.pendingSACMetadata); enrichErr != nil { + log.Ctx(ctx).Errorf("enriching SAC metadata after checkpoint load (defaults retained): %v", enrichErr) + } + } + return nil +} + +// enrichSACMetadata fetches name/symbol/decimals for the given SAC contracts +// via RPC and updates their contract_tokens rows in a short transaction. It +// runs after PopulateFromCheckpoint's load transaction has already committed, +// so RPC round-trips (batches of 20 with a sleep between batches) never hold +// the load's row locks. Any error here (fetch or write) is returned for the +// caller to log — it never rolls back the already-completed load. +func (s *checkpointService) enrichSACMetadata(ctx context.Context, contractIDs []string) error { + sacContracts, err := s.contractMetadataService.FetchSACMetadata(ctx, contractIDs) + if err != nil { + return fmt.Errorf("fetching SAC metadata: %w", err) + } + if len(sacContracts) == 0 { + return nil + } + err = db.RunInTransaction(ctx, s.db, func(dbTx pgx.Tx) error { + if txErr := s.contractModel.BatchUpdateMetadata(ctx, dbTx, sacContracts); txErr != nil { + return fmt.Errorf("updating SAC contract_tokens metadata: %w", txErr) + } + return nil + }) + if err != nil { + return fmt.Errorf("running SAC metadata enrichment transaction: %w", err) + } + log.Ctx(ctx).Infof("Enriched %d SAC contract_tokens rows after checkpoint load", len(sacContracts)) return nil } @@ -468,20 +513,15 @@ func (p *checkpointProcessor) flushRemainingBatch(ctx context.Context) error { // finalize identifies SEP-41 contracts, fetches metadata, stores tokens in DB, // and persists protocol WASMs and contracts. func (p *checkpointProcessor) finalize(ctx context.Context, dbTx pgx.Tx) error { - // Identify SAC contracts missing code/issuer and fetch metadata via RPC - var sacContractsNeedingMetadata []string + // Identify SAC contracts missing code/issuer. Their rows are stored below + // with ledger-derived defaults (Code/Name/Symbol/Decimals unset); metadata + // is fetched via RPC and applied afterward, in a short follow-up + // transaction once this load has committed (see PopulateFromCheckpoint / + // enrichSACMetadata) so RPC round-trips never extend this transaction's + // row locks. for _, contract := range p.data.uniqueContractTokens { if contract.Type == string(types.ContractTypeSAC) && contract.Code == nil { - sacContractsNeedingMetadata = append(sacContractsNeedingMetadata, contract.ContractID) - } - } - if len(sacContractsNeedingMetadata) > 0 { - sacContracts, err := p.service.contractMetadataService.FetchSACMetadata(ctx, sacContractsNeedingMetadata) - if err != nil { - return fmt.Errorf("fetching SAC metadata: %w", err) - } - for _, contract := range sacContracts { - p.data.uniqueContractTokens[contract.ID] = contract + p.pendingSACMetadata = append(p.pendingSACMetadata, contract.ContractID) } } diff --git a/internal/services/checkpoint_test.go b/internal/services/checkpoint_test.go index c76a4cb15..811b4ce3d 100644 --- a/internal/services/checkpoint_test.go +++ b/internal/services/checkpoint_test.go @@ -371,6 +371,150 @@ func TestCheckpointService_PopulateFromCheckpoint_ContractDataEntry(t *testing.T require.NoError(t, err) } +// makeSACBalanceChange builds an ingest.Change for a ContractData Balance +// entry whose holder is itself a contract — the shape +// sac.ContractBalanceFromContractData requires to recognize a SAC balance. +// With no preceding contract-instance entry in the same checkpoint, this is +// how a SAC contract ends up in contract_tokens with Code/Name/Symbol unset, +// needing RPC enrichment (see finalize's pendingSACMetadata bookkeeping). +func makeSACBalanceChange(tokenContractHash, holderContractHash [32]byte) ingest.Change { + return ingest.Change{ + Type: xdr.LedgerEntryTypeContractData, + Post: &xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeContractData, + ContractData: &xdr.ContractDataEntry{ + Contract: xdr.ScAddress{ + Type: xdr.ScAddressTypeScAddressTypeContract, + ContractId: (*xdr.ContractId)(&tokenContractHash), + }, + Key: xdr.ScVal{ + Type: xdr.ScValTypeScvVec, + Vec: ptrToScVec([]xdr.ScVal{ + {Type: xdr.ScValTypeScvSymbol, Sym: ptrToScSymbol("Balance")}, + { + Type: xdr.ScValTypeScvAddress, + Address: &xdr.ScAddress{ + Type: xdr.ScAddressTypeScAddressTypeContract, + ContractId: (*xdr.ContractId)(&holderContractHash), + }, + }, + }), + }, + Durability: xdr.ContractDataDurabilityPersistent, + Val: makeBalanceMapVal(500, true, false), + }, + }, + }, + } +} + +// checkpointCommitProbeKey is a scratch ingest_store row written by an +// initializeCursors callback, inside the load's own transaction, purely so a +// test can prove — via a real, separate connection — whether the load +// transaction has actually committed yet. Postgres MVCC hides the row from +// any other connection until commit, which is what makes this a genuine +// after-commit check rather than an assertion on mock call ordering alone. +const checkpointCommitProbeKey = "checkpoint_commit_probe" + +func writeCheckpointCommitProbe(dbTx pgx.Tx) error { + _, err := dbTx.Exec(context.Background(), + `INSERT INTO ingest_store (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value`, + checkpointCommitProbeKey, "committed") + return err +} + +// TestCheckpointService_PopulateFromCheckpoint_SACMetadataEnrichedAfterCommit +// is the ING-10 regression test: it proves FetchSACMetadata (the RPC call) +// only happens once the load transaction — including cursor initialization — +// has already committed, by querying the commit probe row from the real DB +// pool (a connection independent of the load's transaction) from inside the +// FetchSACMetadata mock's callback. If the RPC call were still made inside +// the load transaction (the ING-10 bug), the probe row would not yet be +// visible and this assertion would fail. +func TestCheckpointService_PopulateFromCheckpoint_SACMetadataEnrichedAfterCommit(t *testing.T) { + f := setupCheckpointTest(t) + + tokenHash := [32]byte{9, 9, 9} + holderHash := [32]byte{8, 8, 8} + change := makeSACBalanceChange(tokenHash, holderHash) + tokenAddr := strkey.MustEncode(strkey.VersionByteContract, tokenHash[:]) + + f.reader.On("Read").Return(change, nil).Once() + f.reader.On("Read").Return(ingest.Change{}, io.EOF).Once() + f.reader.On("Close").Return(nil).Once() + + f.trustlineBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.MatchedBy(func(b []wbdata.TrustlineBalance) bool { return len(b) == 0 })).Return(nil).Once() + f.nativeBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.MatchedBy(func(b []wbdata.NativeBalance) bool { return len(b) == 0 })).Return(nil).Once() + f.sacBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.MatchedBy(func(b []wbdata.SACBalance) bool { return len(b) == 1 })).Return(nil).Once() + + f.contractModel.On("BatchInsert", mock.Anything, mock.Anything, mock.MatchedBy(func(cs []*wbdata.Contract) bool { + return len(cs) == 1 && cs[0].ContractID == tokenAddr && cs[0].Type == string(types.ContractTypeSAC) && cs[0].Code == nil + })).Return(nil).Once() + + enrichedName := "Test Token" + enrichedSymbol := "TST" + f.contractMetadataService.On("FetchSACMetadata", mock.Anything, []string{tokenAddr}). + Run(func(_ mock.Arguments) { + var value string + queryErr := f.svc.db.QueryRow(context.Background(), + `SELECT value FROM ingest_store WHERE key = $1`, checkpointCommitProbeKey).Scan(&value) + require.NoError(t, queryErr, "the load transaction (including cursor init) must already be committed before SAC metadata enrichment runs") + assert.Equal(t, "committed", value) + }). + Return([]*wbdata.Contract{{ + ID: wbdata.DeterministicContractID(tokenAddr), + ContractID: tokenAddr, + Type: string(types.ContractTypeSAC), + Name: &enrichedName, + Symbol: &enrichedSymbol, + }}, nil).Once() + + f.contractModel.On("BatchUpdateMetadata", mock.Anything, mock.Anything, mock.MatchedBy(func(cs []*wbdata.Contract) bool { + return len(cs) == 1 && cs[0].ContractID == tokenAddr && cs[0].Name != nil && *cs[0].Name == enrichedName + })).Return(nil).Once() + + err := f.svc.PopulateFromCheckpoint(context.Background(), 100, writeCheckpointCommitProbe) + require.NoError(t, err) +} + +// TestCheckpointService_PopulateFromCheckpoint_SACMetadataFetchFailureDoesNotFailLoad +// is the other half of ING-10: a failed SAC metadata fetch must be logged and +// leave the already-committed load's rows in place (defaults, unenriched), +// not fail PopulateFromCheckpoint. +func TestCheckpointService_PopulateFromCheckpoint_SACMetadataFetchFailureDoesNotFailLoad(t *testing.T) { + f := setupCheckpointTest(t) + + tokenHash := [32]byte{7, 7, 7} + holderHash := [32]byte{6, 6, 6} + change := makeSACBalanceChange(tokenHash, holderHash) + + f.reader.On("Read").Return(change, nil).Once() + f.reader.On("Read").Return(ingest.Change{}, io.EOF).Once() + f.reader.On("Close").Return(nil).Once() + + f.trustlineBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + f.nativeBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + f.sacBalanceModel.On("BatchCopy", mock.Anything, mock.Anything, mock.Anything).Return(nil).Maybe() + + f.contractModel.On("BatchInsert", mock.Anything, mock.Anything, mock.Anything).Return(nil).Once() + + f.contractMetadataService.On("FetchSACMetadata", mock.Anything, mock.Anything). + Return(nil, errors.New("rpc unavailable")).Once() + + // No "BatchUpdateMetadata" expectation is registered: if enrichSACMetadata + // called it despite the fetch failing, the mock would fail this test for + // an unexpected call. + + cursorsCalled := false + err := f.svc.PopulateFromCheckpoint(context.Background(), 100, func(_ pgx.Tx) error { + cursorsCalled = true + return nil + }) + require.NoError(t, err, "a SAC metadata fetch failure must not fail the completed load") + assert.True(t, cursorsCalled) +} + func TestCheckpointService_PopulateFromCheckpoint_ErrorPropagation(t *testing.T) { tests := []struct { name string diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index c0030f145..11ec71755 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -30,11 +30,16 @@ const ( // persistLedgerData persists processed ledger data to the database in a single // atomic transaction. It handles: trustline assets, contract tokens, filtered -// data insertion, token changes, and cursor update. +// data insertion, token changes, and cursor update. plan is this ledger's +// classification plan, computed by prepareClassificationPlan before any +// transaction opens (RPC calls already resolved); pass the same plan across +// ingestProcessedDataWithRetry's retry attempts so a retry never re-issues +// RPC calls. plan may be nil when there was nothing to classify this ledger. func (m *ingestService) persistLedgerData( ctx context.Context, ledgerSeq uint32, ledgerMeta *xdr.LedgerCloseMeta, + plan *ClassificationPlan, buffer *indexer.IndexerBuffer, cursorName string, ) (int, int, error) { @@ -61,14 +66,15 @@ func (m *ingestService) persistLedgerData( log.Ctx(ctx).Infof("inserted %d SAC contract tokens", len(contracts)) } - // 2.5: Run protocol classification (black-box per protocol). Per- - // protocol side effects (e.g. SEP-41 contract_tokens metadata) happen - // inside this same dbTx via the validators' Validate calls. Live - // protocol processors then stage ledger state from the classification - // result before the generic protocol_wasms / protocol_contracts rows - // are persisted below. + // 2.5: Apply protocol classification (black-box per protocol). plan was + // computed by prepareClassificationPlan before this transaction opened, + // so any RPC calls (e.g. SEP-41 metadata) already happened; + // ApplyClassificationPlan only performs each validator's DB writes + // here, atomically with the classification verdict and wasm/contract + // rows below. Live protocol processors then stage ledger state from + // the classification result before the generic protocol_wasms / + // protocol_contracts rows are persisted below. bufferedWasms := buffer.GetProtocolWasms() - bufferedBytecodes := buffer.GetProtocolWasmBytecodes() bufferedContracts := buffer.GetProtocolContracts() contractSlice := make([]data.ProtocolContracts, 0, len(bufferedContracts)) @@ -76,9 +82,12 @@ func (m *ingestService) persistLedgerData( contractSlice = append(contractSlice, c) } - classification, classifyErr := m.runClassification(ctx, dbTx, bufferedWasms, bufferedBytecodes, contractSlice) - if classifyErr != nil { - return fmt.Errorf("classifying ledger %d: %w", ledgerSeq, classifyErr) + var classification map[types.HashBytea]string + if plan != nil { + classification = plan.Matches + } + if txErr = ApplyClassificationPlan(ctx, dbTx, m.models, plan, m.appMetrics.Ingestion.WasmClassificationFailuresTotal); txErr != nil { + return fmt.Errorf("applying classification for ledger %d: %w", ledgerSeq, txErr) } // 2.6: Per-protocol CAS-gated state production. The compare-and-swap on each @@ -356,9 +365,21 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 } m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("process_ledger").Observe(time.Since(processStart).Seconds()) + // Classification runs once here, entirely before any database + // transaction opens (RPC prefetch happens now, not while row locks are + // held), and the resulting plan is reused verbatim across every retry + // attempt below. + classifyStart := time.Now() + plan, err := m.prepareClassificationPlan(ctx, buffer.GetProtocolWasms(), buffer.GetProtocolWasmBytecodes(), buffer.GetProtocolContracts()) + if err != nil { + m.appMetrics.Ingestion.ErrorsTotal.WithLabelValues("ingest_live").Inc() + return fmt.Errorf("preparing classification plan for ledger %d: %w", currentLedger, err) + } + m.appMetrics.Ingestion.PhaseDuration.WithLabelValues("prepare_classification").Observe(time.Since(classifyStart).Seconds()) + // All DB operations in a single atomic transaction with retry dbStart := time.Now() - numTransactionProcessed, numOperationProcessed, err := m.ingestProcessedDataWithRetry(ctx, currentLedger, ledgerMeta, buffer) + numTransactionProcessed, numOperationProcessed, err := m.ingestProcessedDataWithRetry(ctx, currentLedger, ledgerMeta, plan, buffer) if err != nil { m.appMetrics.Ingestion.ErrorsTotal.WithLabelValues("ingest_live").Inc() return fmt.Errorf("processing ledger %d: %w", currentLedger, err) @@ -462,10 +483,14 @@ func getEffectiveProtocolContracts( } // ingestProcessedDataWithRetry wraps persistLedgerData with retry logic. +// plan was computed once by the caller before this loop started and is reused +// verbatim across every attempt, so a retried attempt never re-issues the +// classification RPC calls plan already resolved. func (m *ingestService) ingestProcessedDataWithRetry( ctx context.Context, currentLedger uint32, ledgerMeta xdr.LedgerCloseMeta, + plan *ClassificationPlan, buffer *indexer.IndexerBuffer, ) (int, int, error) { var lastErr error @@ -476,7 +501,7 @@ func (m *ingestService) ingestProcessedDataWithRetry( default: } - numTxs, numOps, err := m.persistLedgerData(ctx, currentLedger, &ledgerMeta, buffer, data.LatestLedgerCursorName) + numTxs, numOps, err := m.persistLedgerData(ctx, currentLedger, &ledgerMeta, plan, buffer, data.LatestLedgerCursorName) if err == nil { return numTxs, numOps, nil } @@ -504,23 +529,26 @@ func (m *ingestService) ingestProcessedDataWithRetry( return 0, 0, fmt.Errorf("ingesting processed data failed after %d attempts: %w", maxIngestProcessedDataRetries, lastErr) } -// runClassification builds a ValidationInput from this ledger's buffered -// raw WASMs and contracts and dispatches to each registered ProtocolValidator -// in priority order. It returns a wasm hash → protocol_id map covering both -// previously-committed classifications and this-ledger matches, so live -// ProcessLedger can reason about same-ledger deploy/upgrade events before -// protocol_contracts is committed. +// prepareClassificationPlan runs Phase A of protocol classification for this +// ledger's buffered raw WASMs and contracts: pure signature matching plus any +// RPC-sourced enrichment prefetch (e.g. SEP-41 token metadata), +// entirely before any database transaction opens. Callers must compute this +// once per ledger and reuse the same plan across retry attempts (like +// buffer already is) — recomputing it would re-issue RPC calls +// on every retry. ApplyClassificationPlan (called from inside +// persistLedgerData's transaction) finishes the job with DB-only writes. // -// Validator side effects (e.g. SEP-41 contract_tokens metadata writes) happen -// inside dbTx via the validators themselves and commit atomically with the -// classification verdict. -func (m *ingestService) runClassification( +// The known-classification lookup is a non-transactional pool read: +// staleness is harmless (this-ledger uploads resolve from the buffer; prior +// rows are immutable once classified), and reading before any transaction +// opens means it never contends with the CAS/cursor row locks +// persistLedgerData holds later. +func (m *ingestService) prepareClassificationPlan( ctx context.Context, - dbTx pgx.Tx, bufferedWasms map[string]data.ProtocolWasms, bufferedBytecodes map[string][]byte, - bufferedContracts []data.ProtocolContracts, -) (map[types.HashBytea]string, error) { + bufferedContracts map[string]data.ProtocolContracts, +) (*ClassificationPlan, error) { if len(bufferedWasms) == 0 && len(bufferedContracts) == 0 { return nil, nil } @@ -533,44 +561,42 @@ func (m *ingestService) runClassification( thisBatch[h] = struct{}{} } - // Resolve hashes classified in earlier ledgers, skipping this-ledger - // uploads (thisBatch) which the dispatcher classifies below. - knownHashes := make([]types.HashBytea, 0, len(bufferedContracts)) + contractSlice := make([]data.ProtocolContracts, 0, len(bufferedContracts)) for _, c := range bufferedContracts { + contractSlice = append(contractSlice, c) + } + + // Resolve hashes classified in earlier ledgers, skipping this-ledger + // uploads (thisBatch) which PrepareClassification matches below. + knownHashes := make([]types.HashBytea, 0, len(contractSlice)) + for _, c := range contractSlice { if _, inBatch := thisBatch[c.WasmHash]; inBatch { continue } knownHashes = append(knownHashes, c.WasmHash) } - known, err := m.models.ProtocolWasms.GetClassifiedByHashes(ctx, dbTx, knownHashes) + known, err := m.models.ProtocolWasms.GetClassifiedByHashes(ctx, m.models.DB, knownHashes) if err != nil { return nil, fmt.Errorf("resolving known protocol classifications: %w", err) } - // protocolByWasm answers "what protocol owns this wasm hash?": previously - // committed classifications overlaid with this-ledger matches. The two key - // sets are disjoint (knownHashes excludes thisBatch), so the overlay never - // actually collides. - protocolByWasm := make(map[types.HashBytea]string, len(known)) - for hash, pid := range known { - protocolByWasm[hash] = pid - } if len(m.protocolValidators) == 0 { - return protocolByWasm, nil + plan := &ClassificationPlan{Matches: make(map[types.HashBytea]string, len(known))} + for hash, pid := range known { + plan.Matches[hash] = pid + } + return plan, nil } - matches, err := DispatchClassification( - ctx, dbTx, m.wasmSpecExtractor, m.protocolValidators, - bytecodesByHash, bufferedContracts, m.rpcService, m.models, known, + plan, err := PrepareClassification( + ctx, m.wasmSpecExtractor, m.protocolValidators, + bytecodesByHash, contractSlice, m.rpcService, known, m.appMetrics.Ingestion.WasmClassificationFailuresTotal, ) if err != nil { - return nil, fmt.Errorf("dispatching classification: %w", err) - } - for hash, pid := range matches { - protocolByWasm[hash] = pid + return nil, fmt.Errorf("preparing classification: %w", err) } - return protocolByWasm, nil + return plan, nil } // prepareNewSACContracts filters out existing contracts and returns new SAC contracts for insertion. diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 69b0e221d..97188b27e 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1605,7 +1605,7 @@ func Test_ingestProcessedDataWithRetry(t *testing.T) { // Call ingestProcessedDataWithRetry - should succeed // Note: assetIDMap and contractIDMap are no longer passed - operations use direct DB queries - numTx, numOps, err := svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, buffer) + numTx, numOps, err := svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, nil, buffer) // Verify success require.NoError(t, err) @@ -1686,7 +1686,7 @@ func Test_ingestProcessedDataWithRetry(t *testing.T) { // Call ingestProcessedDataWithRetry - should fail after retries due to DB error // Note: assetIDMap and contractIDMap are no longer passed - operations use direct DB queries - _, _, err = svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, buffer) + _, _, err = svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, nil, buffer) // Verify error propagates with retry failure message require.Error(t, err) @@ -1776,7 +1776,7 @@ func Test_ingestProcessedDataWithRetry(t *testing.T) { // Call ingestProcessedDataWithRetry - should succeed after retry // Note: assetIDMap and contractIDMap are no longer passed - operations use direct DB queries - numTx, numOps, err := svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, buffer) + numTx, numOps, err := svc.ingestProcessedDataWithRetry(ctx, 100, xdr.LedgerCloseMeta{}, nil, buffer) // Verify success after retry require.NoError(t, err) @@ -1972,7 +1972,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.NoError(t, err) // Both protocol cursors should advance to 100 @@ -2004,7 +2004,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.NoError(t, err) // Cursors should stay at 100 (CAS expected 99 but found 100) @@ -2036,7 +2036,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.NoError(t, err) // Cursors should stay at 98 (behind, so entire block is skipped) @@ -2073,7 +2073,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.NoError(t, err) // Main cursor advances; protocol persist methods were not called and @@ -2110,7 +2110,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { require.NoError(t, err) meta := dummyLedgerMeta(100) - _, _, err = svc.persistLedgerData(ctx, 100, &meta, indexer.NewIndexerBuffer(), "latest_ledger_cursor") + _, _, err = svc.persistLedgerData(ctx, 100, &meta, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") require.NoError(t, err) // History CAS succeeded. @@ -2149,7 +2149,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { require.NoError(t, err) meta := dummyLedgerMeta(100) - _, _, err = svc.persistLedgerData(ctx, 100, &meta, indexer.NewIndexerBuffer(), "latest_ledger_cursor") + _, _, err = svc.persistLedgerData(ctx, 100, &meta, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") require.NoError(t, err) // Current-state CAS succeeded. @@ -2178,7 +2178,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.NoError(t, err) // Main cursor should advance @@ -2197,14 +2197,14 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { // First ledger succeeds and advances the current-state cursor to 100. processor.processedLedger = 100 meta100 := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta100, indexer.NewIndexerBuffer(), "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta100, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") require.NoError(t, err) // Next ledger fails inside PersistCurrentState, rolling back the whole // transaction — the current-state cursor must stay at 100. processor.processedLedger = 101 meta101 := dummyLedgerMeta(101) - _, _, err = svc.persistLedgerData(ctx, 101, &meta101, indexer.NewIndexerBuffer(), "latest_ledger_cursor") + _, _, err = svc.persistLedgerData(ctx, 101, &meta101, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") require.Error(t, err) currentStateCursor, err := models.IngestStore.Get(ctx, "protocol_testproto_current_state_cursor") @@ -2214,7 +2214,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { // Retrying the same ledger succeeds and advances the cursor. processor.failPersistCurrentStateAt = 0 processor.processedLedger = 101 - _, _, err = svc.persistLedgerData(ctx, 101, &meta101, indexer.NewIndexerBuffer(), "latest_ledger_cursor") + _, _, err = svc.persistLedgerData(ctx, 101, &meta101, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") require.NoError(t, err) currentStateCursor, err = models.IngestStore.Get(ctx, "protocol_testproto_current_state_cursor") @@ -2249,7 +2249,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { ) meta := dummyLedgerMeta(100) - _, _, err := svc.persistLedgerData(ctx, 100, &meta, buffer, "latest_ledger_cursor") + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") require.ErrorContains(t, err, "resolving protocol contracts for ledger 100") // The transaction rolled back: the protocol history cursor stayed at 99. diff --git a/internal/services/protocol_setup.go b/internal/services/protocol_setup.go index f40ca3131..2963e7356 100644 --- a/internal/services/protocol_setup.go +++ b/internal/services/protocol_setup.go @@ -173,10 +173,12 @@ func (s *protocolSetupService) classify(ctx context.Context) error { return fmt.Errorf("loading protocol_contracts for unclassified wasms: %w", err) } - // Run dispatcher and persist matches inside one tx so a validator's side - // effects (e.g. contract_tokens metadata writes) commit atomically with - // the protocol_wasms.protocol_id stamp. - byProtocol, err := s.dispatchAndPersist(ctx, bytecodesByHash, contracts) + // Prepare classification (matching plus any RPC-sourced enrichment + // prefetch) before opening any transaction, then persist matches and each + // validator's DB writes inside one tx so they commit atomically with the + // protocol_wasms.protocol_id stamp. No RPC handle is available past this + // point. + byProtocol, err := s.prepareAndPersist(ctx, bytecodesByHash, contracts) if err != nil { return fmt.Errorf("dispatching classification: %w", err) } @@ -187,24 +189,30 @@ func (s *protocolSetupService) classify(ctx context.Context) error { return nil } -// dispatchAndPersist runs DispatchClassification inside a single tx and -// updates protocol_wasms.protocol_id from the matches, returning them grouped -// per protocol. Per-protocol contract_tokens writes happen inside this same tx -// via validator side effects. -func (s *protocolSetupService) dispatchAndPersist(ctx context.Context, bytecodesByHash map[types.HashBytea][]byte, contracts []data.ProtocolContracts) (map[string][]types.HashBytea, error) { +// prepareAndPersist runs PrepareClassification (pure matching plus RPC +// prefetch) before opening any transaction, then applies the resulting plan +// and updates protocol_wasms.protocol_id inside a single tx, returning the +// matches grouped per protocol. Per-protocol DB writes (e.g. contract_tokens +// metadata) happen inside this same tx via ApplyClassificationPlan; no RPC +// call happens once the transaction opens. +func (s *protocolSetupService) prepareAndPersist(ctx context.Context, bytecodesByHash map[types.HashBytea][]byte, contracts []data.ProtocolContracts) (map[string][]types.HashBytea, error) { + // All candidates here are unclassified, so KnownProtocolID is empty. + var knownByHash map[types.HashBytea]string + plan, err := PrepareClassification( + ctx, s.specExtractor, s.validators, + bytecodesByHash, contracts, s.rpcService, knownByHash, + s.wasmClassificationFailuresTotal, + ) + if err != nil { + return nil, fmt.Errorf("preparing classification: %w", err) + } + var byProtocol map[string][]types.HashBytea - err := db.RunInTransaction(ctx, s.db, func(dbTx pgx.Tx) error { - // All candidates here are unclassified, so KnownProtocolID is empty. - var knownByHash map[types.HashBytea]string - matches, dispatchErr := DispatchClassification( - ctx, dbTx, s.specExtractor, s.validators, - bytecodesByHash, contracts, s.rpcService, s.models, knownByHash, - s.wasmClassificationFailuresTotal, - ) - if dispatchErr != nil { - return fmt.Errorf("dispatching: %w", dispatchErr) + err = db.RunInTransaction(ctx, s.db, func(dbTx pgx.Tx) error { + if applyErr := ApplyClassificationPlan(ctx, dbTx, s.models, plan, s.wasmClassificationFailuresTotal); applyErr != nil { + return fmt.Errorf("applying classification: %w", applyErr) } - byProtocol = bucketByProtocol(matches) + byProtocol = bucketByProtocol(plan.Matches) for protocolID, hashes := range byProtocol { if err := s.models.ProtocolWasms.BatchUpdateProtocolID(ctx, dbTx, hashes, protocolID); err != nil { return fmt.Errorf("updating protocol_id for %s: %w", protocolID, err) diff --git a/internal/services/protocol_setup_test.go b/internal/services/protocol_setup_test.go index 8acd058ab..ef73b3c4f 100644 --- a/internal/services/protocol_setup_test.go +++ b/internal/services/protocol_setup_test.go @@ -39,18 +39,23 @@ func newStubValidator(claim ...types.HashBytea) *stubValidator { func (s *stubValidator) ProtocolID() string { return testProtocolID } -func (s *stubValidator) Validate(_ context.Context, _ pgx.Tx, input ValidationInput) (ValidationResult, error) { - s.gotBatch++ - if s.classifyErr != nil { - return ValidationResult{}, s.classifyErr - } - out := ValidationResult{} - for _, c := range input.Candidates { +func (s *stubValidator) Match(candidates []WasmCandidate) map[types.HashBytea]struct{} { + matched := map[types.HashBytea]struct{}{} + for _, c := range candidates { if _, ok := s.claim[c.Hash]; ok { - out.MatchedWasms = append(out.MatchedWasms, c.Hash) + matched[c.Hash] = struct{}{} } } - return out, nil + return matched +} + +func (s *stubValidator) Prefetch(_ context.Context, _ RPCService, _ []WasmCandidate, _ map[types.HashBytea]struct{}, _ []ContractCandidate) (any, error) { + s.gotBatch++ + return nil, s.classifyErr +} + +func (s *stubValidator) Apply(_ context.Context, _ pgx.Tx, _ map[types.HashBytea]struct{}, _ []ContractCandidate, _ any, _ *data.Models) error { + return nil } func TestProtocolSetupService_Run(t *testing.T) { diff --git a/internal/services/protocol_validation_dispatch.go b/internal/services/protocol_validation_dispatch.go index dbc5af48f..a0d5ba043 100644 --- a/internal/services/protocol_validation_dispatch.go +++ b/internal/services/protocol_validation_dispatch.go @@ -12,48 +12,78 @@ import ( "github.com/stellar/wallet-backend/internal/indexer/types" ) -// DispatchClassification runs each validator against the supplied batch in -// the supplied order, enforcing first-match-wins. The caller is responsible -// for persisting protocol_wasms / protocol_contracts based on the returned -// matches; the dispatcher only wires the per-protocol seam. +// ClassificationPlan is the RPC-free, ready-to-apply output of +// PrepareClassification: the final hash → protocol_id map for a batch (prior +// classifications overlaid with this batch's new matches) plus, for each +// validator that had anything to act on, the RPC-sourced enrichment it +// resolved via Prefetch. Building a plan opens no database transaction and, +// once built, requires no further RPC calls — ApplyClassificationPlan +// consumes it purely against dbTx. A plan is safe to reuse verbatim across +// retry attempts of the same ledger. +type ClassificationPlan struct { + // Matches is the full hash -> protocol_id map: the knownByHash overlay + // passed to PrepareClassification, overlaid with this batch's new + // first-match-wins matches. + Matches map[types.HashBytea]string + + perValidator []validatorPlan +} + +// validatorPlan pairs one validator with the matched-hash set and RPC-sourced +// enrichment PrepareClassification resolved for it, ready for +// ApplyClassificationPlan to persist. +type validatorPlan struct { + validator ProtocolValidator + matched map[types.HashBytea]struct{} + contracts []ContractCandidate + prefetch any +} + +// PrepareClassification runs pure signature matching for every candidate WASM +// against every validator in priority order (first-match-wins), then calls +// each relevant validator's Prefetch to resolve RPC-sourced enrichment. No +// database transaction is opened and no RPC call happens after this function +// returns — ApplyClassificationPlan finishes the job purely against dbTx. // // Spec extraction is performed once per candidate WASM. A candidate whose // spec cannot be extracted (e.g. a hostile blob that fails wazero validation) // is logged, counted, and dropped from the candidate set — no signature-based // validator can act on a spec-less candidate. The caller still persists the -// underlying wasm with protocol_id = NULL because it is absent from the -// returned matches. +// underlying wasm with protocol_id = NULL because it is absent from +// plan.Matches. // -// A validator that returns an error (or panics) aborts classification for the -// entire batch: the error propagates to the caller so its surrounding -// RunInTransaction rolls back. Validators may write to the shared dbTx during -// Validate (e.g. contract_tokens enrichment), so a failure cannot be treated -// as a best-effort no-match — that would commit the failed validator's partial -// writes. Any failure must roll back the whole transaction. -func DispatchClassification( +// An error from a validator's Prefetch aborts the whole batch (the plan is +// discarded, matching DispatchClassification's previous fail-fast contract +// for validator-level errors); in practice today's validators never return a +// hard Prefetch error — per-contract RPC failures are absorbed internally +// and reflected as missing entries in the plan instead. +func PrepareClassification( ctx context.Context, - dbTx pgx.Tx, extractor WasmSpecExtractor, validators []ProtocolValidator, bytecodesByHash map[types.HashBytea][]byte, contracts []data.ProtocolContracts, rpc RPCService, - models *data.Models, knownByHash map[types.HashBytea]string, failureCounter *prometheus.CounterVec, -) (map[types.HashBytea]string, error) { +) (*ClassificationPlan, error) { if extractor == nil { return nil, fmt.Errorf("dispatch: spec extractor required") } + + plan := &ClassificationPlan{Matches: make(map[types.HashBytea]string, len(knownByHash))} + for hash, pid := range knownByHash { + plan.Matches[hash] = pid + } if len(bytecodesByHash) == 0 && len(contracts) == 0 { - return nil, nil + return plan, nil } candidates := make([]WasmCandidate, 0, len(bytecodesByHash)) for hash, bytecode := range bytecodesByHash { specs, err := extractor.ExtractSpec(ctx, bytecode) if err != nil { - log.Ctx(ctx).Warnf("validation dispatch: spec extraction failed for wasm %s: %v", hash, err) + log.Ctx(ctx).Warnf("classification prepare: spec extraction failed for wasm %s: %v", hash, err) if failureCounter != nil { failureCounter.WithLabelValues("unknown", "spec_extraction_error").Inc() } @@ -64,33 +94,59 @@ func DispatchClassification( annotated := annotateContracts(contracts, knownByHash) - matches := make(map[types.HashBytea]string) for _, v := range validators { - filtered := filterCandidates(candidates, matches) + filtered := filterCandidates(candidates, plan.Matches) if len(filtered) == 0 && len(annotated) == 0 { break } - input := ValidationInput{ - Candidates: filtered, - Contracts: annotated, - RPC: rpc, - Models: models, + + matched := v.Match(filtered) + for hash := range matched { + if _, alreadyClaimed := plan.Matches[hash]; alreadyClaimed { + continue + } + plan.Matches[hash] = v.ProtocolID() } - result, err := v.Validate(ctx, dbTx, input) + + prefetch, err := v.Prefetch(ctx, rpc, filtered, matched, annotated) if err != nil { if failureCounter != nil { + // Reuses the "validate_error" label from before the + // Match/Prefetch/Apply split so existing dashboards/alerts + // on this counter keep working unchanged. failureCounter.WithLabelValues(v.ProtocolID(), "validate_error").Inc() } - return nil, fmt.Errorf("validator %s: %w", v.ProtocolID(), err) + return nil, fmt.Errorf("validator %s: prefetch: %w", v.ProtocolID(), err) } - for _, hash := range result.MatchedWasms { - if _, alreadyClaimed := matches[hash]; alreadyClaimed { - continue + plan.perValidator = append(plan.perValidator, validatorPlan{ + validator: v, + matched: matched, + contracts: annotated, + prefetch: prefetch, + }) + } + return plan, nil +} + +// ApplyClassificationPlan persists each validator's classification side +// effects (e.g. contract_tokens) inside dbTx, using only the +// enrichment data PrepareClassification already resolved via RPC. No RPC +// handle is reachable from here — this is a compile-level guarantee: Apply +// has no RPCService parameter, so a validator cannot make a network call +// while dbTx (and the row locks it holds) is open. +func ApplyClassificationPlan(ctx context.Context, dbTx pgx.Tx, models *data.Models, plan *ClassificationPlan, failureCounter *prometheus.CounterVec) error { + if plan == nil { + return nil + } + for _, vp := range plan.perValidator { + if err := vp.validator.Apply(ctx, dbTx, vp.matched, vp.contracts, vp.prefetch, models); err != nil { + if failureCounter != nil { + failureCounter.WithLabelValues(vp.validator.ProtocolID(), "validate_error").Inc() } - matches[hash] = v.ProtocolID() + return fmt.Errorf("validator %s: apply: %w", vp.validator.ProtocolID(), err) } } - return matches, nil + return nil } // filterCandidates returns the subset of candidates whose hash is not yet diff --git a/internal/services/protocol_validation_dispatch_test.go b/internal/services/protocol_validation_dispatch_test.go index ddca063ba..94659c111 100644 --- a/internal/services/protocol_validation_dispatch_test.go +++ b/internal/services/protocol_validation_dispatch_test.go @@ -18,15 +18,20 @@ import ( "github.com/stellar/wallet-backend/internal/indexer/types" ) -// recordingValidator captures the inputs each Validate call observed and -// returns a configurable set of matches. +// recordingValidator captures the inputs each phase call observed and +// returns a configurable set of matches. Its Apply method signature has no +// RPCService parameter at all — the same compile-level guarantee production +// validators get from the real services.ProtocolValidator interface. type recordingValidator struct { - mu sync.Mutex - id string - matches map[types.HashBytea]struct{} - err error - calls int - lastInput ValidationInput + mu sync.Mutex + id string + matches map[types.HashBytea]struct{} + prefetchErr error + applyErr error + matchCalls int + prefetchCalls int + applyCalls int + lastContracts []ContractCandidate } func newRecordingValidator(id string, matches ...types.HashBytea) *recordingValidator { @@ -39,49 +44,64 @@ func newRecordingValidator(id string, matches ...types.HashBytea) *recordingVali func (r *recordingValidator) ProtocolID() string { return r.id } -func (r *recordingValidator) Validate(_ context.Context, _ pgx.Tx, input ValidationInput) (ValidationResult, error) { +func (r *recordingValidator) Match(candidates []WasmCandidate) map[types.HashBytea]struct{} { r.mu.Lock() defer r.mu.Unlock() - r.calls++ - r.lastInput = input - if r.err != nil { - return ValidationResult{}, r.err - } - out := ValidationResult{} - for _, c := range input.Candidates { + r.matchCalls++ + out := map[types.HashBytea]struct{}{} + for _, c := range candidates { if _, ok := r.matches[c.Hash]; ok { - out.MatchedWasms = append(out.MatchedWasms, c.Hash) + out[c.Hash] = struct{}{} } } - return out, nil + return out } -func TestDispatchClassification(t *testing.T) { +func (r *recordingValidator) Prefetch(_ context.Context, _ RPCService, _ []WasmCandidate, _ map[types.HashBytea]struct{}, contracts []ContractCandidate) (any, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.prefetchCalls++ + r.lastContracts = contracts + if r.prefetchErr != nil { + return nil, r.prefetchErr + } + return nil, nil +} + +func (r *recordingValidator) Apply(_ context.Context, _ pgx.Tx, _ map[types.HashBytea]struct{}, _ []ContractCandidate, _ any, _ *data.Models) error { + r.mu.Lock() + defer r.mu.Unlock() + r.applyCalls++ + return r.applyErr +} + +func TestPrepareClassification(t *testing.T) { ctx := context.Background() t.Run("nil extractor returns error", func(t *testing.T) { hash := types.HashBytea("aabb") c := newRecordingValidator("A") - matches, err := DispatchClassification( - ctx, nil, nil, + plan, err := PrepareClassification( + ctx, nil, []ProtocolValidator{c}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, nil, + nil, nil, nil, nil, ) require.Error(t, err) - assert.Nil(t, matches) - assert.Equal(t, 0, c.calls) + assert.Nil(t, plan) + assert.Equal(t, 0, c.matchCalls) }) - t.Run("no candidates and no contracts returns nil", func(t *testing.T) { + t.Run("no candidates and no contracts returns an empty plan", func(t *testing.T) { c := newRecordingValidator("A") extractor := NewWasmSpecExtractorMock(t) - matches, err := DispatchClassification(ctx, nil, extractor, []ProtocolValidator{c}, nil, nil, nil, nil, nil, nil) + plan, err := PrepareClassification(ctx, extractor, []ProtocolValidator{c}, nil, nil, nil, nil, nil) require.NoError(t, err) - assert.Nil(t, matches) - assert.Equal(t, 0, c.calls) + require.NotNil(t, plan) + assert.Empty(t, plan.Matches) + assert.Equal(t, 0, c.matchCalls) }) t.Run("first match wins", func(t *testing.T) { @@ -92,19 +112,21 @@ func TestDispatchClassification(t *testing.T) { extractor := NewWasmSpecExtractorMock(t) extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() - matches, err := DispatchClassification( - ctx, nil, extractor, + plan, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{cA, cB}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, nil, + nil, nil, nil, nil, ) require.NoError(t, err) - assert.Equal(t, "A", matches[hash]) - - // B saw filtered candidates (none, since A claimed the only one). - assert.Equal(t, 1, cA.calls) - // B is not called when there are no remaining candidates AND no contracts. - assert.Equal(t, 0, cB.calls) + assert.Equal(t, "A", plan.Matches[hash]) + + // B saw filtered candidates (none, since A claimed the only one) and, + // with nothing left to do, is never called. + assert.Equal(t, 1, cA.matchCalls) + assert.Equal(t, 1, cA.prefetchCalls) + assert.Equal(t, 0, cB.matchCalls) + assert.Equal(t, 0, cB.prefetchCalls) }) t.Run("no match leaves empty map", func(t *testing.T) { @@ -114,15 +136,16 @@ func TestDispatchClassification(t *testing.T) { extractor := NewWasmSpecExtractorMock(t) extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() - matches, err := DispatchClassification( - ctx, nil, extractor, + plan, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{c}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, nil, + nil, nil, nil, nil, ) require.NoError(t, err) - assert.Empty(t, matches) - assert.Equal(t, 1, c.calls) + assert.Empty(t, plan.Matches) + assert.Equal(t, 1, c.matchCalls) + assert.Equal(t, 1, c.prefetchCalls) }) t.Run("spec extraction failure drops candidate", func(t *testing.T) { @@ -132,60 +155,60 @@ func TestDispatchClassification(t *testing.T) { extractor := NewWasmSpecExtractorMock(t) extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return(nil, errors.New("compile fail")).Once() - matches, err := DispatchClassification( - ctx, nil, extractor, + plan, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{c}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, nil, + nil, nil, nil, nil, ) require.NoError(t, err) - assert.Empty(t, matches) + assert.Empty(t, plan.Matches) // A spec-less candidate cannot match any signature-based validator, so it is // dropped from the candidate set and never reaches the validator. The caller - // persists the underlying wasm with protocol_id = NULL (absent from matches). - assert.Equal(t, 0, c.calls) + // persists the underlying wasm with protocol_id = NULL (absent from Matches). + assert.Equal(t, 0, c.matchCalls) }) - t.Run("validator error aborts dispatch", func(t *testing.T) { + t.Run("validator prefetch error aborts dispatch", func(t *testing.T) { hash := types.HashBytea("aabb") - cBoom := newRecordingValidator("A") - cBoom.err = errors.New("boom") - cOK := newRecordingValidator("B", hash) + cBoom := newRecordingValidator("A", hash) + cBoom.prefetchErr = errors.New("boom") + cOK := newRecordingValidator("B") extractor := NewWasmSpecExtractorMock(t) extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() - matches, err := DispatchClassification( - ctx, nil, extractor, + plan, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{cBoom, cOK}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, nil, + nil, nil, nil, nil, ) require.Error(t, err) - assert.Nil(t, matches) - assert.Equal(t, 0, cOK.calls, "later validators must not run after one aborts") + assert.Nil(t, plan) + assert.Equal(t, 0, cOK.matchCalls, "later validators must not run after one aborts") }) - t.Run("known protocol annotation reaches validator", func(t *testing.T) { + t.Run("known protocol annotation reaches validator and seeds Matches", func(t *testing.T) { wasmHash := types.HashBytea("ccdd") contractID := types.HashBytea("11") cA := newRecordingValidator("A") extractor := NewWasmSpecExtractorMock(t) - matches, err := DispatchClassification( - ctx, nil, extractor, + plan, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{cA}, nil, // no in-batch candidates []data.ProtocolContracts{{ContractID: contractID, WasmHash: wasmHash}}, - nil, nil, + nil, map[types.HashBytea]string{wasmHash: "A"}, nil, ) require.NoError(t, err) - assert.Empty(t, matches) - require.Equal(t, 1, cA.calls) - require.Len(t, cA.lastInput.Contracts, 1) - assert.Equal(t, "A", cA.lastInput.Contracts[0].KnownProtocolID) + assert.Equal(t, "A", plan.Matches[wasmHash], "known classification is preserved in the plan") + require.Equal(t, 1, cA.matchCalls) + require.Len(t, cA.lastContracts, 1) + assert.Equal(t, "A", cA.lastContracts[0].KnownProtocolID) }) } @@ -196,7 +219,7 @@ func newFailureCounterForTest() *prometheus.CounterVec { ) } -func TestDispatchClassification_FailureCounter(t *testing.T) { +func TestPrepareClassification_FailureCounter(t *testing.T) { ctx := context.Background() t.Run("spec extraction failure increments counter", func(t *testing.T) { @@ -208,11 +231,11 @@ func TestDispatchClassification_FailureCounter(t *testing.T) { counter := newFailureCounterForTest() - _, err := DispatchClassification( - ctx, nil, extractor, + _, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{c}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, + nil, nil, nil, counter, ) require.NoError(t, err) @@ -220,21 +243,21 @@ func TestDispatchClassification_FailureCounter(t *testing.T) { assert.Equal(t, 0.0, testutil.ToFloat64(counter.WithLabelValues("A", "validate_error"))) }) - t.Run("validator error increments counter", func(t *testing.T) { + t.Run("validator prefetch error increments counter", func(t *testing.T) { hash := types.HashBytea("aabb") - cBoom := newRecordingValidator("sep41") - cBoom.err = errors.New("boom") + cBoom := newRecordingValidator("sep41", hash) + cBoom.prefetchErr = errors.New("boom") extractor := NewWasmSpecExtractorMock(t) extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() counter := newFailureCounterForTest() - _, err := DispatchClassification( - ctx, nil, extractor, + _, err := PrepareClassification( + ctx, extractor, []ProtocolValidator{cBoom}, map[types.HashBytea][]byte{hash: {1, 2, 3}}, - nil, nil, nil, nil, + nil, nil, nil, counter, ) require.Error(t, err) @@ -242,3 +265,90 @@ func TestDispatchClassification_FailureCounter(t *testing.T) { assert.Equal(t, 0.0, testutil.ToFloat64(counter.WithLabelValues("unknown", "spec_extraction_error"))) }) } + +// wantApplyClassificationPlanSignature pins the exact signature +// ApplyClassificationPlan must have; assigning the real function to this +// named type below is what makes TestApplyClassificationPlan_NoRPCService a +// compile-level check rather than an inferred-type no-op. +type wantApplyClassificationPlanSignature func(ctx context.Context, dbTx pgx.Tx, models *data.Models, plan *ClassificationPlan, failureCounter *prometheus.CounterVec) error + +// TestApplyClassificationPlan_NoRPCService is the compile-level guarantee the +// RPC-out-of-transaction hardening rests on: ApplyClassificationPlan's +// signature has no RPCService (or anything RPC-capable) parameter, so it is +// impossible — not just unlikely — for a validator's Apply call reached from +// here to make a network call while dbTx is open. If this test compiles, the +// guarantee holds; nothing at runtime needs to prove it. +func TestApplyClassificationPlan_NoRPCService(t *testing.T) { + var _ wantApplyClassificationPlanSignature = ApplyClassificationPlan +} + +func TestApplyClassificationPlan(t *testing.T) { + ctx := context.Background() + + t.Run("nil plan is a no-op", func(t *testing.T) { + require.NoError(t, ApplyClassificationPlan(ctx, nil, nil, nil, nil)) + }) + + t.Run("applies every validator in the plan using only prefetched data", func(t *testing.T) { + hash := types.HashBytea("aabb") + cA := newRecordingValidator("A", hash) + extractor := NewWasmSpecExtractorMock(t) + extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() + + plan, err := PrepareClassification( + ctx, extractor, + []ProtocolValidator{cA}, + map[types.HashBytea][]byte{hash: {1, 2, 3}}, + nil, nil, nil, nil, + ) + require.NoError(t, err) + + require.NoError(t, ApplyClassificationPlan(ctx, nil, nil, plan, nil)) + assert.Equal(t, 1, cA.applyCalls) + }) + + t.Run("apply error is counted and propagated", func(t *testing.T) { + hash := types.HashBytea("aabb") + cBoom := newRecordingValidator("sep41", hash) + cBoom.applyErr = errors.New("boom") + extractor := NewWasmSpecExtractorMock(t) + extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() + + plan, err := PrepareClassification( + ctx, extractor, + []ProtocolValidator{cBoom}, + map[types.HashBytea][]byte{hash: {1, 2, 3}}, + nil, nil, nil, nil, + ) + require.NoError(t, err) + + counter := newFailureCounterForTest() + err = ApplyClassificationPlan(ctx, nil, nil, plan, counter) + require.Error(t, err) + assert.Equal(t, 1.0, testutil.ToFloat64(counter.WithLabelValues("sep41", "validate_error"))) + }) + + t.Run("retrying Apply with the same plan does not re-invoke Prefetch", func(t *testing.T) { + hash := types.HashBytea("aabb") + cA := newRecordingValidator("A", hash) + extractor := NewWasmSpecExtractorMock(t) + extractor.On("ExtractSpec", mock.Anything, mock.Anything).Return([]xdr.ScSpecEntry{{}}, nil).Once() + + plan, err := PrepareClassification( + ctx, extractor, + []ProtocolValidator{cA}, + map[types.HashBytea][]byte{hash: {1, 2, 3}}, + nil, nil, nil, nil, + ) + require.NoError(t, err) + require.Equal(t, 1, cA.prefetchCalls) + + // Simulate ingestProcessedDataWithRetry's retry loop: the same plan is + // applied across multiple attempts. + for i := 0; i < 3; i++ { + require.NoError(t, ApplyClassificationPlan(ctx, nil, nil, plan, nil)) + } + assert.Equal(t, 3, cA.applyCalls) + assert.Equal(t, 1, cA.prefetchCalls, "prefetch must not be re-issued on retry") + }) +} diff --git a/internal/services/protocol_validator.go b/internal/services/protocol_validator.go index 73a6a1e11..d234a3f7d 100644 --- a/internal/services/protocol_validator.go +++ b/internal/services/protocol_validator.go @@ -16,41 +16,52 @@ import ( ) // ProtocolValidator is the per-protocol seam the data-migration framework -// calls during classification. The framework hands the validator a batch of -// candidate WASMs (with parsed spec entries) plus the contracts referencing -// those WASMs, and the validator returns the wasm hashes it claims. A -// validator is free to do anything else it needs inside the supplied dbTx — -// fetch on-chain metadata, write protocol-specific side tables, anything — -// and the framework treats those side effects as invisible. Only the returned -// matches drive generic persistence into protocol_wasms.protocol_id. +// calls during classification. Classification is split into three phases so +// the framework can prefetch RPC-sourced enrichment before opening any +// database transaction, then apply pure DB writes inside it: +// +// 1. Match: pure signature check, no RPC, no DB. +// 2. Prefetch: resolves on-chain enrichment (e.g. SEP-41 token +// metadata) via RPC. Called before any transaction opens. +// 3. Apply: persists the protocol's side effects (e.g. +// contract_tokens rows) inside the caller's dbTx, using only what Prefetch +// already resolved. No RPC handle reaches this phase. // // First-match-wins ordering across protocols is enforced by the dispatcher: -// by the time Validate is called for a given protocol, candidates already +// by the time Match is called for a given protocol, candidates already // claimed by a higher-priority protocol have been filtered out. type ProtocolValidator interface { ProtocolID() string - Validate(ctx context.Context, dbTx pgx.Tx, input ValidationInput) (ValidationResult, error) -} -// ValidationInput is the batch handed to a protocol's Validate call. -type ValidationInput struct { - // Candidates are WASMs awaiting classification in this batch. Each entry - // carries the raw bytecode and pre-extracted ScSpecEntry items. - Candidates []WasmCandidate - // Contracts references both in-batch candidates and previously-classified - // WASMs. KnownProtocolID is set for hashes already classified by an - // earlier ledger or protocol-setup run; the empty string means the wasm - // has not been classified yet (or was classified to no match). - Contracts []ContractCandidate - // RPC is available for validators that need live network reads (e.g. - // fetching token metadata via simulation). May be nil; validators must - // degrade gracefully. - RPC RPCService - // Models provides the full data layer. Validators should write only to - // protocol-specific tables here; the framework persists the generic - // protocol_wasms / protocol_contracts mapping based on the returned - // matches. - Models *data.Models + // Match runs the protocol's signature check against each candidate's + // pre-extracted spec entries and returns the hashes it claims. Pure — no + // RPC, no DB access — safe to call before any transaction or network + // round-trip. + Match(candidates []WasmCandidate) map[types.HashBytea]struct{} + + // Prefetch resolves RPC-sourced enrichment for contracts whose wasm is + // newly matched (per matched) or already classified by an earlier run + // (per each ContractCandidate.KnownProtocolID). candidates is the same + // filtered slice handed to Match, in case a validator needs to + // re-derive per-candidate detail Match's uniform return type can't + // carry (e.g. a protocol-specific contract role). Called before any + // database transaction opens; rpc may be nil (e.g. offline + // protocol-migrate paths without RPC configured), in which case + // implementations must degrade to an empty/zero-value plan rather than + // erroring. Per-contract fetch failures are absorbed here (logged, + // omitted from the returned plan) — mirroring today's best-effort + // semantics, Prefetch should not fail the batch. + // + // The returned plan is opaque to the framework; it is passed back + // verbatim to Apply. + Prefetch(ctx context.Context, rpc RPCService, candidates []WasmCandidate, matched map[types.HashBytea]struct{}, contracts []ContractCandidate) (plan any, err error) + + // Apply persists this batch's classification side effects (e.g. + // contract_tokens rows) inside dbTx, using only the plan + // Prefetch already resolved. No RPC handle is available here — a + // validator whose Prefetch could not reach RPC already reflects that in + // its plan (e.g. an empty enrichment map), not as an Apply-time error. + Apply(ctx context.Context, dbTx pgx.Tx, matched map[types.HashBytea]struct{}, contracts []ContractCandidate, plan any, models *data.Models) error } // WasmCandidate represents one WASM awaiting classification. Bytecode and @@ -71,16 +82,6 @@ type ContractCandidate struct { KnownProtocolID string } -// ValidationResult is what a protocol returns from Validate. MatchedWasms -// drives the framework's stamp of protocol_wasms.protocol_id. Contract claims -// are an internal detail of the validator's own enrichment path — the -// framework persists protocol_contracts unconditionally as a contract→wasm -// map and the wasm's protocol_id (set via this match) is what downstream -// JOINs use to resolve the protocol. -type ValidationResult struct { - MatchedWasms []types.HashBytea -} - const ( contractSpecV0SectionName = "contractspecv0" diff --git a/internal/services/sep41/validator.go b/internal/services/sep41/validator.go index f567b6ae6..0d5a84322 100644 --- a/internal/services/sep41/validator.go +++ b/internal/services/sep41/validator.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "fmt" - "sort" "github.com/alitto/pond/v2" set "github.com/deckarep/golang-set/v2" @@ -32,8 +31,8 @@ const contractTokenType = "sep41" // satisfies the framework's per-protocol seam: signature-check candidate // WASMs against the SEP-41 token interface and, for any contract whose wasm // is now (or was already) classified as SEP-41, fetch on-chain -// name/symbol/decimals via RPC and write them to contract_tokens inside the -// supplied dbTx. +// name/symbol/decimals via RPC (Prefetch) and write them to contract_tokens +// (Apply). // // Contract write strategy: contract_tokens rows are inserted with // deterministic IDs and ON CONFLICT DO NOTHING; metadata is then upserted via @@ -71,35 +70,61 @@ func newValidator(deps services.ProtocolDeps) *Validator { func (v *Validator) ProtocolID() string { return ProtocolID } -// Validate runs the SEP-41 signature check over each candidate WASM and, for -// every contract whose wasm is now (or was already) classified as SEP-41, -// enriches contract_tokens with metadata fetched via RPC. See the +// Match runs the SEP-41 signature check over each candidate WASM's +// pre-extracted spec entries. Pure — no RPC, no DB access. +func (v *Validator) Match(candidates []services.WasmCandidate) map[types.HashBytea]struct{} { + return v.matchCandidates(candidates) +} + +// sep41Prefetch is the RPC-sourced enrichment Prefetch resolves: token +// metadata keyed by C-address, ready for Apply to write without any further +// network access. +type sep41Prefetch struct { + metaByAddr map[string]*data.Contract +} + +// Prefetch resolves on-chain name/symbol/decimals for every contract whose +// wasm is now (or was already) classified as SEP-41. See the // services.ProtocolValidator godoc for the framework-level contract. -func (v *Validator) Validate(ctx context.Context, dbTx pgx.Tx, input services.ValidationInput) (services.ValidationResult, error) { - matched := v.matchCandidates(input.Candidates) - if len(matched) == 0 && !v.hasKnownClaims(input.Contracts) { - return services.ValidationResult{}, nil +func (v *Validator) Prefetch(ctx context.Context, _ services.RPCService, _ []services.WasmCandidate, matched map[types.HashBytea]struct{}, contracts []services.ContractCandidate) (any, error) { + contractsForUs := v.collectClaimedContracts(contracts, matched) + if len(contractsForUs) == 0 || v.fetcher == nil { + return sep41Prefetch{}, nil } - contractsForUs := v.collectClaimedContracts(input.Contracts, matched) - if len(contractsForUs) > 0 { - if err := v.enrichContractTokens(ctx, dbTx, input.Models, contractsForUs); err != nil { - // Metadata enrichment is best-effort. A DB-level error inside the - // caller's tx still propagates, but we wrap it so callers can - // distinguish enrichment failures from validation failures when - // surfacing logs. - return services.ValidationResult{}, fmt.Errorf("sep41 enrichment: %w", err) - } + addrs := v.decodeClaimedAddrs(ctx, contractsForUs) + if len(addrs) == 0 { + return sep41Prefetch{}, nil + } + + metaByAddr, err := v.fetcher.FetchMetadata(ctx, addrs) + if err != nil { + // Fetch-level errors are best-effort — Apply still inserts default + // rows for contractsForUs; metadata is filled on a later + // classification pass once RPC is reachable. + log.Ctx(ctx).Warnf("sep41 prefetch: metadata fetch returned error, continuing with defaults: %v", err) + return sep41Prefetch{}, nil } + return sep41Prefetch{metaByAddr: metaByAddr}, nil +} - out := services.ValidationResult{MatchedWasms: make([]types.HashBytea, 0, len(matched))} - for h := range matched { - out.MatchedWasms = append(out.MatchedWasms, h) +// Apply persists this batch's contract_tokens rows inside dbTx: default rows +// for every claimed contract (idempotent, ON CONFLICT DO NOTHING) followed by +// a metadata upsert for whatever Prefetch resolved via RPC. No RPC handle is +// available here. +func (v *Validator) Apply(ctx context.Context, dbTx pgx.Tx, matched map[types.HashBytea]struct{}, contracts []services.ContractCandidate, plan any, models *data.Models) error { + contractsForUs := v.collectClaimedContracts(contracts, matched) + if len(contractsForUs) == 0 { + return nil } - sort.Slice(out.MatchedWasms, func(i, j int) bool { - return out.MatchedWasms[i] < out.MatchedWasms[j] - }) - return out, nil + prefetch, _ := plan.(sep41Prefetch) + if err := v.applyContractTokens(ctx, dbTx, models, contractsForUs, prefetch); err != nil { + // A DB-level error inside the caller's tx still propagates, but we + // wrap it so callers can distinguish enrichment failures from + // matching failures when surfacing logs. + return fmt.Errorf("sep41 enrichment: %w", err) + } + return nil } // Close releases any resources owned by this validator (worker pool). @@ -127,17 +152,6 @@ func (v *Validator) matchCandidates(candidates []services.WasmCandidate) map[typ return matched } -// hasKnownClaims reports whether any contract in the batch references a wasm -// already classified as SEP-41 by an earlier ledger or protocol-setup run. -func (v *Validator) hasKnownClaims(contracts []services.ContractCandidate) bool { - for _, ct := range contracts { - if ct.KnownProtocolID == ProtocolID { - return true - } - } - return false -} - // collectClaimedContracts returns the contracts whose wasm hash is matched in // this batch or already classified as SEP-41 from a prior run. func (v *Validator) collectClaimedContracts(contracts []services.ContractCandidate, matched map[types.HashBytea]struct{}) []services.ContractCandidate { @@ -154,26 +168,38 @@ func (v *Validator) collectClaimedContracts(contracts []services.ContractCandida return out } -// enrichContractTokens decodes contract IDs to C-addresses, fetches metadata -// via RPC, and writes contract_tokens rows inside dbTx. -func (v *Validator) enrichContractTokens(ctx context.Context, dbTx pgx.Tx, models *data.Models, contracts []services.ContractCandidate) error { - if models == nil || models.Contract == nil { - return nil - } +// decodeClaimedAddrs decodes each claimed contract's hex ID into its +// C-address form, deduplicating and skipping any that fail to decode. Shared +// by Prefetch (which needs the address list to fetch metadata) and Apply +// (which needs it again to insert default rows), so no per-contract decode +// state has to be smuggled across the RPC boundary. +func (v *Validator) decodeClaimedAddrs(ctx context.Context, contracts []services.ContractCandidate) []string { + seen := make(map[string]struct{}, len(contracts)) addrs := make([]string, 0, len(contracts)) - addrToHash := make(map[string]types.HashBytea, len(contracts)) for _, ct := range contracts { addr, ok := decodeContractAddr(ct.ContractID) if !ok { log.Ctx(ctx).Debugf("sep41: skipping contract with undecodable ID %q", ct.ContractID) continue } - if _, dup := addrToHash[addr]; dup { + if _, dup := seen[addr]; dup { continue } + seen[addr] = struct{}{} addrs = append(addrs, addr) - addrToHash[addr] = ct.WasmHash } + return addrs +} + +// applyContractTokens writes contract_tokens rows inside dbTx: default rows +// for every claimed contract, then a metadata upsert for whatever Prefetch +// resolved via RPC (prefetch.metaByAddr may be empty when RPC was +// unavailable or every fetch failed, in which case the defaults stand). +func (v *Validator) applyContractTokens(ctx context.Context, dbTx pgx.Tx, models *data.Models, contracts []services.ContractCandidate, prefetch sep41Prefetch) error { + if models == nil || models.Contract == nil { + return nil + } + addrs := v.decodeClaimedAddrs(ctx, contracts) if len(addrs) == 0 { return nil } @@ -193,26 +219,12 @@ func (v *Validator) enrichContractTokens(ctx context.Context, dbTx pgx.Tx, model return fmt.Errorf("inserting default contract_tokens: %w", err) } - if v.fetcher == nil { - // No RPC available — leave defaults in place. Metadata is filled on a - // later classification pass once RPC is reachable; there is no - // processor-side fallback. - return nil - } - - metaByAddr, err := v.fetcher.FetchMetadata(ctx, addrs) - if err != nil { - // Treat fetch-level errors as best-effort; the rows already exist - // with defaults. - log.Ctx(ctx).Warnf("sep41 enrich: metadata fetch returned error, continuing with defaults: %v", err) - return nil - } - if len(metaByAddr) == 0 { + if len(prefetch.metaByAddr) == 0 { return nil } - updates := make([]*data.Contract, 0, len(metaByAddr)) - for _, contract := range metaByAddr { + updates := make([]*data.Contract, 0, len(prefetch.metaByAddr)) + for _, contract := range prefetch.metaByAddr { // Force the row's type to the lower-case SEP-41 discriminator so the // source of truth stays in protocols/protocol_wasms — defensive // against future fetcher refactors that might omit the type field. diff --git a/internal/services/sep41/validator_test.go b/internal/services/sep41/validator_test.go index 61385f38c..a5d9dcb56 100644 --- a/internal/services/sep41/validator_test.go +++ b/internal/services/sep41/validator_test.go @@ -432,26 +432,23 @@ func TestValidator_RealWasm(t *testing.T) { }) } -// TestValidator_NoMatch covers the case where the signature check rejects the -// candidate spec. No contract_tokens writes happen and no matches are -// returned. -func TestValidator_NoMatch(t *testing.T) { +// TestValidator_Match_NoMatch covers the case where the signature check +// rejects the candidate spec: no hashes are returned. +func TestValidator_Match_NoMatch(t *testing.T) { v := NewValidator() hash := indexerTypes.HashBytea("aabb") - out, err := v.Validate(context.Background(), nil, services.ValidationInput{ - Candidates: []services.WasmCandidate{ - {Hash: hash, SpecEntries: []xdr.ScSpecEntry{{Kind: xdr.ScSpecEntryKindScSpecEntryFunctionV0}}}, - }, + matched := v.Match([]services.WasmCandidate{ + {Hash: hash, SpecEntries: []xdr.ScSpecEntry{{Kind: xdr.ScSpecEntryKindScSpecEntryFunctionV0}}}, }) - require.NoError(t, err) - assert.Empty(t, out.MatchedWasms) + assert.Empty(t, matched) } -// TestValidator_KnownContractEnrichmentRunsWithoutCandidate verifies that the -// validator enriches contracts whose wasm hash was classified as SEP-41 in a -// prior batch (KnownProtocolID = "SEP41") even when the wasm itself is not -// in this batch's Candidates. -func TestValidator_KnownContractEnrichmentRunsWithoutCandidate(t *testing.T) { +// TestValidator_Apply_KnownContractEnrichmentRunsWithoutCandidate verifies +// that Apply enriches contracts whose wasm hash was classified as SEP-41 in a +// prior batch (KnownProtocolID = "SEP41") even when the wasm itself is not in +// this batch's candidates, and that Prefetch (no fetcher configured) makes no +// RPC call before Apply writes the default row. +func TestValidator_Apply_KnownContractEnrichmentRunsWithoutCandidate(t *testing.T) { contractsMock := data.NewContractModelMock(t) models := &data.Models{Contract: contractsMock} @@ -466,14 +463,16 @@ func TestValidator_KnownContractEnrichmentRunsWithoutCandidate(t *testing.T) { require.NoError(t, err) contractID := indexerTypes.HashBytea(hex.EncodeToString(rawAddr)) - out, err := v.Validate(context.Background(), nil, services.ValidationInput{ - Contracts: []services.ContractCandidate{ - {ContractID: contractID, WasmHash: indexerTypes.HashBytea("aabb"), KnownProtocolID: ProtocolID}, - }, - Models: models, - }) + matched := map[indexerTypes.HashBytea]struct{}{} + contracts := []services.ContractCandidate{ + {ContractID: contractID, WasmHash: indexerTypes.HashBytea("aabb"), KnownProtocolID: ProtocolID}, + } + + ctx := context.Background() + plan, err := v.Prefetch(ctx, nil, nil, matched, contracts) require.NoError(t, err) - assert.Empty(t, out.MatchedWasms, "no in-batch candidates → no match returned") + + require.NoError(t, v.Apply(ctx, nil, matched, contracts, plan, models)) } func createScSpecFunctionEntry(name string, inputs []xdr.ScSpecFunctionInputV0, outputs []xdr.ScSpecTypeDef) xdr.ScSpecEntry { diff --git a/internal/services/validator_registry_test.go b/internal/services/validator_registry_test.go index ad23af46f..de64e77ef 100644 --- a/internal/services/validator_registry_test.go +++ b/internal/services/validator_registry_test.go @@ -7,6 +7,9 @@ import ( "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/wallet-backend/internal/data" + "github.com/stellar/wallet-backend/internal/indexer/types" ) func withCleanValidatorRegistry(t *testing.T) { @@ -21,9 +24,14 @@ func withCleanValidatorRegistry(t *testing.T) { // ProtocolValidatorMock. type idValidator struct{ id string } -func (v idValidator) ProtocolID() string { return v.id } -func (v idValidator) Validate(context.Context, pgx.Tx, ValidationInput) (ValidationResult, error) { - return ValidationResult{}, nil +func (v idValidator) ProtocolID() string { return v.id } +func (v idValidator) Match([]WasmCandidate) map[types.HashBytea]struct{} { return nil } +func (v idValidator) Prefetch(context.Context, RPCService, []WasmCandidate, map[types.HashBytea]struct{}, []ContractCandidate) (any, error) { + return nil, nil +} + +func (v idValidator) Apply(context.Context, pgx.Tx, map[types.HashBytea]struct{}, []ContractCandidate, any, *data.Models) error { + return nil } func TestRegisterValidator(t *testing.T) { From 377f66bbc5e928040f96b6e7849589e71f14e8f1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 01:56:59 -0400 Subject: [PATCH 3/4] fix(ingest): split-brain defenses, permanent-error fast fail, bounded pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two defenses against concurrent ingesters after silent advisory-lock loss (a CNPG failover kills the lock session while the loop keeps writing through other pool connections): the loop now probes the lock-holding connection every ledger and exits fatally when the session is dead, and the latest-ledger cursor advances through a guarded update that refuses any value it doesn't own (ErrCursorGuardFailed) instead of blindly upserting — cursor regression is now impossible. Persist retries classify errors: SQLSTATE classes 22/23/42 and cursor guard failures fail on the first attempt instead of burning five retries before a crash loop; a self-cancelled datastore buffer (ErrBufferDead) exits immediately instead of a 10-attempt backoff ladder against a dead buffer. Protocol cursor CAS is gated on a startup snapshot (re-probed on the oldest-ledger sync cadence): never-initialized cursors skip their CAS entirely, so cursor_missing now fires only on the genuine existed-then- vanished incident and becomes alertable. Also: ledger-fetch duration/retry metrics now observed (help texts corrected); pond pools bounded (indexer 2xNumCPU, RPC-facing pools match the simulate batch size — the sep41 comment previously claimed a cap that did not exist); per-ledger insert/upsert detail logs demoted to debug; IndexerBuffer getter aliasing contracts documented. --- internal/data/ingest_store.go | 52 ++++- internal/data/ingest_store_test.go | 93 +++++++++ internal/indexer/indexer_buffer.go | 33 +-- internal/ingest/datastore_backend.go | 22 +- internal/ingest/datastore_backend_test.go | 82 ++++++++ internal/ingest/ingest.go | 22 +- internal/metrics/ingestion.go | 14 +- internal/services/contract_metadata.go | 13 +- internal/services/ingest.go | 27 ++- internal/services/ingest_live.go | 233 +++++++++++++++++++--- internal/services/ingest_live_test.go | 51 +++++ internal/services/ingest_test.go | 137 ++++++++++++- internal/services/sep41/validator.go | 8 +- internal/services/token_ingestion.go | 10 +- internal/utils/retry.go | 15 +- internal/utils/retry_test.go | 57 ++++++ 16 files changed, 789 insertions(+), 80 deletions(-) diff --git a/internal/data/ingest_store.go b/internal/data/ingest_store.go index 642b924c4..0c62ad13f 100644 --- a/internal/data/ingest_store.go +++ b/internal/data/ingest_store.go @@ -24,10 +24,12 @@ const ( // not exist in ingest_store. The model layer keeps this distinct from the // ordinary value-mismatch race so callers can decide what to do — a missing // row may be operationally normal (cursor not yet initialized by protocol-setup -// / protocol-migrate) or a real incident (dropped row, bad restore). The -// service layer treats this as a soft skip: the `cursor_missing` query-error -// metric and a per-ledger warn provide the observability signal without -// killing live ingest. See ingest_live.go PersistLedgerData for the handling. +// / protocol-migrate) or a real incident (dropped row, bad restore). Live +// ingestion (see ingest_live.go's casProtocolCursor) only ever calls +// CompareAndSwap for a cursor its protocolCursorSnapshot believes exists, so +// from that caller this error always means the genuine incident case: the +// `cursor_missing` query-error metric recorded below fires accordingly, +// rather than on every not-yet-initialized ledger. var ErrCASCursorMissing = errors.New("ingest_store cursor row missing") type LedgerRange struct { @@ -118,6 +120,48 @@ func (m *IngestStoreModel) Update(ctx context.Context, dbTx pgx.Tx, cursorName s return nil } +// ErrCursorGuardFailed is returned by UpdateGuarded when the cursor's current +// value is neither ledger-1 nor ledger (see UpdateGuarded), or the row does +// not exist. Either way, a writer other than the one calling UpdateGuarded +// has moved the cursor past what it expected — most commonly a second live +// ingestion instance that acquired the advisory lock after this session's +// Postgres session died in a failover (see startLiveIngestion's +// checkLockSession, which is the primary defense; this guard is the backstop +// for the race window before that probe observes the dead session). +var ErrCursorGuardFailed = errors.New("ingest_store guarded cursor update refused: cursor value not owned by this writer") + +// UpdateGuarded advances cursorName to ledger only if its current value is ledger-1 (the normal +// sequential case: this writer is the sole owner and is advancing by exactly one ledger) or +// ledger itself (the self-value case: the first ledger processed immediately after +// startLiveIngestion's initializeCursors already set the cursor to this same starting ledger). +// Any other current value — including a missing row — means a writer other than the caller has +// moved the cursor, so the swap is refused with ErrCursorGuardFailed instead of silently +// overwriting a value another instance already advanced (which a blind Update would do). +func (m *IngestStoreModel) UpdateGuarded(ctx context.Context, dbTx pgx.Tx, cursorName string, ledger uint32) error { + const query = ` + UPDATE ingest_store + SET value = $1 + WHERE key = $2 AND value IN ($3, $4) + ` + newValue := strconv.FormatUint(uint64(ledger), 10) + prevValue := strconv.FormatUint(uint64(ledger-1), 10) + + start := time.Now() + tag, err := dbTx.Exec(ctx, query, newValue, cursorName, prevValue, newValue) + duration := time.Since(start).Seconds() + m.Metrics.QueryDuration.WithLabelValues("UpdateGuarded", "ingest_store").Observe(duration) + m.Metrics.QueriesTotal.WithLabelValues("UpdateGuarded", "ingest_store").Inc() + if err != nil { + m.Metrics.QueryErrors.WithLabelValues("UpdateGuarded", "ingest_store", utils.GetDBErrorType(err)).Inc() + return fmt.Errorf("guarded update for cursor %s to %d: %w", cursorName, ledger, err) + } + if tag.RowsAffected() == 0 { + m.Metrics.QueryErrors.WithLabelValues("UpdateGuarded", "ingest_store", "cursor_guard_failed").Inc() + return fmt.Errorf("guarded update for cursor %s to %d: %w", cursorName, ledger, ErrCursorGuardFailed) + } + return nil +} + func (m *IngestStoreModel) CompareAndSwap(ctx context.Context, dbTx pgx.Tx, cursorName string, expectedValue string, newValue string) (bool, error) { // A plain UPDATE returns RowsAffected=0 for both "value mismatch" and // "row missing" — callers treat false as a race loss and mark the diff --git a/internal/data/ingest_store_test.go b/internal/data/ingest_store_test.go index 5fb8ed802..972265e55 100644 --- a/internal/data/ingest_store_test.go +++ b/internal/data/ingest_store_test.go @@ -227,6 +227,99 @@ func Test_IngestStoreModel_UpdateLatestLedgerSynced(t *testing.T) { } } +func Test_IngestStoreModel_UpdateGuarded(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + dbConnectionPool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer dbConnectionPool.Close() + + const key = "guarded_cursor" + + testCases := []struct { + name string + setupDB func(t *testing.T) // nil means no row inserted + ledger uint32 + expectErr bool + expectedValue string // DB value after the call; only checked when !expectErr + }{ + { + // Normal sequential case: current value is ledger-1. + name: "advances_from_ledger_minus_one", + setupDB: func(t *testing.T) { + _, err := dbConnectionPool.Exec(ctx, `INSERT INTO ingest_store (key, value) VALUES ($1, '99')`, key) + require.NoError(t, err) + }, + ledger: 100, + expectedValue: "100", + }, + { + // Self-value case: the first ledger processed right after + // startLiveIngestion's initializeCursors already set the cursor to + // this same starting ledger. + name: "self_value_is_a_noop_success", + setupDB: func(t *testing.T) { + _, err := dbConnectionPool.Exec(ctx, `INSERT INTO ingest_store (key, value) VALUES ($1, '100')`, key) + require.NoError(t, err) + }, + ledger: 100, + expectedValue: "100", + }, + { + // Regression guard: a second writer already advanced the cursor past + // what this caller expected (e.g. a second live-ingestion instance + // that acquired the lock after this session's Postgres session died + // in a failover). Must refuse rather than overwrite. + name: "refuses_when_cursor_already_advanced_past_expected", + setupDB: func(t *testing.T) { + _, err := dbConnectionPool.Exec(ctx, `INSERT INTO ingest_store (key, value) VALUES ($1, '105')`, key) + require.NoError(t, err) + }, + ledger: 100, + expectErr: true, + }, + { + name: "errors_when_row_missing", + ledger: 100, + expectErr: true, + }, + } + + dbMetrics := metrics.NewMetrics(prometheus.NewRegistry()).DB + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := dbConnectionPool.Exec(ctx, "DELETE FROM ingest_store") + require.NoError(t, err) + + m := &IngestStoreModel{ + DB: dbConnectionPool, + Metrics: dbMetrics, + } + if tc.setupDB != nil { + tc.setupDB(t) + } + + err = db.RunInTransaction(ctx, m.DB, func(dbTx pgx.Tx) error { + return m.UpdateGuarded(ctx, dbTx, key, tc.ledger) + }) + + if tc.expectErr { + require.Error(t, err) + assert.True(t, errors.Is(err, ErrCursorGuardFailed), "expected ErrCursorGuardFailed, got %v", err) + return + } + require.NoError(t, err) + + var dbValue string + scanErr := m.DB.QueryRow(ctx, `SELECT value FROM ingest_store WHERE key = $1`, key).Scan(&dbValue) + require.NoError(t, scanErr) + assert.Equal(t, tc.expectedValue, dbValue) + }) + } +} + func Test_IngestStoreModel_CompareAndSwap(t *testing.T) { dbt := dbtest.Open(t) defer dbt.Close() diff --git a/internal/indexer/indexer_buffer.go b/internal/indexer/indexer_buffer.go index aa24dcb3d..70aab97de 100644 --- a/internal/indexer/indexer_buffer.go +++ b/internal/indexer/indexer_buffer.go @@ -197,7 +197,9 @@ func (b *IndexerBuffer) GetTransactions() []*types.Transaction { return txs } -// GetTransactionsParticipants returns a map of transaction ToIDs to its participants. +// GetTransactionsParticipants returns a map of transaction ToIDs to its +// participants. The map itself is a shallow clone, but each set.Set[string] +// value is the buffer's live object; callers must not modify the sets. func (b *IndexerBuffer) GetTransactionsParticipants() map[int64]set.Set[string] { b.mu.RLock() defer b.mu.RUnlock() @@ -326,8 +328,8 @@ func (b *IndexerBuffer) PushTrustlineChange(trustlineChange types.TrustlineChang pushWithTombstone(b.trustlineChangesByTrustlineKey, b.trustlineTombstones, changeKey, trustlineChange, trustlineOrder, trustlineIsNoopRemove) } -// GetTrustlineChanges returns all trustline changes stored in the buffer. -// Thread-safe: uses read lock. +// GetTrustlineChanges returns the buffer's internal map of trustline changes; +// callers must not modify it. Thread-safe: uses read lock. func (b *IndexerBuffer) GetTrustlineChanges() map[TrustlineChangeKey]types.TrustlineChange { b.mu.RLock() defer b.mu.RUnlock() @@ -346,8 +348,8 @@ func (b *IndexerBuffer) PushAccountChange(accountChange types.AccountChange) { pushWithTombstone(b.accountChangesByAccountID, b.accountTombstones, accountChange.AccountID, accountChange, accountOrder, accountIsNoopRemove) } -// GetAccountChanges returns all account changes stored in the buffer. -// Thread-safe: uses read lock. +// GetAccountChanges returns the buffer's internal map of account changes; +// callers must not modify it. Thread-safe: uses read lock. func (b *IndexerBuffer) GetAccountChanges() map[string]types.AccountChange { b.mu.RLock() defer b.mu.RUnlock() @@ -370,8 +372,8 @@ func (b *IndexerBuffer) PushSACBalanceChange(sacBalanceChange types.SACBalanceCh pushWithTombstone(b.sacBalanceChangesByKey, b.sacTombstones, key, sacBalanceChange, sacBalanceOrder, sacBalanceIsNoopRemove) } -// GetSACBalanceChanges returns all SAC balance changes stored in the buffer. -// Thread-safe: uses read lock. +// GetSACBalanceChanges returns the buffer's internal map of SAC balance +// changes; callers must not modify it. Thread-safe: uses read lock. func (b *IndexerBuffer) GetSACBalanceChanges() map[SACBalanceChangeKey]types.SACBalanceChange { b.mu.RLock() defer b.mu.RUnlock() @@ -394,8 +396,9 @@ func (b *IndexerBuffer) PushLiquidityPoolShareChange(change types.LiquidityPoolS pushWithTombstone(b.lpShareChangesByKey, b.lpShareTombstones, key, change, lpShareOrder, lpShareIsNoopRemove) } -// GetLiquidityPoolShareChanges returns all pool-share balance changes stored in the buffer. -// Thread-safe: uses read lock. +// GetLiquidityPoolShareChanges returns the buffer's internal map of +// pool-share balance changes; callers must not modify it. Thread-safe: uses +// read lock. func (b *IndexerBuffer) GetLiquidityPoolShareChanges() map[LiquidityPoolShareChangeKey]types.LiquidityPoolShareChange { b.mu.RLock() defer b.mu.RUnlock() @@ -414,8 +417,8 @@ func (b *IndexerBuffer) PushLiquidityPoolChange(change types.LiquidityPoolChange pushWithTombstone(b.lpChangesByPoolID, b.lpTombstones, change.PoolID, change, lpOrder, lpIsNoopRemove) } -// GetLiquidityPoolChanges returns all pool reserve changes stored in the buffer. -// Thread-safe: uses read lock. +// GetLiquidityPoolChanges returns the buffer's internal map of pool reserve +// changes; callers must not modify it. Thread-safe: uses read lock. func (b *IndexerBuffer) GetLiquidityPoolChanges() map[string]types.LiquidityPoolChange { b.mu.RLock() defer b.mu.RUnlock() @@ -447,7 +450,9 @@ func (b *IndexerBuffer) GetOperations() []*types.Operation { return ops } -// GetOperationsParticipants returns a map of operation IDs to its participants. +// GetOperationsParticipants returns a map of operation IDs to its +// participants. The map itself is a shallow clone, but each set.Set[string] +// value is the buffer's live object; callers must not modify the sets. func (b *IndexerBuffer) GetOperationsParticipants() map[int64]set.Set[string] { b.mu.RLock() defer b.mu.RUnlock() @@ -485,8 +490,8 @@ func (b *IndexerBuffer) PushStateChange(transaction types.Transaction, operation } } -// GetStateChanges returns a copy of all state changes stored in the buffer. -// Thread-safe: uses read lock. +// GetStateChanges returns the buffer's internal slice of state changes; +// callers must not modify it. Thread-safe: uses read lock. func (b *IndexerBuffer) GetStateChanges() []types.StateChange { b.mu.RLock() defer b.mu.RUnlock() diff --git a/internal/ingest/datastore_backend.go b/internal/ingest/datastore_backend.go index c0bde5338..6340f5ee1 100644 --- a/internal/ingest/datastore_backend.go +++ b/internal/ingest/datastore_backend.go @@ -43,6 +43,15 @@ const ( defaultRetryWait = 5 * time.Second ) +// ErrBufferDead marks a GetLedger failure as terminal: the ledgerBuffer's +// internal context was cancelled by the buffer itself (a worker exhausted +// its download retry budget, or hit a hard NotExist on a bounded range), +// independent of the caller's context lifecycle. The buffer never recovers +// from this — every subsequent GetLedger call on this backend instance +// returns the same error — so callers should treat it as permanent rather +// than retrying against a dead buffer. +var ErrBufferDead = errors.New("ledger buffer permanently cancelled") + var _ ledgerbackend.LedgerBackend = (*datastoreBackend)(nil) // datastoreBackend implements ledgerbackend.LedgerBackend with optimizations @@ -417,10 +426,21 @@ func (buf *ledgerBuffer) downloadAndDecode(sequence uint32) (xdr.LedgerCloseMeta // getNextBatch receives the next decoded batch from the buffer in sequence order. // After receiving, it enqueues a new download task to maintain the buffer invariant. +// +// buf.ctx is derived from the caller-supplied ctx via context.WithCancelCause, so +// buf.ctx.Done() fires both when that parent ctx is cancelled (shutdown — its cause is +// context.Canceled, matching close()'s own buf.cancel(context.Canceled)) and when +// downloadAndStore gives up on a sequence and calls buf.cancel with a real error. Only the +// latter means the buffer itself has permanently died; distinguishing on the cause lets +// callers (the ledger-fetch retry ladder) tell "will never recover" apart from "shutting down". func (buf *ledgerBuffer) getNextBatch(ctx context.Context) (xdr.LedgerCloseMetaBatch, error) { select { case <-buf.ctx.Done(): - return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("buffer context cancelled: %w", context.Cause(buf.ctx)) + cause := context.Cause(buf.ctx) + if errors.Is(cause, context.Canceled) { + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("buffer context cancelled: %w", cause) + } + return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("%w: %w", ErrBufferDead, cause) case <-ctx.Done(): return xdr.LedgerCloseMetaBatch{}, fmt.Errorf("caller context cancelled: %w", ctx.Err()) case batch := <-buf.batchQueue: diff --git a/internal/ingest/datastore_backend_test.go b/internal/ingest/datastore_backend_test.go index 18e809df6..d87f334b8 100644 --- a/internal/ingest/datastore_backend_test.go +++ b/internal/ingest/datastore_backend_test.go @@ -451,10 +451,92 @@ func TestDatastoreBackend_WorkerRetryNotExist(t *testing.T) { _, err = backend.GetLedger(ctx, 0) require.Error(t, err) assert.Contains(t, err.Error(), "not found") + assert.ErrorIs(t, err, ErrBufferDead, "a hard NotExist on a bounded range permanently kills the buffer") require.NoError(t, backend.Close()) } +// TestDatastoreBackend_WorkerExhaustsRetries_BufferDies covers the other path +// that self-cancels the buffer: a transient error that never clears within +// RetryLimit attempts. GetLedger must surface ErrBufferDead so callers (the +// live-ingestion ledger-fetch retry ladder) know retrying is pointless — this +// buffer is dead and will return the same error forever. +func TestDatastoreBackend_WorkerExhaustsRetries_BufferDies(t *testing.T) { + ds := &mockDataStore{} + schema := testSchema(10) + + backend, err := newDatastoreBackend(ledgerbackend.BufferedStorageBackendConfig{ + BufferSize: 2, + NumWorkers: 1, + RetryLimit: 2, + RetryWait: 10 * time.Millisecond, + }, ds, schema) + require.NoError(t, err) + + objectKey0 := schema.GetObjectKeyFromSequenceNumber(0) + ds.On("GetFile", mock.Anything, objectKey0). + Return(nil, fmt.Errorf("transient network error")) + + ctx := context.Background() + require.NoError(t, backend.PrepareRange(ctx, ledgerbackend.BoundedRange(0, 9))) + + _, err = backend.GetLedger(ctx, 0) + require.Error(t, err) + assert.ErrorIs(t, err, ErrBufferDead) + assert.NotErrorIs(t, err, context.Canceled, "a dead buffer must be distinguishable from a caller-driven shutdown") + + // A second call against the same (now-dead) backend returns the same + // permanent error rather than hanging or succeeding. + _, err = backend.GetLedger(ctx, 0) + require.Error(t, err) + assert.ErrorIs(t, err, ErrBufferDead) + + require.NoError(t, backend.Close()) +} + +func TestDatastoreBackend_ParentContextCancelled_IsNotBufferDead(t *testing.T) { + // The buffer's ctx is derived from the ctx passed to PrepareRange, so + // cancelling that parent (a shutdown signal, not a worker giving up) also + // fires buf.ctx.Done() with cause context.Canceled. GetLedger's error must + // not claim ErrBufferDead in that case, or a graceful shutdown would be + // misclassified as a permanent ledger-fetch failure worth failing fast on. + // + // GetFile always returns a transient error with a long RetryWait, so the + // worker is reliably still sleeping between attempts (never exhausting + // RetryLimit itself, which would call buf.cancel with a real cause) when + // the test cancels the parent context. + ds := &mockDataStore{} + schema := testSchema(10) + + backend, err := newDatastoreBackend(ledgerbackend.BufferedStorageBackendConfig{ + BufferSize: 2, + NumWorkers: 1, + RetryLimit: 5, + RetryWait: 5 * time.Second, + }, ds, schema) + require.NoError(t, err) + + // mock.Anything for the path too: with BufferSize 2 and NumWorkers 1, the + // construction seed loop queues more than one task (sequences 0, 10, 20), + // and which one the single worker is retrying when the parent context is + // cancelled is not deterministic. + ds.On("GetFile", mock.Anything, mock.Anything).Return(nil, fmt.Errorf("transient network error")) + + parentCtx, cancel := context.WithCancel(context.Background()) + require.NoError(t, backend.PrepareRange(parentCtx, ledgerbackend.UnboundedRange(0))) + cancel() + + require.Eventually(t, func() bool { + _, getErr := backend.GetLedger(context.Background(), 0) + return getErr != nil + }, time.Second, time.Millisecond, "GetLedger should error once the parent context cancellation propagates") + + _, err = backend.GetLedger(context.Background(), 0) + require.Error(t, err) + assert.ErrorIs(t, err, context.Canceled) + assert.NotErrorIs(t, err, ErrBufferDead) +} + func TestDatastoreBackend_IsPrepared(t *testing.T) { ds := &mockDataStore{} schema := testSchema(10) diff --git a/internal/ingest/ingest.go b/internal/ingest/ingest.go index 494fbaba0..dc3682249 100644 --- a/internal/ingest/ingest.go +++ b/internal/ingest/ingest.go @@ -186,8 +186,9 @@ func setupDeps(ctx context.Context, cfg Configs) (services.IngestService, func() return nil, nil, fmt.Errorf("creating ledger backend: %w", err) } - // Create pond pool for contract metadata fetching - contractMetadataPool := pond.NewPool(0) + // Create pond pool for contract metadata fetching, bounded to the batch size + // ContractMetadataService itself submits per round-trip (pond.NewPool(0) is unbounded). + contractMetadataPool := pond.NewPool(services.SimulateTransactionBatchSize) metrics.RegisterPoolMetrics(m.Registry(), "contract_metadata", contractMetadataPool) // Create ContractMetadataService for fetching and storing token metadata @@ -271,13 +272,16 @@ func setupDeps(ctx context.Context, cfg Configs) (services.IngestService, func() wasmExtractor := services.NewWasmSpecExtractor() ingestService, err := services.NewIngestService(services.IngestServiceConfig{ - IngestionMode: cfg.IngestionMode, - Models: models, - OldestLedgerCursorName: cfg.OldestLedgerCursorName, - AppTracker: cfg.AppTracker, - RPCService: rpcService, - LedgerBackend: ledgerBackend, - LedgerBackendFactory: ledgerBackendFactory, + IngestionMode: cfg.IngestionMode, + Models: models, + OldestLedgerCursorName: cfg.OldestLedgerCursorName, + AppTracker: cfg.AppTracker, + RPCService: rpcService, + LedgerBackend: ledgerBackend, + LedgerBackendFactory: ledgerBackendFactory, + // Never matches for the RPC backend (it never wraps ErrBufferDead), so this is safe + // to wire unconditionally rather than branching on cfg.LedgerBackendType. + IsPermanentFetchError: func(err error) bool { return errors.Is(err, ErrBufferDead) }, TokenIngestionService: tokenIngestionService, CheckpointService: checkpointService, Metrics: m, diff --git a/internal/metrics/ingestion.go b/internal/metrics/ingestion.go index efa538ab5..4ca6853fa 100644 --- a/internal/metrics/ingestion.go +++ b/internal/metrics/ingestion.go @@ -10,7 +10,10 @@ type IngestionMetrics struct { // OldestLedger tracks the oldest ingested ledger (backfill boundary). // PromQL: wallet_ingestion_oldest_ledger OldestLedger prometheus.Gauge - // Duration observes end-to-end ledger ingestion time (fetch + process + persist). + // Duration observes ledger processing time: process_ledger + prepare_classification + + // insert_into_db. It excludes the ledger-fetch phase (including tip-wait) by design — see + // LedgerFetchDuration for that — since fetch time is bounded by backend/tip latency, not by + // this process's own work. // PromQL: histogram_quantile(0.99, rate(wallet_ingestion_duration_seconds_bucket[5m])) Duration prometheus.Histogram // PhaseDuration observes per-phase ingestion time, labeled by phase. @@ -31,7 +34,10 @@ type IngestionMetrics struct { // LagLedgers tracks how far behind ingestion is from the backend tip. // PromQL: wallet_ingestion_lag_ledgers > 100 LagLedgers prometheus.Gauge - // LedgerFetchDuration observes time to fetch a ledger from the backend (including retries). + // LedgerFetchDuration observes time to fetch a ledger from the backend, including retry + // overhead AND tip-wait: at the live tip, the backend blocks GetLedger until the next ledger + // closes, so this metric's steady-state floor tracks ledger close cadence (~5-6s) rather than + // pure I/O latency — that is the intended semantics, not a measurement bug. // PromQL: histogram_quantile(0.99, rate(wallet_ingestion_ledger_fetch_duration_seconds_bucket[5m])) LedgerFetchDuration prometheus.Histogram // RetriesTotal counts individual retry attempts by operation. @@ -72,7 +78,7 @@ func newIngestionMetrics(reg prometheus.Registerer) *IngestionMetrics { }), Duration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "wallet_ingestion_duration_seconds", - Help: "Duration of ledger ingestion.", + Help: "Duration of ledger processing (process + persist). Excludes the ledger-fetch phase, including tip-wait; see wallet_ingestion_ledger_fetch_duration_seconds for that.", Buckets: []float64{0.05, 0.1, 0.15, 0.25, 0.5, 0.75, 1, 1.5, 2, 3, 5, 7, 10}, }), PhaseDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{ @@ -103,7 +109,7 @@ func newIngestionMetrics(reg prometheus.Registerer) *IngestionMetrics { }), LedgerFetchDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "wallet_ingestion_ledger_fetch_duration_seconds", - Help: "Time to fetch a ledger from the backend including retry overhead.", + Help: "Time to fetch a ledger from the backend, including retry overhead and tip-wait. At the live tip, this tracks ledger close cadence rather than pure fetch latency — that is the intended semantics.", Buckets: []float64{0.01, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30}, }), RetriesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ diff --git a/internal/services/contract_metadata.go b/internal/services/contract_metadata.go index 6a3c7df1b..6d1fd2c7f 100644 --- a/internal/services/contract_metadata.go +++ b/internal/services/contract_metadata.go @@ -25,9 +25,12 @@ import ( // errors / strings imports above already cover everything the retry helpers need. const ( - // simulateTransactionBatchSize is the number of contracts to process in parallel - // when fetching metadata via RPC simulation. - simulateTransactionBatchSize = 20 + // SimulateTransactionBatchSize is the number of contracts to process in parallel + // when fetching metadata via RPC simulation. Exported so callers that size a + // worker pool to serve this batch size (cmd/ingest's contract-metadata pool, + // the SEP-41 validator's private pool) reference the same value instead of a + // hardcoded copy that can drift out of sync. + SimulateTransactionBatchSize = 20 // batchSleepDuration is the delay between batches to avoid overwhelming the RPC. batchSleepDuration = 2 * time.Second @@ -142,8 +145,8 @@ func (s *contractMetadataService) FetchSACMetadata(ctx context.Context, contract ) // Process in batches to avoid overwhelming the RPC - for i := 0; i < len(contractIDs); i += simulateTransactionBatchSize { - end := min(i+simulateTransactionBatchSize, len(contractIDs)) + for i := 0; i < len(contractIDs); i += SimulateTransactionBatchSize { + end := min(i+SimulateTransactionBatchSize, len(contractIDs)) batch := contractIDs[i:end] group := s.pool.NewGroupContext(ctx) diff --git a/internal/services/ingest.go b/internal/services/ingest.go index 3e3efdb95..e035e7e8f 100644 --- a/internal/services/ingest.go +++ b/internal/services/ingest.go @@ -56,6 +56,11 @@ type IngestServiceConfig struct { // === Ledger Backend === LedgerBackend ledgerbackend.LedgerBackend LedgerBackendFactory LedgerBackendFactory + // IsPermanentFetchError classifies a GetLedger error as permanent, i.e. no + // amount of retrying will fix it (e.g. the datastore backend's prefetch + // buffer has died). Optional: nil (the zero value) retries every + // ledger-fetch error as before. + IsPermanentFetchError func(error) bool // === Cursors === OldestLedgerCursorName string @@ -114,6 +119,7 @@ type ingestService struct { rpcService RPCService ledgerBackend ledgerbackend.LedgerBackend ledgerBackendFactory LedgerBackendFactory + isPermanentFetchError func(error) bool tokenIngestionService TokenIngestionService checkpointService CheckpointService appMetrics *metrics.Metrics @@ -129,11 +135,21 @@ type ingestService struct { protocolProcessors map[string]ProtocolProcessor protocolValidators []ProtocolValidator wasmSpecExtractor WasmSpecExtractor + // protocolCursors tracks, per protocol, whether its history/current-state + // ingest_store cursor rows exist. See casProtocolCursor and + // snapshotProtocolCursors in ingest_live.go. Always non-nil; empty maps + // (the zero value for every protocol) mean "not yet initialized", + // matching the pre-startLiveIngestion state. + protocolCursors *protocolCursorSnapshot } func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { - // Create worker pool for the ledger indexer (parallel transaction processing within a ledger) - ledgerIndexerPool := pond.NewPool(0) + // Create worker pool for the ledger indexer (parallel transaction processing within a + // ledger). This is CPU-bound XDR decode/processing work, not RPC-bound, so it's sized off + // NumCPU rather than an RPC batch size; 2x gives headroom for goroutines blocked on the + // occasional DB lookup without letting the pool grow unbounded (pond.NewPool(0) is + // unbounded). + ledgerIndexerPool := pond.NewPool(2 * runtime.NumCPU()) cfg.Metrics.RegisterPoolMetrics("ledger_indexer", ledgerIndexerPool) // Create backfill pool with bounded size to control memory usage. @@ -167,6 +183,7 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { rpcService: cfg.RPCService, ledgerBackend: cfg.LedgerBackend, ledgerBackendFactory: cfg.LedgerBackendFactory, + isPermanentFetchError: cfg.IsPermanentFetchError, tokenIngestionService: cfg.TokenIngestionService, checkpointService: cfg.CheckpointService, appMetrics: cfg.Metrics, @@ -182,6 +199,10 @@ func NewIngestService(cfg IngestServiceConfig) (*ingestService, error) { backfillDBInsertBatchSize: uint32(cfg.BackfillDBInsertBatchSize), knownContractIDs: set.NewSet[string](), protocolProcessors: ppMap, + protocolCursors: &protocolCursorSnapshot{ + historyExists: make(map[string]bool, len(ppMap)), + currentStateExists: make(map[string]bool, len(ppMap)), + }, }, nil } @@ -226,7 +247,7 @@ func (m *ingestService) insertIntoDB(ctx context.Context, dbTx pgx.Tx, buffer in if err := m.insertStateChanges(ctx, dbTx, stateChanges); err != nil { return 0, 0, err } - log.Ctx(ctx).Infof("✅ inserted %d txs, %d ops, %d state_changes", len(txs), len(ops), len(stateChanges)) + log.Ctx(ctx).Debugf("✅ inserted %d txs, %d ops, %d state_changes", len(txs), len(ops), len(stateChanges)) return len(txs), len(ops), nil } diff --git a/internal/services/ingest_live.go b/internal/services/ingest_live.go index 11ec71755..a962c14f3 100644 --- a/internal/services/ingest_live.go +++ b/internal/services/ingest_live.go @@ -10,6 +10,7 @@ import ( set "github.com/deckarep/golang-set/v2" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/stellar/go-stellar-sdk/ingest/ledgerbackend" "github.com/stellar/go-stellar-sdk/support/log" "github.com/stellar/go-stellar-sdk/xdr" @@ -115,20 +116,34 @@ func (m *ingestService) persistLedgerData( } for protocolID, processor := range m.protocolProcessors { - historyCursor := utils.ProtocolHistoryCursorName(protocolID) - currentStateCursor := utils.ProtocolCurrentStateCursorName(protocolID) - - historySwapped, casErr := m.casProtocolCursor(ctx, dbTx, historyCursor, expected, next) - if casErr != nil { - return casErr + // Only attempt the CAS for a cursor m.protocolCursors believes exists (see + // snapshotProtocolCursors/reprobeProtocolCursors): a cursor known not yet + // initialized is skipped entirely — no DB round trip, no metric — since + // there is nothing to CAS against. This also means casProtocolCursor only + // ever sees ErrCASCursorMissing for a cursor that existed as of the last + // snapshot/re-probe, i.e. a genuine incident, not the operationally normal + // not-yet-initialized case. + var historySwapped, currentStateSwapped bool + if m.protocolCursors.historyExists[protocolID] { + var casErr error + historyCursor := utils.ProtocolHistoryCursorName(protocolID) + historySwapped, casErr = m.casProtocolCursor(ctx, dbTx, historyCursor, expected, next) + if casErr != nil { + return casErr + } } - currentStateSwapped, casErr := m.casProtocolCursor(ctx, dbTx, currentStateCursor, expected, next) - if casErr != nil { - return casErr + if m.protocolCursors.currentStateExists[protocolID] { + var casErr error + currentStateCursor := utils.ProtocolCurrentStateCursorName(protocolID) + currentStateSwapped, casErr = m.casProtocolCursor(ctx, dbTx, currentStateCursor, expected, next) + if casErr != nil { + return casErr + } } if !historySwapped && !currentStateSwapped { - // Behind tip (value mismatch) or not yet set up (missing row): another - // process owns this ledger, so there is nothing to stage. + // Behind tip (value mismatch), not yet set up (both cursors known + // missing), or the value mismatch a live CAS returns when another + // process already owns this ledger: nothing to stage. continue } @@ -206,8 +221,16 @@ func (m *ingestService) persistLedgerData( return fmt.Errorf("processing token changes for ledger %d: %w", ledgerSeq, txErr) } - // 6. Update the specified cursor - if txErr = m.models.IngestStore.Update(ctx, dbTx, cursorName, ledgerSeq); txErr != nil { + // 6. Update the specified cursor. The live latest-ledger cursor is guarded: a session + // that silently lost its advisory lock (server-side failover, see startLiveIngestion's + // checkLockSession) must not blindly overwrite a value a second instance already + // advanced, or the cursor could regress. All other cursors (loadtest, generic) keep the + // plain blind upsert — only one process ever owns them by construction. + if cursorName == data.LatestLedgerCursorName { + if txErr = m.models.IngestStore.UpdateGuarded(ctx, dbTx, cursorName, ledgerSeq); txErr != nil { + return fmt.Errorf("updating cursor for ledger %d: %w", ledgerSeq, txErr) + } + } else if txErr = m.models.IngestStore.Update(ctx, dbTx, cursorName, ledgerSeq); txErr != nil { return fmt.Errorf("updating cursor for ledger %d: %w", ledgerSeq, txErr) } @@ -248,6 +271,12 @@ func (m *ingestService) startLiveIngestion(ctx context.Context) error { } }() + // Snapshot which protocols' history/current-state cursors already exist, once, right + // after the lock is confirmed held. See casProtocolCursor and snapshotProtocolCursors. + if err := m.snapshotProtocolCursors(ctx); err != nil { + return fmt.Errorf("snapshotting protocol cursors: %w", err) + } + // Get latest ingested ledger to determine DB state latestIngestedLedger, err := m.models.IngestStore.Get(ctx, data.LatestLedgerCursorName) if err != nil { @@ -284,7 +313,18 @@ func (m *ingestService) startLiveIngestion(ctx context.Context) error { return fmt.Errorf("preparing unbounded ledger backend range from %d: %w", startLedger, err) } - return m.ingestLiveLedgers(ctx, startLedger) + // checkLockSession probes the SAME connection that holds the advisory lock: since we + // never unlock mid-run, that session staying alive is equivalent to the lock still being + // held. A CNPG failover kills the session server-side without this process observing the + // TCP disconnect, silently releasing the lock while pgxpool never destroys the (now + // server-dead) pooled conn — so without this probe, ingestLiveLedgers would keep writing + // through other pool connections even after a second instance acquires the lock. + checkLockSession := func(probeCtx context.Context) error { + var one int + return conn.QueryRow(probeCtx, "SELECT 1").Scan(&one) + } + + return m.ingestLiveLedgers(ctx, startLedger, checkLockSession) } // initializeCursors initializes both latest and oldest cursors to the same starting ledger. @@ -310,8 +350,15 @@ func lagLedgers(backendTip, latestIngested uint32) (float64, bool) { } // ingestLiveLedgers continuously processes ledgers starting from startLedger, -// updating cursors and metrics after each successful ledger. -func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint32) error { +// updating cursors and metrics after each successful ledger. checkLockSession +// is called once per ledger to verify the advisory-lock-holding Postgres +// session is still alive (see startLiveIngestion): a CNPG failover can kill +// that session server-side without this process observing the disconnect, so +// the lock is silently released while this loop keeps writing through other +// pool connections. Failing this probe is treated as fatal so the process +// exits and can re-acquire the lock cleanly on restart, rather than racing a +// second instance that acquired it in the meantime. +func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint32, checkLockSession func(ctx context.Context) error) error { currentLedger := startLedger log.Ctx(ctx).Infof("Starting ingestion from ledger: %d", currentLedger) @@ -341,19 +388,28 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 }() for { + if probeErr := checkLockSession(ctx); probeErr != nil { + m.appMetrics.Ingestion.ErrorsTotal.WithLabelValues("ingest_live").Inc() + return fmt.Errorf("advisory lock session is no longer alive, the lock may have been lost: %w", probeErr) + } + + fetchStart := time.Now() ledgerMeta, ledgerErr := utils.RetryWithBackoff(ctx, maxLedgerFetchRetries, maxRetryBackoff, func(ctx context.Context) (xdr.LedgerCloseMeta, error) { return m.ledgerBackend.GetLedger(ctx, currentLedger) }, func(attempt int, err error, backoff time.Duration) { + m.appMetrics.Ingestion.RetriesTotal.WithLabelValues("ledger_fetch").Inc() log.Ctx(ctx).Warnf("Error fetching ledger %d (attempt %d/%d): %v, retrying in %v...", currentLedger, attempt+1, maxLedgerFetchRetries, err, backoff) }, + m.isPermanentFetchError, ) if ledgerErr != nil { m.appMetrics.Ingestion.ErrorsTotal.WithLabelValues("ingest_live").Inc() return fmt.Errorf("fetching ledger %d: %w", currentLedger, ledgerErr) } + m.appMetrics.Ingestion.LedgerFetchDuration.Observe(time.Since(fetchStart).Seconds()) totalStart := time.Now() processStart := time.Now() @@ -395,11 +451,15 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 // Publish the just-ingested ledger for the lag updater goroutine started above. latestIngested.Store(currentLedger) - // Periodically sync oldest ledger metric from DB (picks up changes from backfill jobs) + // Periodically sync oldest ledger metric from DB (picks up changes from backfill jobs), + // and re-probe protocol cursors that were missing at the last snapshot/re-probe (picks + // up a protocol-setup/migrate run that has initialized one since — see + // reprobeProtocolCursors). if currentLedger%oldestLedgerSyncInterval == 0 { if oldest, syncErr := m.models.IngestStore.Get(ctx, m.oldestLedgerCursorName); syncErr == nil { m.appMetrics.Ingestion.OldestLedger.Set(float64(oldest)) } + m.reprobeProtocolCursors(ctx) } log.Ctx(ctx).Infof("Ingested ledger %d in %.4fs", currentLedger, totalIngestionDuration) @@ -407,19 +467,106 @@ func (m *ingestService) ingestLiveLedgers(ctx context.Context, startLedger uint3 } } -// casProtocolCursor performs the authoritative compare-and-swap on a protocol -// cursor. A missing cursor row (ErrCASCursorMissing) is operationally normal — the -// protocol is registered but protocol-setup / protocol-migrate has not initialized -// its cursor yet — so it is folded into a soft skip (false, nil); the data layer -// still records the cursor_missing query-error metric. A value mismatch (another -// process already owns this ledger) likewise returns (false, nil) from the model. -// Any other error is returned so the surrounding transaction aborts and retries. +// protocolCursorSnapshot records, per protocol, whether its history and +// current-state ingest_store cursor rows exist. It only ever promotes an +// entry from missing to existing (see reprobeProtocolCursors) — a row +// vanishing after having existed is the genuine incident casProtocolCursor's +// error path handles, not something this snapshot demotes on its own. It is +// read and mutated only from the single live-ingestion goroutine — including +// retried persistLedgerData attempts, which run synchronously on it — so it +// needs no locking. +type protocolCursorSnapshot struct { + historyExists map[string]bool + currentStateExists map[string]bool +} + +// snapshotProtocolCursors populates m.protocolCursors from the DB, once, so +// casProtocolCursor's callers know which protocols' history/current-state +// cursors are operationally not-yet-initialized (skip the CAS silently) +// versus expected-present (a CAS reporting the row missing is a genuine +// incident). Called once by startLiveIngestion right after the advisory lock +// is confirmed held; ingestLiveLedgers re-probes the still-missing subset on +// the oldestLedgerSyncInterval cadence via reprobeProtocolCursors. +func (m *ingestService) snapshotProtocolCursors(ctx context.Context) error { + if len(m.protocolProcessors) == 0 { + return nil + } + keys := make([]string, 0, len(m.protocolProcessors)*2) + for protocolID := range m.protocolProcessors { + keys = append(keys, utils.ProtocolHistoryCursorName(protocolID), utils.ProtocolCurrentStateCursorName(protocolID)) + } + existing, err := m.models.IngestStore.GetMany(ctx, keys) + if err != nil { + return fmt.Errorf("getting protocol cursor rows: %w", err) + } + for protocolID := range m.protocolProcessors { + _, hExists := existing[utils.ProtocolHistoryCursorName(protocolID)] + _, csExists := existing[utils.ProtocolCurrentStateCursorName(protocolID)] + m.protocolCursors.historyExists[protocolID] = hExists + m.protocolCursors.currentStateExists[protocolID] = csExists + if !hExists { + log.Ctx(ctx).Infof("protocol %s history production disabled; cursor not initialized", protocolID) + } + if !csExists { + log.Ctx(ctx).Infof("protocol %s current-state production disabled; cursor not initialized", protocolID) + } + } + return nil +} + +// reprobeProtocolCursors re-checks the ingest_store rows for protocols whose history or +// current-state cursor was missing at the last snapshot/re-probe, promoting missing -> +// existing when a protocol-setup/migrate run has initialized the row since (so production +// starts without a restart). Cursors already known to exist are never re-checked here — a row +// vanishing after having existed is the genuine incident casProtocolCursor's error path +// handles. Runs outside any transaction, on the oldestLedgerSyncInterval cadence; a DB error is +// logged and skipped (best-effort, like the oldest-ledger metric sync it runs alongside), not +// fatal — the next cadence tick tries again. +func (m *ingestService) reprobeProtocolCursors(ctx context.Context) { + var keys []string + for protocolID := range m.protocolProcessors { + if !m.protocolCursors.historyExists[protocolID] { + keys = append(keys, utils.ProtocolHistoryCursorName(protocolID)) + } + if !m.protocolCursors.currentStateExists[protocolID] { + keys = append(keys, utils.ProtocolCurrentStateCursorName(protocolID)) + } + } + if len(keys) == 0 { + return + } + existing, err := m.models.IngestStore.GetMany(ctx, keys) + if err != nil { + log.Ctx(ctx).Warnf("re-probing protocol cursors: %v", err) + return + } + for protocolID := range m.protocolProcessors { + if !m.protocolCursors.historyExists[protocolID] { + if _, ok := existing[utils.ProtocolHistoryCursorName(protocolID)]; ok { + m.protocolCursors.historyExists[protocolID] = true + log.Ctx(ctx).Infof("protocol %s history cursor initialized; production enabled", protocolID) + } + } + if !m.protocolCursors.currentStateExists[protocolID] { + if _, ok := existing[utils.ProtocolCurrentStateCursorName(protocolID)]; ok { + m.protocolCursors.currentStateExists[protocolID] = true + log.Ctx(ctx).Infof("protocol %s current-state cursor initialized; production enabled", protocolID) + } + } + } +} + +// casProtocolCursor performs the authoritative compare-and-swap on a protocol cursor. +// Callers only invoke this for a cursor m.protocolCursors marks as existing (see the gating in +// persistLedgerData) — a cursor known not yet initialized skips this call entirely, so +// ErrCASCursorMissing here always means a row that existed has since vanished: a genuine +// incident (dropped row, bad restore), not the operationally normal not-yet-initialized case. +// That is surfaced as an error like any other so the transaction aborts and live ingestion +// retries and eventually exits, rather than silently losing protocol state. A value mismatch +// (another process already owns this ledger — the normal, harmless CAS-lost-the-race case) +// still returns (false, nil). func (m *ingestService) casProtocolCursor(ctx context.Context, dbTx pgx.Tx, cursorName, expected, next string) (bool, error) { swapped, err := m.models.IngestStore.CompareAndSwap(ctx, dbTx, cursorName, expected, next) - if errors.Is(err, data.ErrCASCursorMissing) { - log.Ctx(ctx).Debugf("protocol cursor %s missing at ledger %s; skipping production for this ledger", cursorName, next) - return false, nil - } if err != nil { return false, fmt.Errorf("comparing and swapping protocol cursor %s: %w", cursorName, err) } @@ -506,6 +653,10 @@ func (m *ingestService) ingestProcessedDataWithRetry( return numTxs, numOps, nil } lastErr = err + if isPermanentPersistError(err) { + m.appMetrics.Ingestion.ErrorsTotal.WithLabelValues("db_persist").Inc() + return 0, 0, fmt.Errorf("ingesting processed data for ledger %d failed with a permanent error: %w", currentLedger, err) + } m.appMetrics.Ingestion.RetriesTotal.WithLabelValues("db_persist").Inc() if attempt == maxIngestProcessedDataRetries-1 { break @@ -529,6 +680,32 @@ func (m *ingestService) ingestProcessedDataWithRetry( return 0, 0, fmt.Errorf("ingesting processed data failed after %d attempts: %w", maxIngestProcessedDataRetries, lastErr) } +// isPermanentPersistError classifies a persistLedgerData failure using its PostgreSQL SQLSTATE, +// when available, via errors.As through the wrap chain. SQLSTATE class 22 (data exception), 23 +// (integrity constraint violation), and 42 (syntax or access-rule violation, e.g. an undefined +// column after schema drift) can never succeed by retrying the same statement, so they are +// permanent. Class 40 (40001 serialization_failure, 40P01 deadlock_detected), class 08 +// (connection exception), and 57P0x (57P01 admin_shutdown, 57P02 crash_shutdown, 57P03 +// cannot_connect_now — all raised during a CNPG failover) are transient and fall through to the +// default retry behavior, same as any other SQLSTATE or a non-PgError (e.g. a context deadline). +// ErrCursorGuardFailed (see IngestStoreModel.UpdateGuarded) is also permanent: it means another +// writer already owns this ledger's cursor range, so retrying the same write cannot succeed. +func isPermanentPersistError(err error) bool { + if errors.Is(err, data.ErrCursorGuardFailed) { + return true + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) || len(pgErr.Code) < 2 { + return false + } + switch pgErr.Code[:2] { + case "22", "23", "42": + return true + default: + return false + } +} + // prepareClassificationPlan runs Phase A of protocol classification for this // ledger's buffered raw WASMs and contracts: pure signature matching plus any // RPC-sourced enrichment prefetch (e.g. SEP-41 token metadata), diff --git a/internal/services/ingest_live_test.go b/internal/services/ingest_live_test.go index f2965b5e1..7becb8651 100644 --- a/internal/services/ingest_live_test.go +++ b/internal/services/ingest_live_test.go @@ -2,8 +2,11 @@ package services import ( "context" + "errors" + "fmt" "testing" + "github.com/jackc/pgx/v5/pgconn" "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go-stellar-sdk/network" "github.com/stretchr/testify/assert" @@ -81,3 +84,51 @@ func Test_startLiveIngestion_ReleasesAdvisoryLockWhenContextCancelledMidStartup( require.NoError(t, err) assert.True(t, acquired, "advisory lock should have been released during shutdown despite the cancelled context") } + +// Test_isPermanentPersistError covers ING-06's classifier: SQLSTATE class 22/23/42 (and +// ErrCursorGuardFailed) must fail an ingestion attempt immediately instead of burning the full +// 5-attempt retry ladder; every other error (including no PgError at all) must still retry, same +// as before this classifier existed. +func Test_isPermanentPersistError(t *testing.T) { + pgErrWithCode := func(code string) error { + return &pgconn.PgError{Code: code, Message: "boom"} + } + + testCases := []struct { + name string + err error + wantPermanent bool + }{ + {name: "nil_error", err: nil, wantPermanent: false}, + {name: "plain_error_no_pgerror", err: errors.New("db connection failed"), wantPermanent: false}, + {name: "data_exception_22001_string_data_right_truncation", err: pgErrWithCode("22001"), wantPermanent: true}, + {name: "integrity_constraint_23505_unique_violation", err: pgErrWithCode("23505"), wantPermanent: true}, + {name: "integrity_constraint_23503_foreign_key_violation", err: pgErrWithCode("23503"), wantPermanent: true}, + {name: "syntax_or_access_rule_42501_insufficient_privilege", err: pgErrWithCode("42501"), wantPermanent: true}, + {name: "syntax_or_access_rule_42P01_undefined_table", err: pgErrWithCode("42P01"), wantPermanent: true}, + {name: "serialization_failure_40001_is_transient", err: pgErrWithCode("40001"), wantPermanent: false}, + {name: "deadlock_detected_40P01_is_transient", err: pgErrWithCode("40P01"), wantPermanent: false}, + {name: "connection_exception_08006_is_transient", err: pgErrWithCode("08006"), wantPermanent: false}, + {name: "admin_shutdown_57P01_is_transient_cnpg_failover", err: pgErrWithCode("57P01"), wantPermanent: false}, + {name: "crash_shutdown_57P02_is_transient_cnpg_failover", err: pgErrWithCode("57P02"), wantPermanent: false}, + {name: "cannot_connect_now_57P03_is_transient_cnpg_failover", err: pgErrWithCode("57P03"), wantPermanent: false}, + {name: "unknown_sqlstate_defaults_to_transient", err: pgErrWithCode("99999"), wantPermanent: false}, + { + name: "wrapped_pgerror_still_classified_via_errors_As", + err: fmt.Errorf("persisting ledger data for ledger 100: running atomic function in RunInTransaction: %w", pgErrWithCode("23505")), + wantPermanent: true, + }, + {name: "cursor_guard_failed_is_permanent", err: data.ErrCursorGuardFailed, wantPermanent: true}, + { + name: "wrapped_cursor_guard_failed_is_permanent", + err: fmt.Errorf("updating cursor for ledger 100: %w", data.ErrCursorGuardFailed), + wantPermanent: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.wantPermanent, isPermanentPersistError(tc.err)) + }) + } +} diff --git a/internal/services/ingest_test.go b/internal/services/ingest_test.go index 97188b27e..a34bbcec3 100644 --- a/internal/services/ingest_test.go +++ b/internal/services/ingest_test.go @@ -1969,6 +1969,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { processor.processedLedger = 100 setupDBCursors(t, ctx, pool, 99, 99) setupProtocolCursors(t, ctx, pool, 99, 99) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) @@ -2001,6 +2002,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { processor.processedLedger = 100 setupDBCursors(t, ctx, pool, 99, 99) setupProtocolCursors(t, ctx, pool, 100, 100) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) @@ -2033,6 +2035,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { processor.processedLedger = 100 setupDBCursors(t, ctx, pool, 99, 99) setupProtocolCursors(t, ctx, pool, 98, 98) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) buffer := indexer.NewIndexerBuffer() meta := dummyLedgerMeta(100) @@ -2108,6 +2111,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { `INSERT INTO ingest_store (key, value) VALUES ($1, $2)`, utils.ProtocolHistoryCursorName("testproto"), "99") require.NoError(t, err) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) meta := dummyLedgerMeta(100) _, _, err = svc.persistLedgerData(ctx, 100, &meta, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") @@ -2147,6 +2151,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { `INSERT INTO ingest_store (key, value) VALUES ($1, $2)`, utils.ProtocolCurrentStateCursorName("testproto"), "99") require.NoError(t, err) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) meta := dummyLedgerMeta(100) _, _, err = svc.persistLedgerData(ctx, 100, &meta, nil, indexer.NewIndexerBuffer(), "latest_ledger_cursor") @@ -2193,6 +2198,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { processor.ingestStore = models.IngestStore setupDBCursors(t, ctx, pool, 99, 99) setupProtocolCursors(t, ctx, pool, 99, 99) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) // First ledger succeeds and advances the current-state cursor to 100. processor.processedLedger = 100 @@ -2226,6 +2232,70 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { assert.Equal(t, uint32(1), stagedCount) // retry re-staged cleanly; not doubled }) + t.Run("K: cursor existed at snapshot but is now missing — hard error (incident)", func(t *testing.T) { + // ING-12: casProtocolCursor only calls CompareAndSwap for a cursor + // snapshotProtocolCursors believed existed. If the row has since + // vanished (dropped row, bad restore — never a normal live-ingestion + // state transition), that must abort the transaction as a genuine + // incident rather than the operationally-normal soft skip. + processor := &testProtocolProcessor{id: "testproto"} + ctx, svc, models, pool := setupTest(t, []ProtocolProcessor{processor}) + processor.ingestStore = models.IngestStore + processor.processedLedger = 100 + setupDBCursors(t, ctx, pool, 99, 99) + setupProtocolCursors(t, ctx, pool, 99, 99) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) // snapshot sees both cursors existing + + // Simulate the incident: the history cursor row vanishes after the snapshot. + _, delErr := pool.Exec(ctx, `DELETE FROM ingest_store WHERE key = $1`, utils.ProtocolHistoryCursorName("testproto")) + require.NoError(t, delErr) + + buffer := indexer.NewIndexerBuffer() + meta := dummyLedgerMeta(100) + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") + require.Error(t, err) + assert.ErrorIs(t, err, data.ErrCASCursorMissing) + + // The transaction rolled back entirely: even the unrelated main cursor + // (a different cursorName, not gated by any snapshot) did not advance. + mainCursor, getErr := models.IngestStore.Get(ctx, "latest_ledger_cursor") + require.NoError(t, getErr) + assert.Equal(t, uint32(0), mainCursor) + + // The still-existing current-state cursor is untouched (whole tx rolled back). + csCursor, getErr := models.IngestStore.Get(ctx, "protocol_testproto_current_state_cursor") + require.NoError(t, getErr) + assert.Equal(t, uint32(99), csCursor) + }) + + t.Run("L: reprobeProtocolCursors promotes a cursor initialized after the snapshot", func(t *testing.T) { + processor := &testProtocolProcessor{id: "testproto"} + ctx, svc, models, pool := setupTest(t, []ProtocolProcessor{processor}) + processor.ingestStore = models.IngestStore + setupDBCursors(t, ctx, pool, 99, 99) + // No protocol cursors at snapshot time: both start out known-missing. + require.NoError(t, svc.snapshotProtocolCursors(ctx)) + assert.False(t, svc.protocolCursors.historyExists["testproto"]) + assert.False(t, svc.protocolCursors.currentStateExists["testproto"]) + + // A protocol-setup/migrate run initializes both cursors afterward, without a restart. + setupProtocolCursors(t, ctx, pool, 99, 99) + svc.reprobeProtocolCursors(ctx) + assert.True(t, svc.protocolCursors.historyExists["testproto"]) + assert.True(t, svc.protocolCursors.currentStateExists["testproto"]) + + // Production is now enabled: a ledger processed after the re-probe actually CASes. + processor.processedLedger = 100 + buffer := indexer.NewIndexerBuffer() + meta := dummyLedgerMeta(100) + _, _, err := svc.persistLedgerData(ctx, 100, &meta, nil, buffer, "latest_ledger_cursor") + require.NoError(t, err) + + histCursor, getErr := models.IngestStore.Get(ctx, "protocol_testproto_history_cursor") + require.NoError(t, getErr) + assert.Equal(t, uint32(100), histCursor) + }) + t.Run("G: contract-id lookup failure fails the ledger", func(t *testing.T) { processor := &testProtocolProcessor{id: "testproto"} ctx, svc, models, pool := setupTest(t, []ProtocolProcessor{processor}) @@ -2233,6 +2303,7 @@ func Test_PersistLedgerData_ProtocolCASGating(t *testing.T) { processor.processedLedger = 100 setupDBCursors(t, ctx, pool, 99, 99) setupProtocolCursors(t, ctx, pool, 99, 99) + require.NoError(t, svc.snapshotProtocolCursors(ctx)) // Inject a failing contract-id lookup over the otherwise-real models. contractsMock := data.NewProtocolContractsModelMock(t) @@ -2349,7 +2420,12 @@ func Test_ingestService_ingestLiveLedgers_LagReadDoesNotBlockConsumer(t *testing require.NoError(t, err) defer pool.Close() - setupDBCursors(t, ctx, pool, 0, 0) + // The guarded latest cursor (see IngestStoreModel.UpdateGuarded) requires the current DB + // value to be startLedger-1 or startLedger, matching the real startLiveIngestion precondition + // (initializeCursors/the previous ledger's write always leave the cursor one behind the next + // ledger to process). + const startLedger = uint32(51) // not a multiple of oldestLedgerSyncInterval (100) + setupDBCursors(t, ctx, pool, startLedger-1, startLedger-1) m := metrics.NewMetrics(prometheus.NewRegistry()) models, err := data.NewModels(pool, m.DB) @@ -2391,9 +2467,9 @@ func Test_ingestService_ingestLiveLedgers_LagReadDoesNotBlockConsumer(t *testing }) require.NoError(t, err) - const startLedger = uint32(51) // not a multiple of oldestLedgerSyncInterval (100) + noopCheckLockSession := func(context.Context) error { return nil } done := make(chan error, 1) - go func() { done <- svc.ingestLiveLedgers(ctx, startLedger) }() + go func() { done <- svc.ingestLiveLedgers(ctx, startLedger, noopCheckLockSession) }() // The consumer must keep draining and advancing the cursor despite the blocked lag read. require.Eventually(t, func() bool { @@ -2414,3 +2490,58 @@ func Test_ingestService_ingestLiveLedgers_LagReadDoesNotBlockConsumer(t *testing t.Fatal("ingestLiveLedgers did not return after context cancellation") } } + +// Test_ingestService_ingestLiveLedgers_DeadLockSessionExitsFatally is a regression test for +// ING-05: a CNPG failover can kill the Postgres session holding the advisory lock without this +// process observing the disconnect, silently releasing the lock while pgxpool never destroys the +// (now server-dead) pooled connection. checkLockSession must be probed every ledger and, on +// failure, ingestLiveLedgers must return immediately — not after exhausting the ledger-fetch +// retry ladder (maxLedgerFetchRetries attempts, up to ~2.5 minutes) — so the process can exit and +// re-acquire the lock cleanly on restart. +func Test_ingestService_ingestLiveLedgers_DeadLockSessionExitsFatally(t *testing.T) { + dbt := dbtest.Open(t) + defer dbt.Close() + ctx := context.Background() + + pool, err := db.OpenDBConnectionPool(ctx, dbt.DSN) + require.NoError(t, err) + defer pool.Close() + + const startLedger = uint32(51) + setupDBCursors(t, ctx, pool, startLedger-1, startLedger-1) + + m := metrics.NewMetrics(prometheus.NewRegistry()) + models, err := data.NewModels(pool, m.DB) + require.NoError(t, err) + + // GetLedger must never be reached: the liveness probe is checked at the top of the loop, + // before any ledger fetch. + mockBackend := &LedgerBackendMock{} + + svc, err := NewIngestService(IngestServiceConfig{ + IngestionMode: IngestionModeLive, + Models: models, + OldestLedgerCursorName: "oldest_ledger_cursor", + AppTracker: &apptracker.MockAppTracker{}, + RPCService: &RPCServiceMock{}, + LedgerBackend: mockBackend, + Metrics: m, + GetLedgersLimit: defaultGetLedgersLimit, + Network: network.TestNetworkPassphrase, + NetworkPassphrase: network.TestNetworkPassphrase, + Archive: &HistoryArchiveMock{}, + }) + require.NoError(t, err) + + sessionDeadErr := fmt.Errorf("driver: bad connection") + deadLockSession := func(context.Context) error { return sessionDeadErr } + + start := time.Now() + runErr := svc.ingestLiveLedgers(ctx, startLedger, deadLockSession) + elapsed := time.Since(start) + + require.Error(t, runErr) + assert.ErrorIs(t, runErr, sessionDeadErr) + assert.Less(t, elapsed, time.Second, "a dead lock session must fail fast, not after the ledger-fetch retry ladder") + mockBackend.AssertNotCalled(t, "GetLedger", mock.Anything, mock.Anything) +} diff --git a/internal/services/sep41/validator.go b/internal/services/sep41/validator.go index 0d5a84322..97efa0122 100644 --- a/internal/services/sep41/validator.go +++ b/internal/services/sep41/validator.go @@ -60,9 +60,11 @@ func NewValidator() *Validator { func newValidator(deps services.ProtocolDeps) *Validator { v := &Validator{} if deps.ContractMetadataService != nil { - // Owned worker pool — capped here to keep the public RPC endpoint - // happy under bursts. - v.pool = pond.NewPool(0) + // Owned worker pool, bounded to services.SimulateTransactionBatchSize to match + // the batch size the metadata fetcher itself submits per RPC round-trip + // (pond.NewPool(0) is unbounded and would let a large ledger burst the public + // RPC endpoint with unthrottled concurrent requests). + v.pool = pond.NewPool(services.SimulateTransactionBatchSize) v.fetcher = newMetadataFetcher(deps.ContractMetadataService, v.pool) } return v diff --git a/internal/services/token_ingestion.go b/internal/services/token_ingestion.go index db6f816a0..548881bcd 100644 --- a/internal/services/token_ingestion.go +++ b/internal/services/token_ingestion.go @@ -112,7 +112,7 @@ func (s *tokenIngestionService) processTrustlineChanges(ctx context.Context, dbT return fmt.Errorf("upserting trustline balances: %w", err) } } - log.Ctx(ctx).Infof("upserted %d trustlines, deleted %d trustlines", len(upserts), len(deletes)) + log.Ctx(ctx).Debugf("upserted %d trustlines, deleted %d trustlines", len(upserts), len(deletes)) return nil } @@ -144,7 +144,7 @@ func (s *tokenIngestionService) processNativeBalanceChanges(ctx context.Context, if err := s.nativeBalanceModel.BatchUpsert(ctx, dbTx, upserts, deletes); err != nil { return fmt.Errorf("upserting native balances: %w", err) } - log.Ctx(ctx).Infof("upserted %d native balances, deleted %d native balances", len(upserts), len(deletes)) + log.Ctx(ctx).Debugf("upserted %d native balances, deleted %d native balances", len(upserts), len(deletes)) } return nil } @@ -178,7 +178,7 @@ func (s *tokenIngestionService) processSACBalanceChanges(ctx context.Context, db if err := s.sacBalanceModel.BatchUpsert(ctx, dbTx, upserts, deletes); err != nil { return fmt.Errorf("upserting SAC balances: %w", err) } - log.Ctx(ctx).Infof("upserted %d SAC balances, deleted %d SAC balances", len(upserts), len(deletes)) + log.Ctx(ctx).Debugf("upserted %d SAC balances, deleted %d SAC balances", len(upserts), len(deletes)) } return nil } @@ -211,7 +211,7 @@ func (s *tokenIngestionService) processLiquidityPoolChanges(ctx context.Context, if err := s.liquidityPoolModel.BatchUpsert(ctx, dbTx, upserts, deletes); err != nil { return fmt.Errorf("upserting liquidity pools: %w", err) } - log.Ctx(ctx).Infof("upserted %d liquidity pools, deleted %d liquidity pools", len(upserts), len(deletes)) + log.Ctx(ctx).Debugf("upserted %d liquidity pools, deleted %d liquidity pools", len(upserts), len(deletes)) } return nil } @@ -242,7 +242,7 @@ func (s *tokenIngestionService) processLiquidityPoolShareChanges(ctx context.Con if err := s.liquidityPoolBalanceModel.BatchUpsert(ctx, dbTx, upserts, deletes); err != nil { return fmt.Errorf("upserting liquidity pool balances: %w", err) } - log.Ctx(ctx).Infof("upserted %d liquidity pool balances, deleted %d liquidity pool balances", len(upserts), len(deletes)) + log.Ctx(ctx).Debugf("upserted %d liquidity pool balances, deleted %d liquidity pool balances", len(upserts), len(deletes)) } return nil } diff --git a/internal/utils/retry.go b/internal/utils/retry.go index d4068c27f..58907b027 100644 --- a/internal/utils/retry.go +++ b/internal/utils/retry.go @@ -9,13 +9,19 @@ import ( // RetryWithBackoff calls fn up to maxRetries times with exponential backoff // capped at maxBackoff. It respects context cancellation between attempts. // onRetry, if non-nil, is called before each backoff wait with the attempt -// number (0-indexed), the error, and the backoff duration. +// number (0-indexed), the error, and the backoff duration. isPermanent is an +// optional classifier (omit it, or pass nil, to retry every error as +// before): when supplied and it reports an error as permanent, RetryWithBackoff +// returns immediately without backing off or consuming further attempts — +// for an error no amount of retrying can fix, running the full ladder only +// delays an unavoidable failure. func RetryWithBackoff[T any]( ctx context.Context, maxRetries int, maxBackoff time.Duration, fn func(ctx context.Context) (T, error), onRetry func(attempt int, err error, backoff time.Duration), + isPermanent ...func(error) bool, ) (T, error) { var zero T if maxRetries <= 0 { @@ -24,6 +30,10 @@ func RetryWithBackoff[T any]( if maxBackoff <= 0 { return zero, fmt.Errorf("RetryWithBackoff: maxBackoff must be > 0, got %s", maxBackoff) } + var permanent func(error) bool + if len(isPermanent) > 0 { + permanent = isPermanent[0] + } var lastErr error for attempt := 0; attempt < maxRetries; attempt++ { select { @@ -37,6 +47,9 @@ func RetryWithBackoff[T any]( return result, nil } lastErr = err + if permanent != nil && permanent(err) { + return zero, fmt.Errorf("permanent error on attempt %d: %w", attempt+1, err) + } if attempt == maxRetries-1 { break } diff --git a/internal/utils/retry_test.go b/internal/utils/retry_test.go index de53478e1..56ea351f7 100644 --- a/internal/utils/retry_test.go +++ b/internal/utils/retry_test.go @@ -92,6 +92,63 @@ func TestRetryWithBackoff_CapsBackoff(t *testing.T) { assert.Equal(t, []time.Duration{time.Second, maxBackoff, maxBackoff, maxBackoff}, observedBackoffs) } +func TestRetryWithBackoff_StopsOnPermanentError(t *testing.T) { + sentinel := errors.New("permanent failure") + attempts := 0 + var onRetryCalls int + + _, err := RetryWithBackoff(context.Background(), 5, 1*time.Second, + func(ctx context.Context) (string, error) { + attempts++ + return "", sentinel + }, + func(attempt int, err error, backoff time.Duration) { + onRetryCalls++ + }, + func(err error) bool { + return errors.Is(err, sentinel) + }, + ) + require.Error(t, err) + assert.ErrorIs(t, err, sentinel) + assert.Contains(t, err.Error(), "permanent error on attempt 1") + assert.Equal(t, 1, attempts, "a permanent error must not be retried") + assert.Equal(t, 0, onRetryCalls, "onRetry must not fire for a permanent error") +} + +func TestRetryWithBackoff_TransientErrorStillRetriesWithClassifier(t *testing.T) { + attempts := 0 + _, err := RetryWithBackoff(context.Background(), 3, 1*time.Second, + func(ctx context.Context) (int, error) { + attempts++ + if attempts < 2 { + return 0, errors.New("transient") + } + return 7, nil + }, + nil, + func(err error) bool { return false }, + ) + require.NoError(t, err) + assert.Equal(t, 2, attempts) +} + +func TestRetryWithBackoff_NilClassifierRetriesAsBefore(t *testing.T) { + sentinel := errors.New("fail") + attempts := 0 + _, err := RetryWithBackoff(context.Background(), 3, 1*time.Second, + func(ctx context.Context) (string, error) { + attempts++ + return "", sentinel + }, + nil, + nil, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed after 3 attempts") + assert.Equal(t, 3, attempts) +} + func TestRetryWithBackoff_RejectsZeroMaxRetries(t *testing.T) { _, err := RetryWithBackoff(context.Background(), 0, 5*time.Second, func(ctx context.Context) (string, error) { From 50ca382a52499c156298ba5705e02a75212f28f5 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 15 Jul 2026 11:42:57 -0400 Subject: [PATCH 4/4] fix(ingest): exempt the checkpoint load transaction from idle-in-transaction timeout Between batch flushes the load transaction's connection sits idle while the next entries are decoded from the history archive stream. A production idle_in_transaction_session_timeout (which protects the vacuum horizon from abandoned transactions) would kill the legitimately long-lived cold-start load mid-way and force a full redo; SET LOCAL scopes the exemption to this transaction only. --- internal/services/checkpoint.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/internal/services/checkpoint.go b/internal/services/checkpoint.go index 8b1a6fbd9..b3f6f095f 100644 --- a/internal/services/checkpoint.go +++ b/internal/services/checkpoint.go @@ -274,6 +274,15 @@ func (s *checkpointService) PopulateFromCheckpoint(ctx context.Context, checkpoi if _, txErr := dbTx.Exec(ctx, "SET LOCAL synchronous_commit = off"); txErr != nil { return fmt.Errorf("setting synchronous_commit=off: %w", txErr) } + // The connection sits idle-in-transaction between batch flushes while the + // next 250k entries are decoded from the history archive stream. An + // instance-level idle_in_transaction_session_timeout (set in production to + // protect the vacuum horizon from abandoned transactions) would kill this + // legitimately long-lived load mid-way and force a full redo, so exempt + // this transaction; SET LOCAL scopes the exemption to it alone. + if _, txErr := dbTx.Exec(ctx, "SET LOCAL idle_in_transaction_session_timeout = 0"); txErr != nil { + return fmt.Errorf("setting idle_in_transaction_session_timeout=0: %w", txErr) + } proc = &checkpointProcessor{ service: s,