Pre-production hardening: schema, queries, ingestion, and API

.github/workflows/build-cnpg-timescaledb-dev.yml

@@ -8,8 +8,8 @@ on:
         default: '17'
         required: true
       tsdb_version:
-        description: TimescaleDB version (semver, e.g. 2.25.0)
-        default: '2.25.0'
+        description: TimescaleDB version (semver, e.g. 2.28.2)
+        default: '2.28.2'
         required: true
 
 permissions:

.github/workflows/build-cnpg-timescaledb-stg.yml

@@ -8,8 +8,8 @@ on:
         default: '17'
         required: true
       tsdb_version:
-        description: TimescaleDB version (semver, e.g. 2.25.0)
-        default: '2.25.0'
+        description: TimescaleDB version (semver, e.g. 2.28.2)
+        default: '2.28.2'
         required: true
 
 permissions:

.github/workflows/go.yaml

@@ -82,7 +82,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: timescale/timescaledb:2.25.0-pg17
+        image: timescale/timescaledb:2.28.2-pg17
         env:
           POSTGRES_USER: postgres
           POSTGRES_DB: postgres

Dockerfile-timescale-cnpg

@@ -5,7 +5,7 @@
 # TimescaleDB from the apt repo and layers it onto the CNPG base image.
 #
 # Usage:
-#   docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.25.0 -t cnpg-timescaledb .
+#   docker build -f Dockerfile-timescale-cnpg --build-arg PG_MAJOR=17 --build-arg TSDB_VERSION=2.28.2 -t cnpg-timescaledb .
 
 ARG PG_MAJOR=17
 
@@ -15,7 +15,7 @@ ARG PG_MAJOR=17
 FROM ghcr.io/cloudnative-pg/postgresql:${PG_MAJOR}-bookworm AS timescaledb-builder
 
 ARG PG_MAJOR
-ARG TSDB_VERSION=2.25.0
+ARG TSDB_VERSION=2.28.2
 
 USER root
 

cmd/root.go

@@ -49,7 +49,6 @@ func SetupCLI(cfg RootConfig) {
 	rootCmd.AddCommand((&serveCmd{}).Command())
 	rootCmd.AddCommand((&ingestCmd{}).Command())
 	rootCmd.AddCommand((&migrateCmd{}).Command())
-	rootCmd.AddCommand((&loadtestCmd{}).Command())
 	rootCmd.AddCommand((&protocolSetupCmd{}).Command())
 	rootCmd.AddCommand((&protocolMigrateCmd{}).Command())
 }

cmd/serve.go

@@ -29,6 +29,7 @@ func (c *serveCmd) Command() *cobra.Command {
 		utils.StellarEnvironmentOption(&stellarEnvironment),
 		utils.ServerBaseURLOption(&cfg.ServerBaseURL),
 		utils.GraphQLComplexityLimitOption(&cfg.GraphQLComplexityLimit),
+		utils.GraphQLIntrospectionEnabledOption(&cfg.GraphQLIntrospectionEnabled),
 		utils.AdminPortOption(&cfg.AdminPort),
 		{
 			Name:        "port",

cmd/utils/global_options.go

@@ -192,6 +192,20 @@ func GraphQLComplexityLimitOption(configKey *int) *config.ConfigOption {
 	}
 }
 
+// GraphQLIntrospectionEnabledOption controls whether GraphQL schema introspection (__schema,
+// __type) is served. Defaults to disabled: introspection makes the full schema (including any
+// unreleased or internal-only fields) discoverable to anyone who can reach the endpoint.
+func GraphQLIntrospectionEnabledOption(configKey *bool) *config.ConfigOption {
+	return &config.ConfigOption{
+		Name:        "graphql-introspection-enabled",
+		Usage:       "Whether to enable GraphQL schema introspection (__schema, __type). Disabled by default in production.",
+		OptType:     types.Bool,
+		ConfigKey:   configKey,
+		FlagDefault: false,
+		Required:    false,
+	}
+}
+
 // DBPoolOptions returns config options for tuning the pgxpool connection pool.
 func DBPoolOptions(maxConns *int, minConns *int, maxConnLifetime *time.Duration, maxConnIdleTime *time.Duration) config.ConfigOptions {
 	return config.ConfigOptions{

docker-compose.yaml

@@ -7,7 +7,7 @@ services:
 
   db:
     container_name: db
-    image: timescale/timescaledb:2.25.0-pg17
+    image: timescale/timescaledb:2.28.2-pg17
     command: ["postgres", "-c", "timescaledb.enable_chunk_skipping=on", "-c", "timescaledb.enable_sparse_index_bloom=on"]
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres -d wallet-backend"]

docs/data-migrations/adding-a-protocol.md

@@ -166,13 +166,12 @@ CREATE TABLE <protocol>_<entity> (
     last_modified_ledger INTEGER NOT NULL DEFAULT 0,
     PRIMARY KEY (...)  -- Choose a primary key that fits your state's natural identity
 ) WITH (
-    fillfactor = 80,
+    fillfactor = 90,
     autovacuum_vacuum_scale_factor = 0.02,
     autovacuum_vacuum_threshold = 50,
     autovacuum_analyze_scale_factor = 0.01,
     autovacuum_analyze_threshold = 50,
-    autovacuum_vacuum_cost_delay = 0,
-    autovacuum_vacuum_cost_limit = 1000
+    autovacuum_vacuum_cost_delay = 0
 );
 
 -- Add indexes based on your query patterns
@@ -182,7 +181,19 @@ CREATE TABLE <protocol>_<entity> (
 DROP TABLE IF EXISTS <protocol>_<entity>;
 ```
 
-The storage parameters (`fillfactor = 80`, aggressive autovacuum) are tuned for heavy UPSERT workloads during ingestion. Adjust based on your protocol's write pattern.
+`fillfactor = 90` is the default: it reserves 10% free space per page for HOT (Heap-Only
+Tuple) updates without over-reserving for tables whose UPSERTs only touch non-indexed
+columns. Use `fillfactor = 80` instead only for tables whose rows update on essentially
+every ledger (e.g. AMM-style reserves that shift on every trade). `autovacuum_vacuum_cost_delay
+= 0` disables cost-based vacuum throttling for this table's worker; there is no
+`autovacuum_vacuum_cost_limit` to set alongside it since a delay of 0 makes any cost
+limit a no-op.
+
+The aggressive autovacuum block above is for tables whose row count grows with users or
+usage (positions, balances, allowances). Skip it entirely — keep only `fillfactor = 90`
+— for small, protocol-bounded current-state tables (pool configs, reserve configs) whose
+row count is capped by the number of pools/reserves your protocol will ever have
+deployed; default autovacuum behavior is fine for those.
 
 ### Data Model
 

docs/operations.md

@@ -0,0 +1,62 @@
+# Production Operations
+
+Reference for operating the wallet-backend's TimescaleDB-backed ingestion and API in production.
+
+## Production Database Checklist
+
+**CNPG `postgresql.parameters`:**
+
+| Parameter | Value | Why |
+|---|---|---|
+| `timescaledb.enable_chunk_skipping` | `on` | Off by default. Min/max ranges used for chunk skipping are recorded at compression time, so this must be on **before** the first ingest — enabling it later requires recompressing all existing history to backfill the ranges. |
+| `timescaledb.enable_sparse_index_bloom` | `on` | Enables bloom-filter sparse indexes on compressed chunks. |
+| `timescaledb.stats_max_chunks` | `2048` (2.28+ only) | Steady-state compressed chunk count at 1-year retention is ~1,820, which exceeds the 1,024 default stats cache. |
+
+**Instance sizing:**
+
+| Resource | Target | Why |
+|---|---|---|
+| Memory | >= 64 GB, `shared_buffers` 16 GB, `effective_cache_size` ~48 GB | The actively-ingested day's chunk indexes total ~14 GB, fitting the shared_buffers 25%-of-RAM guidance. |
+| Volume | >= 1.5 TB | Steady state at 1-year retention is ~1 TB (roughly 2.3 GB/day compressed across hypertables, plus balance tables), with the remainder as WAL/headroom. |
+
+**Runtime flags** (`ingest` command):
+
+| Flag / env var | Value | Why |
+|---|---|---|
+| `RETENTION_PERIOD` | `1 year` | Chunks older than this are dropped automatically. |
+| `COMPRESSION_COMPRESS_AFTER` | `1 day` | Leaves one uncompressed day of headroom so gap-backfill writes land in the rowstore instead of a compressed chunk. |
+| `COMPRESSION_SCHEDULE_INTERVAL` | `1 day` | How often the compression job checks for eligible chunks. |
+| `COMPRESSION_MAX_CHUNKS` | `10` | Caps chunks compressed per job run to prevent overlapping runs. |
+| `CHUNK_INTERVAL` | `1 day` | Hypertable chunk time interval; only affects newly created chunks. |
+
+**GraphQL:** `GRAPHQL_INTROSPECTION_ENABLED` should be unset (or `false`) in production. Introspection makes the full schema, including any unreleased or internal-only fields, discoverable to anyone who can reach the endpoint. Leave it `true` in dev environments.
+
+## Alerting & Failure Modes
+
+**Alert on:**
+- `wallet_ingestion_lag_ledgers` growth — ingestion falling behind the backend tip.
+- Container restart counts — the ingest process exits immediately after a permanent failure.
+- `wallet_db_query_errors_total{error_type="cursor_missing"}` — fires only when a protocol cursor that previously existed disappears, which is always a genuine incident (not a normal not-yet-initialized state).
+
+**Do not alert on** `wallet_ingestion_retry_exhaustions_total` directly — the process exits right after incrementing it, so by the time an alert fires the process has already restarted. Treat it as a post-mortem breadcrumb, not a live signal.
+
+**Poisoned ledgers:** a ledger that deterministically fails ingestion cannot self-heal by retrying — the ingest loop fails fast on permanent SQLSTATE classes, and recovery requires either a code fix or a manually-guarded cursor bump past the ledger.
+
+**TimescaleDB job health** has no application-level metric. Monitor `timescaledb_information.job_stats` and `timescaledb_information.job_errors` directly via SQL (through the CNPG metrics exporter or a scheduled check) to catch stalled or failing compression/retention jobs.
+
+**Hypertable index usage:** `pg_stat_user_indexes` on the `public` schema always reports `idx_scan = 0` for hypertable indexes — scans accrue on the per-chunk indexes in `_timescaledb_internal`, and the parent's index entry is only a template. To measure real usage (e.g. for an unused-index audit), aggregate chunk-index stats:
+
+```sql
+SELECT ht.table_name AS hypertable, regexp_replace(sui.indexrelname, '^_hyper_\d+_\d+_chunk_', '') AS index_name,
+       sum(sui.idx_scan) AS scans
+FROM pg_stat_user_indexes sui
+JOIN _timescaledb_catalog.chunk c ON c.schema_name = sui.schemaname AND c.table_name = sui.relname
+JOIN _timescaledb_catalog.hypertable ht ON ht.id = c.hypertable_id
+GROUP BY 1, 2 ORDER BY 1, 3 DESC;
+```
+
+Every index in the schema must have a nameable consumer query; secondary indexes duplicating a primary key's column set are not kept (the PK's column order is chosen to serve its consumers directly — B-tree scans run forward or backward, so ASC/DESC variants of the same key are one index, not two).
+
+## Connection Topology Constraint
+
+Ingestion connects directly to the CNPG `rw` service and relies on pgx's server-side prepared-statement caching for performance. If a transaction-pooling PgBouncer/Pooler is ever placed in front of the ingest path, its pool mode must be switched to `QueryExecModeExec` — transaction pooling is incompatible with server-side prepared statements. The API server already runs in `Exec` mode.