perf(tests): parallelize e2e test suite, add sync RPC latency metric

bchain/coins/eth/ethrpc.go

@@ -395,6 +395,17 @@ func (b *EthereumRPC) observeEthSyncRpcError(method string, err error) {
 	b.metrics.EthSyncRpcErrors.With(common.Labels{"method": method, "status": ethSyncRpcErrStatus(err)}).Inc()
 }
 
+func (b *EthereumRPC) observeSyncRPCLatency(method string, start time.Time, err error) {
+	if b.metrics == nil {
+		return
+	}
+	errorLabel := ""
+	if err != nil {
+		errorLabel = "failure"
+	}
+	b.metrics.RPCSyncLatency.With(common.Labels{"method": method, "error": errorLabel}).Observe(float64(time.Since(start)) / 1e6)
+}
+
 // EnsureSameRPCHost validates both RPC URLs and logs a warning if hosts differ.
 func EnsureSameRPCHost(httpURL, wsURL string) error {
 	if httpURL == "" || wsURL == "" {
@@ -960,12 +971,17 @@ func (b *EthereumRPC) GetChainInfo() (*bchain.ChainInfo, error) {
 	}
 	ctx, cancel := context.WithTimeout(context.Background(), b.Timeout)
 	defer cancel()
+	netStart := time.Now()
 	id, err := b.Client.NetworkID(ctx)
+	b.observeSyncRPCLatency("net_version", netStart, err)
 	if err != nil {
 		return nil, err
 	}
 	var ver string
-	if err := b.RPC.CallContext(ctx, &ver, "web3_clientVersion"); err != nil {
+	web3Start := time.Now()
+	err = b.RPC.CallContext(ctx, &ver, "web3_clientVersion")
+	b.observeSyncRPCLatency("web3_clientVersion", web3Start, err)
+	if err != nil {
 		return nil, err
 	}
 	rv := &bchain.ChainInfo{
@@ -996,7 +1012,9 @@ func (b *EthereumRPC) getBestHeader() (bchain.EVMHeader, error) {
 		var err error
 		ctx, cancel := context.WithTimeout(context.Background(), b.Timeout)
 		defer cancel()
+		headerStart := time.Now()
 		b.bestHeader, err = b.Client.HeaderByNumber(ctx, nil)
+		b.observeSyncRPCLatency("eth_getBlockByNumber", headerStart, err)
 		if err != nil {
 			b.bestHeader = nil
 			return nil, err
@@ -1363,6 +1381,7 @@ func (b *EthereumRPC) getBlockRaw(hash string, height uint32, fullTxs bool) (jso
 	var raw json.RawMessage
 	var err error
 	var method string
+	defer func(s time.Time) { b.observeSyncRPCLatency(method, s, err) }(time.Now())
 	if hash != "" {
 		if hash == "pending" {
 			method = "eth_getBlockByNumber"
@@ -1395,7 +1414,9 @@ func (b *EthereumRPC) processEventsForBlock(blockNumber string) (map[string][]*b
 	var logs []rpcLogWithTxHash
 	var ensRecords []bchain.AddressAliasRecord
 	var method = "eth_getLogs"
-	err := b.RPC.CallContext(ctx, &logs, method, map[string]interface{}{
+	var err error
+	defer func(s time.Time) { b.observeSyncRPCLatency(method, s, err) }(time.Now())
+	err = b.RPC.CallContext(ctx, &logs, method, map[string]interface{}{
 		"fromBlock": blockNumber,
 		"toBlock":   blockNumber,
 	})
@@ -1499,7 +1520,9 @@ func (b *EthereumRPC) getInternalDataForBlock(ctx context.Context, blockHash str
 		if b.ChainConfig.TraceTimeout != "" {
 			traceConfig["timeout"] = b.ChainConfig.TraceTimeout
 		}
+		traceStart := time.Now()
 		err := b.RPC.CallContext(ctx, &trace, "debug_traceBlockByHash", blockHash, traceConfig) // Use caller-provided ctx for timeout/cancel.
+		b.observeSyncRPCLatency("debug_traceBlockByHash", traceStart, err)
 		b.observeEthSyncRpcError("debug_traceBlockByHash", err)
 		if err != nil {
 			glog.Error("debug_traceBlockByHash block ", blockHash, ", error ", err)

common/metrics.go

@@ -102,6 +102,7 @@ type Metrics struct {
 	EthEip1559FeeSource               *prometheus.CounterVec   `metric:"eth_eip1559_fee_source_total"`
 	EthBlockGasUsedRatio              prometheus.Gauge         `metric:"eth_block_gas_used_ratio"`
 	EthBlockBaseFee                   prometheus.Gauge         `metric:"eth_block_base_fee"`
+	RPCSyncLatency                    *prometheus.HistogramVec `metric:"rpc_sync_latency"`
 	EthSyncRpcErrors                  *prometheus.CounterVec   `metric:"eth_sync_rpc_errors"`
 }
 

configs/grafana/panels.yaml

@@ -286,6 +286,16 @@ panels:
       events:
         promql: sum by (subscription, event) (rate({{name:backend_subscription_events}}{job="blockbook", coin="$coin"}[5m]))
         legend: '{{subscription}} - {{event}}'
+  sync.rpc_sync_latency:
+    title: Sync RPC latency (p50/p95)
+    description: Round-trip time of successful sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.) that are not tracked by the general RPC latency panel; these directly affect block indexing speed. Typically sub-100ms; a sustained multi-second p95 means the backend is overloaded. Failed calls are excluded so timeout expiries (~10-15s) do not skew the quantiles; they show up in the Ethereum sync RPC errors panel instead.
+    queries:
+      p95:
+        promql: histogram_quantile(0.95, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin", error=""}[1m])) by (coin, method, le))
+        legend: '{{coin}} - {{method}}'
+      p50:
+        promql: histogram_quantile(0.50, sum(rate({{name:rpc_sync_latency}}_bucket{job="blockbook", coin="$coin", error=""}[1m])) by (coin, method, le))
+        legend: '{{coin}} - {{method}}'
   row.websocket:
     title: Websocket
   websocket.clients:

configs/grafana/template.json

@@ -2436,6 +2436,72 @@
       "type": "timeseries",
       "x-panel-key": "sync.tip_feed_events"
     },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "drawStyle": "line",
+            "fillOpacity": 10,
+            "gradientMode": "none",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "showPoints": "never",
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            }
+          },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": 0
+              }
+            ]
+          },
+          "unit": "ms"
+        },
+        "overrides": []
+      },
+      "id": 348,
+      "options": {
+        "legend": {
+          "calcs": [
+            "lastNotNull",
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "12.3.0",
+      "targets": [
+        {
+          "editorMode": "code",
+          "range": true,
+          "refId": "A",
+          "x-query-key": "p95"
+        },
+        {
+          "editorMode": "code",
+          "range": true,
+          "refId": "B",
+          "x-query-key": "p50"
+        }
+      ],
+      "type": "timeseries",
+      "x-panel-key": "sync.rpc_sync_latency"
+    },
     {
       "collapsed": false,
       "id": 315,

configs/metrics.yaml

@@ -400,6 +400,12 @@ metrics:
     name: blockbook_eth_block_base_fee
     type: gauge
     help: Base fee per gas of the most recently connected EVM block, in wei - divide by 1e9 for Gwei. The realized base fee from the block header (push path); compare with eth_eip1559_base_fee (the pull path's next-block projection) to see whether the projection tracks the chain. Updated once per connected block
+  rpc_sync_latency:
+    name: blockbook_rpc_sync_latency
+    type: histogram_vec
+    help: Blockbook-to-backend round-trip time for sync-internal RPC calls (eth_getBlockByHash, eth_getLogs, debug_traceBlockByHash, etc.), in milliseconds, labeled by method and whether the call failed. Typically sub-100ms; multi-second values mean an overloaded backend, and observations near 10-15s are trace_timeout/rpc_timeout expiries rather than real round trips
+    labels: [method, error]
+    buckets: [1, 5, 10, 25, 50, 100, 250, 500, 1000, 2000, 5000, 10000, 20000]
   eth_sync_rpc_errors:
     name: blockbook_eth_sync_rpc_errors
     type: counter_vec

tests/openapi/src/client.ts

@@ -61,7 +61,7 @@ export class OpenApiFetchClient {
       try {
         const response = await fetch(url, {
           method: "GET",
-          signal: AbortSignal.timeout(30_000),
+          signal: AbortSignal.timeout(15_000),
         });
         const body = await response.text();
         if (attempt < 2 && isRetryableHTTPStatus(response.status)) {

tests/openapi/src/constants.ts

@@ -1,5 +1,5 @@
-export const wsDialTimeoutMs = 5_000;
-export const wsMessageTimeoutMs = 15_000;
+export const wsDialTimeoutMs = 3_000;
+export const wsMessageTimeoutMs = 10_000;
 export const txSearchWindow = 12;
 export const blockPageSize = 1;
 export const sampleBlockPageSize = 3;