diff --git a/cluster/expected/observability/expected.json b/cluster/expected/observability/expected.json index 4e1fd4ab67..6a81cb1cab 100644 --- a/cluster/expected/observability/expected.json +++ b/cluster/expected/observability/expected.json @@ -354,7 +354,7 @@ "data": { "acs-size.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Number of rows in the latest saved ACS snapshot, as reported by the SV Scan app. This is the metric behind the 'ACS growth' alert. Note that it counts rows, not contracts: contracts with multiple stakeholders are counted multiple times. A value of -1 means the size of the last snapshot save is unknown.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"short\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 10,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_acs_snapshots_snapshot_size{namespace=~\\\"$namespace\\\"}\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"ACS size (rows in latest saved snapshot)\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"none\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 20,\n \"w\": 24,\n \"x\": 0,\n \"y\": 10\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"(increase(splice_store_acs_size_increase_total{namespace=~\\\"$namespace\\\",store_name=~\\\"$store_name\\\"}[24h]))-(increase(splice_store_acs_size_decrease_total{namespace=~\\\"$namespace\\\",store_name=~\\\"$store_name\\\"}[24h]))\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{store_name}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Splice Store ACS Change in last 24h\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"refresh\": \"\",\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": \".*\",\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_store_acs_size_increase_total,namespace)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_store_acs_size_increase_total,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"allValue\": \".*\",\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_store_acs_size_increase_total,store_name)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"store_name\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_store_acs_size_increase_total,store_name)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-1h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"ACS Size\",\n \"uid\": \"dduss3xr5or28c\",\n \"version\": 2,\n \"weekStart\": \"\"\n}\n", "acs-snapshots.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 5,\n \"panels\": [],\n \"title\": \"Incremental ACS snapshot generation\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"dateTimeAsIso\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 1\n },\n \"id\": 3,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_acs_snapshots_latest_record_time_update{namespace=~\\\"$namespace\\\"} / 1000\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Record time of last update\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"dateTimeAsIso\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 1\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_acs_snapshots_latest_record_time_save{namespace=~\\\"$namespace\\\"} / 1000\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Record time of last save\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Time to update an incremental snapshot, i.e., the time to update an incremental snapshot by processing ~30sec worth of update history.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 9\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"builder\",\n \"expr\": \"histogram_avg(rate(splice_history_acs_snapshots_latency_update_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Update latency\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Time to save an incremental snapshot, i.e., the time to copy an incremental snapshot to the the big table containing all historical snapshots.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 9\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"builder\",\n \"expr\": \"histogram_avg(rate(splice_history_acs_snapshots_latency_save_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Save latency\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_history_acs_snapshots_latest_record_time_update,namespace)\",\n \"includeAll\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_acs_snapshots_latest_record_time_update,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"ACS snapshots\",\n \"uid\": \"cnqnrp4\",\n \"version\": 3,\n \"weekStart\": \"\"\n}\n", - "bulk-storage.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"dateTimeAsIso\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"legendFormat\": \"{{namespace}} ACS Snapshots\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_updates_segment{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} Updates\",\n \"range\": true,\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Last record time stored\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_bulk_storage_object_count{namespace=~\\\"$namespace\\\"}\",\n \"legendFormat\": \"{{namespace}} {{object_type}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Num S3 objects stored\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": [\n \"All\"\n ],\n \"value\": [\n \"$__all\"\n ]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_history_bulk_storage_updates_count,namespace)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_bulk_storage_updates_count,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-2d\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Bulk Storage\",\n \"uid\": \"cnxl9bn\",\n \"version\": 3,\n \"weekStart\": \"\"\n}\n", + "bulk-storage.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"dateTimeAsIso\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot_staging{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"legendFormat\": \"{{namespace}} ACS Snapshots - Staging\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_updates_segment_staging{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} Updates - Staging\",\n \"range\": true,\n \"refId\": \"B\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot_committed{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"legendFormat\": \"{{namespace}} ACS Snapshots - Committed\",\n \"range\": true,\n \"refId\": \"C\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_bulk_storage_latest_updates_segment_committed{namespace=~\\\"$namespace\\\"} >-62135596800000) / 1000.0)\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} Updates - Committed\",\n \"range\": true,\n \"refId\": \"D\"\n }\n ],\n \"title\": \"Last record time stored\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Rate (compared to wallclock time) at which the latest persisted record time is advancing\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 0\n },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"avg_over_time((deriv(splice_history_bulk_storage_latest_acs_snapshot_staging{namespace=~\\\"$namespace\\\"}[$__rate_interval]) / 1000000)[1h:])\",\n \"legendFormat\": \"{{namespace}} ACS Snapshots - Staging\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"avg_over_time((deriv(splice_history_bulk_storage_latest_updates_segment_staging{namespace=~\\\"$namespace\\\"}[$__rate_interval]) / 1000000)[1h:])\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} Updates - Staging\",\n \"range\": true,\n \"refId\": \"B\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"avg_over_time((deriv(splice_history_bulk_storage_latest_acs_snapshot_committed{namespace=~\\\"$namespace\\\"}[$__rate_interval]) / 1000000)[1h:])\",\n \"legendFormat\": \"{{namespace}} ACS Snapshots - Committed\",\n \"range\": true,\n \"refId\": \"C\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"avg_over_time((deriv(splice_history_bulk_storage_latest_updates_segment_committed{namespace=~\\\"$namespace\\\"}[$__rate_interval]) / 1000000)[1h:])\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} Updates - Committed\",\n \"range\": true,\n \"refId\": \"D\"\n }\n ],\n \"title\": \"Catchup rate\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Counts items from Scan DB processed into the staging bucket\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 8\n },\n \"id\": 3,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_bulk_storage_updates_count{namespace=~\\\"$namespace\\\"}\",\n \"legendFormat\": \"Updates {{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_bulk_storage_contracts_count{namespace=~\\\"$namespace\\\"}\",\n \"instant\": false,\n \"legendFormat\": \"ACS contracts {{namespace}}\",\n \"range\": true,\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Processed items\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Counts items from Scan DB processed into the staging bucket\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 8\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_history_bulk_storage_updates_count{namespace=~\\\"$namespace\\\"}[$__rate_interval])\",\n \"legendFormat\": \"Updates {{namespace}}\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"rate(splice_history_bulk_storage_contracts_count{namespace=~\\\"$namespace\\\"}[$__rate_interval])\",\n \"instant\": false,\n \"legendFormat\": \"ACS contracts {{namespace}}\",\n \"range\": true,\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Rate of processing items\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 16\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"splice_history_bulk_storage_object_count{namespace=~\\\"$namespace\\\"}\",\n \"legendFormat\": \"{{namespace}} {{object_type}} {{encoding}} - {{bucket}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Num S3 objects stored\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 16\n },\n \"id\": 6,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"editorMode\": \"code\",\n \"expr\": \"avg_over_time(rate(splice_history_bulk_storage_object_count{namespace=~\\\"$namespace\\\"}[$__rate_interval])[1h:])\",\n \"legendFormat\": \"{{namespace}} {{object_type}} {{encoding}} - {{bucket}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"S3 objects storage rate\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": [\n \"All\"\n ],\n \"value\": [\n \"$__all\"\n ]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_history_bulk_storage_updates_count,namespace)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_bulk_storage_updates_count,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-12h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Bulk Storage\",\n \"uid\": \"cnxl9bn\",\n \"version\": 13,\n \"weekStart\": \"\"\n}\n", "history-backfilling.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"id\": 1387,\n \"links\": [],\n \"panels\": [\n {\n \"fieldConfig\": {\n \"defaults\": {},\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 7,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 21,\n \"options\": {\n \"code\": {\n \"language\": \"plaintext\",\n \"showLineNumbers\": false,\n \"showMiniMap\": false\n },\n \"content\": \"There are three backfilling background tasks:\\n\\n1. `ScanHistoryBackfillingTrigger` backfills regular updates\\n - Data is loaded from peer scan apps using BFT network calls\\n1. `ScanHistoryBackfillingTrigger` backfills import updates\\n - Data is loaded from peer scan apps using BFT network calls\\n1. `TxLogBackfillingTrigger` backfills txlog entries\\n - Data is loaded from the local UpdateHistory\\n\\nThe tasks are completed sequentially in the above order.\\nE.g., txlog backfilling won't start until update history backfilling is complete.\",\n \"mode\": \"markdown\"\n },\n \"pluginVersion\": \"12.1.1\",\n \"title\": \"Overview\",\n \"type\": \"text\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 7\n },\n \"id\": 7,\n \"panels\": [],\n \"title\": \"Updates (scan)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 8\n },\n \"id\": 17,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_backfilling_transaction_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"})\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Backfilled transactions count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 8\n },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by (namespace)(rate(splice_history_backfilling_transaction_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"}[5m]))\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rate backfilled transactions\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 16\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_backfilling_event_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"})\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Backfilled events count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 16\n },\n \"id\": 6,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(rate(splice_history_backfilling_event_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"}[5m]))\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rate backfilled events\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"custom\": {\n \"axisPlacement\": \"auto\",\n \"fillOpacity\": 70,\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineWidth\": 0,\n \"spanNulls\": false\n },\n \"mappings\": [],\n \"max\": 1,\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"red\",\n \"value\": 0\n },\n {\n \"color\": \"green\",\n \"value\": 1\n }\n ]\n },\n \"unit\": \"bool_yes_no\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 24\n },\n \"id\": 15,\n \"options\": {\n \"alignValue\": \"left\",\n \"legend\": {\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"mergeValues\": true,\n \"rowHeight\": 0.9,\n \"showValue\": \"auto\",\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_backfilling_completed{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"})\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Backfilling completed\",\n \"type\": \"state-timeline\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Value\"\n },\n \"properties\": [\n {\n \"id\": \"unit\",\n \"value\": \"dateTimeAsIso\"\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 24\n },\n \"id\": 3,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_backfilling_latest_record_time{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"}>-62135596800000)/1000)\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"Value\"\n }\n ],\n \"title\": \"Last record time backfilled\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 32\n },\n \"id\": 16,\n \"panels\": [],\n \"title\": \"Import Updates (scan)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 33\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_import_updates_backfilling_contract_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"})\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Backfilled import contracts count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 33\n },\n \"id\": 20,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by (namespace)(rate(splice_history_import_updates_backfilling_contract_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"}[5m]))\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rate backfilled contracts\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"custom\": {\n \"axisPlacement\": \"auto\",\n \"fillOpacity\": 70,\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineWidth\": 0,\n \"spanNulls\": false\n },\n \"mappings\": [],\n \"max\": 1,\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"red\",\n \"value\": 0\n },\n {\n \"color\": \"green\",\n \"value\": 1\n }\n ]\n },\n \"unit\": \"bool_yes_no\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 41\n },\n \"id\": 19,\n \"options\": {\n \"alignValue\": \"left\",\n \"legend\": {\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"mergeValues\": true,\n \"rowHeight\": 0.9,\n \"showValue\": \"auto\",\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_import_updates_backfilling_completed{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"})\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Import update backfilling completed\",\n \"type\": \"state-timeline\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Value\"\n },\n \"properties\": [\n {\n \"id\": \"unit\",\n \"value\": \"\"\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 41\n },\n \"id\": 18,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_import_updates_backfilling_latest_migration_id{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\"}>-1))\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"legendFormat\": \"__auto\",\n \"range\": true,\n \"refId\": \"Value\"\n }\n ],\n \"title\": \"Last migration id backfilled\",\n \"type\": \"timeseries\"\n },\n {\n \"collapsed\": false,\n \"gridPos\": {\n \"h\": 1,\n \"w\": 24,\n \"x\": 0,\n \"y\": 49\n },\n \"id\": 8,\n \"panels\": [],\n \"title\": \"TxLog entries (all apps)\",\n \"type\": \"row\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 50\n },\n \"id\": 9,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_txlog_backfilling_transaction_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"})\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{service}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Processed transactions count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 50\n },\n \"id\": 10,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by (namespace)(rate(splice_history_txlog_backfilling_transaction_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"}[5m]))\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{service}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rate processed transactions\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 58\n },\n \"id\": 11,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(splice_history_txlog_backfilling_event_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"})\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{service}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Processed events count\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 58\n },\n \"id\": 12,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace)(rate(splice_history_txlog_backfilling_event_count{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"}[5m]))\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}}{{service}}\",\n \"range\": true,\n \"refId\": \"A\"\n }\n ],\n \"title\": \"Rate processed events\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"thresholds\"\n },\n \"custom\": {\n \"axisPlacement\": \"auto\",\n \"fillOpacity\": 70,\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineWidth\": 0,\n \"spanNulls\": false\n },\n \"mappings\": [],\n \"max\": 1,\n \"min\": 0,\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"red\",\n \"value\": 0\n },\n {\n \"color\": \"green\",\n \"value\": 1\n }\n ]\n },\n \"unit\": \"bool_yes_no\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 11,\n \"w\": 12,\n \"x\": 0,\n \"y\": 66\n },\n \"id\": 13,\n \"options\": {\n \"alignValue\": \"left\",\n \"legend\": {\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"mergeValues\": true,\n \"rowHeight\": 0.9,\n \"showValue\": \"auto\",\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"max by(namespace, service) (splice_history_txlog_backfilling_completed{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"})\",\n \"format\": \"time_series\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"{{namespace}} {{service}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"TxLog backfilling completed\",\n \"type\": \"state-timeline\"\n },\n {\n \"datasource\": {\n \"default\": true,\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": [\n {\n \"matcher\": {\n \"id\": \"byName\",\n \"options\": \"Value\"\n },\n \"properties\": [\n {\n \"id\": \"unit\",\n \"value\": \"dateTimeAsIso\"\n }\n ]\n }\n ]\n },\n \"gridPos\": {\n \"h\": 11,\n \"w\": 12,\n \"x\": 12,\n \"y\": 66\n },\n \"id\": 14,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.1.1\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"min by(namespace)((splice_history_txlog_backfilling_latest_record_time{namespace=~\\\"$namespace\\\", migration=~\\\"$migration\\\", service=~\\\"$service\\\"}>-62135596800000)/1000)\",\n \"format\": \"time_series\",\n \"instant\": false,\n \"legendFormat\": \"{{namespace}} {{service}}\",\n \"range\": true,\n \"refId\": \"Value\"\n }\n ],\n \"title\": \"Last record time backfilled\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"refresh\": \"\",\n \"schemaVersion\": 41,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": {\n \"text\": [\n \"All\"\n ],\n \"value\": [\n \"$__all\"\n ]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_history_txlog_backfilling_latest_record_time,namespace)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_txlog_backfilling_latest_record_time,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": [\n \"All\"\n ],\n \"value\": [\n \"$__all\"\n ]\n },\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"definition\": \"label_values(splice_history_txlog_backfilling_latest_record_time,migration)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"migration\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_txlog_backfilling_latest_record_time,migration)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"type\": \"query\"\n },\n {\n \"current\": {\n \"text\": \"All\",\n \"value\": \"$__all\"\n },\n \"definition\": \"label_values(splice_history_txlog_backfilling_completed,service)\",\n \"includeAll\": true,\n \"name\": \"service\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_history_txlog_backfilling_completed,service)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-6h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"History backfiling\",\n \"uid\": \"fe12ejqgwdlvke\",\n \"version\": 4\n}\n", "mediator-verdicts-ingestion.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": {\n \"type\": \"grafana\",\n \"uid\": \"-- Grafana --\"\n },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 0,\n \"links\": [],\n \"panels\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Rate of ingestion of mediator verdicts per second based on ingestion_count_total\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 9,\n \"w\": 24,\n \"x\": 0,\n \"y\": 0\n },\n \"id\": 1,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": true,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"exemplar\": false,\n \"expr\": \"rate(splice_scan_verdict_ingestion_count_total{namespace=~\\\"$namespace\\\", node_name=~\\\"$node_name\\\", pod=~\\\"$pod_name\\\"}[$__rate_interval])\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": false,\n \"legendFormat\": \"{{namespace}} {{job}} {{pod}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Verdicts Ingestion Rate\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"The time difference between the last ingested verdict record time and the current wall clock time.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 0,\n \"y\": 9\n },\n \"id\": 2,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"(timestamp(splice_scan_verdict_ingestion_last_record_time_us{namespace=~\\\"$namespace\\\",node_name=~\\\"$node_name\\\", pod_name=~\\\"$pod_name\\\"}) - (splice_scan_verdict_ingestion_last_record_time_us{namespace=~\\\"$namespace\\\",node_name=~\\\"$node_name\\\", pod_name=~\\\"$pod_name\\\"} > 0) / 1e6)\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"{{namespace}} {{job}} {{pod}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Last Ingested Verdict Time Lag\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"dateTimeAsIso\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 12,\n \"x\": 12,\n \"y\": 9\n },\n \"id\": 3,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"single\",\n \"sort\": \"none\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"disableTextWrap\": false,\n \"editorMode\": \"code\",\n \"expr\": \"(splice_scan_verdict_ingestion_last_record_time_us{namespace=~\\\"$namespace\\\",node_name=~\\\"$node_name\\\", pod_name=~\\\"$pod_name\\\"} > 0) / 1e3\",\n \"fullMetaSearch\": false,\n \"includeNullMetadata\": true,\n \"legendFormat\": \"{{namespace}} {{job}} {{pod}}\",\n \"range\": true,\n \"refId\": \"A\",\n \"useBackend\": false\n }\n ],\n \"title\": \"Last Ingested Verdict Record Time\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Average and 99th quantile batch size.\\n\\nVerdicts are streamed in from the mediator, then batched in the scan app. For each batch we do one lookup for traffic summaries in the sequencer, and submit one SQL transaction to store all app activity data.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n }\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 17\n },\n \"id\": 5,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"histogram_avg(rate(splice_scan_verdict_ingestion_batch_size{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}} avg\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"histogram_quantile(0.99, rate(splice_scan_verdict_ingestion_batch_size{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}} 99th\",\n \"range\": true,\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Batch size\",\n \"type\": \"timeseries\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"description\": \"Average and 99th percentile time to ingest one batch.\\n\\nIncludes data processing and waiting for the SQL transaction to complete.\",\n \"fieldConfig\": {\n \"defaults\": {\n \"color\": {\n \"mode\": \"palette-classic\"\n },\n \"custom\": {\n \"axisBorderShow\": false,\n \"axisCenteredZero\": false,\n \"axisColorMode\": \"text\",\n \"axisLabel\": \"\",\n \"axisPlacement\": \"auto\",\n \"barAlignment\": 0,\n \"barWidthFactor\": 0.6,\n \"drawStyle\": \"line\",\n \"fillOpacity\": 0,\n \"gradientMode\": \"none\",\n \"hideFrom\": {\n \"legend\": false,\n \"tooltip\": false,\n \"viz\": false\n },\n \"insertNulls\": false,\n \"lineInterpolation\": \"linear\",\n \"lineWidth\": 1,\n \"pointSize\": 5,\n \"scaleDistribution\": {\n \"type\": \"linear\"\n },\n \"showPoints\": \"auto\",\n \"showValues\": false,\n \"spanNulls\": false,\n \"stacking\": {\n \"group\": \"A\",\n \"mode\": \"none\"\n },\n \"thresholdsStyle\": {\n \"mode\": \"off\"\n }\n },\n \"mappings\": [],\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n {\n \"color\": \"green\",\n \"value\": 0\n },\n {\n \"color\": \"red\",\n \"value\": 80\n }\n ]\n },\n \"unit\": \"s\"\n },\n \"overrides\": []\n },\n \"gridPos\": {\n \"h\": 8,\n \"w\": 24,\n \"x\": 0,\n \"y\": 25\n },\n \"id\": 4,\n \"options\": {\n \"legend\": {\n \"calcs\": [],\n \"displayMode\": \"list\",\n \"placement\": \"bottom\",\n \"showLegend\": true\n },\n \"tooltip\": {\n \"hideZeros\": false,\n \"mode\": \"multi\",\n \"sort\": \"desc\"\n }\n },\n \"pluginVersion\": \"12.4.0\",\n \"targets\": [\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"histogram_avg(rate(splice_scan_verdict_ingestion_latency_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}} avg\",\n \"range\": true,\n \"refId\": \"A\"\n },\n {\n \"datasource\": {\n \"type\": \"prometheus\",\n \"uid\": \"prometheus\"\n },\n \"editorMode\": \"code\",\n \"expr\": \"histogram_quantile(0.99, rate(splice_scan_verdict_ingestion_latency_duration_seconds{namespace=~\\\"$namespace\\\"}[$__rate_interval]))\",\n \"legendFormat\": \"{{namespace}} 99th\",\n \"range\": true,\n \"refId\": \"B\"\n }\n ],\n \"title\": \"Batch latency\",\n \"type\": \"timeseries\"\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"allValue\": \".*\",\n \"current\": {\n \"text\": [\n \"sv-1\"\n ],\n \"value\": [\n \"sv-1\"\n ]\n },\n \"definition\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,namespace)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"namespace\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,namespace)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"allValue\": \".*\",\n \"current\": {\n \"text\": \"All\",\n \"value\": [\n \"$__all\"\n ]\n },\n \"definition\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,node_name)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"node_name\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,node_name)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n },\n {\n \"allValue\": \".*\",\n \"current\": {\n \"text\": \"All\",\n \"value\": [\n \"$__all\"\n ]\n },\n \"definition\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,pod)\",\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"pod_name\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_scan_verdict_ingestion_last_record_time_us,pod)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"regexApplyTo\": \"value\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": {\n \"from\": \"now-12h\",\n \"to\": \"now\"\n },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Mediator Verdicts Ingestion\",\n \"uid\": \"86f70b02-f283-469e-bf90-033dc9f55888\",\n \"version\": 1,\n \"weekStart\": \"\"\n}\n", "store-ingestion-performance.json": "{\n \"annotations\": {\n \"list\": [\n {\n \"builtIn\": 1,\n \"datasource\": { \"type\": \"grafana\", \"uid\": \"-- Grafana --\" },\n \"enable\": true,\n \"hide\": true,\n \"iconColor\": \"rgba(0, 211, 255, 1)\",\n \"name\": \"Annotations & Alerts\",\n \"type\": \"dashboard\"\n }\n ]\n },\n \"editable\": true,\n \"fiscalYearStartMonth\": 0,\n \"graphTooltip\": 2,\n \"id\": null,\n \"links\": [],\n \"panels\": [\n {\n \"collapsed\": false,\n \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 0 },\n \"id\": 1,\n \"repeat\": \"test\",\n \"repeatDirection\": \"h\",\n \"title\": \"$test\",\n \"type\": \"row\"\n },\n {\n \"title\": \"Total Items\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 1 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"short\",\n \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"green\" },\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_total_items{test=\\\"${test}\\\"}\",\n \"legendFormat\": \"Total Items\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n },\n {\n \"title\": \"Total Batches\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 1 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"short\",\n \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"orange\" },\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_total_batches{test=\\\"${test}\\\"}\",\n \"legendFormat\": \"Total Batches\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n },\n {\n \"title\": \"Total Time\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 9 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"s\",\n \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"purple\" },\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_total_time_ns{test=\\\"${test}\\\"} / 1e9\",\n \"legendFormat\": \"Total Time (s)\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n },\n {\n \"title\": \"Avg Item Time\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 9 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"ms\",\n \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"blue\" },\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_avg_item_time_ns{test=\\\"${test}\\\"} / 1e6\",\n \"legendFormat\": \"Avg Item Time (ms)\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n },\n {\n \"title\": \"CPU-to-Wall-Clock Ratio\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 17 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"none\",\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n },\n \"thresholds\": {\n \"mode\": \"absolute\",\n \"steps\": [\n { \"color\": \"blue\", \"value\": null },\n { \"color\": \"green\", \"value\": 0.5 },\n { \"color\": \"orange\", \"value\": 1.0 },\n { \"color\": \"red\", \"value\": 2.0 }\n ]\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_cpu_to_wall_clock_ratio{test=\\\"${test}\\\"}\",\n \"legendFormat\": \"CPU/Wall-clock Ratio (>0.7: CPU-bound, 0.7~0.25: balanced, <0.25:I/O-bound)\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n },\n {\n \"title\": \"Peak Heap Memory\",\n \"type\": \"timeseries\",\n \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 17 },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"fieldConfig\": {\n \"defaults\": {\n \"unit\": \"decbytes\",\n \"color\": { \"mode\": \"fixed\", \"fixedColor\": \"semi-dark-purple\" },\n \"custom\": {\n \"drawStyle\": \"line\",\n \"lineWidth\": 2,\n \"pointSize\": 6,\n \"showPoints\": \"always\",\n \"fillOpacity\": 5,\n \"spanNulls\": false\n }\n },\n \"overrides\": []\n },\n \"options\": {\n \"legend\": { \"calcs\": [\"mean\", \"max\", \"lastNotNull\"], \"displayMode\": \"table\", \"placement\": \"bottom\", \"showLegend\": true },\n \"tooltip\": { \"mode\": \"single\", \"sort\": \"none\" }\n },\n \"targets\": [\n {\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"expr\": \"splice_perf_ingestion_peak_heap_bytes{test=\\\"${test}\\\"}\",\n \"legendFormat\": \"Peak Heap\",\n \"range\": true, \"refId\": \"A\"\n }\n ]\n }\n ],\n \"preload\": false,\n \"schemaVersion\": 42,\n \"tags\": [],\n \"templating\": {\n \"list\": [\n {\n \"current\": { \"text\": \"All\", \"value\": \"$__all\" },\n \"datasource\": { \"type\": \"prometheus\", \"uid\": \"prometheus\" },\n \"definition\": \"label_values(splice_perf_ingestion_total_items, test)\",\n \"hide\": 2,\n \"includeAll\": true,\n \"multi\": true,\n \"name\": \"test\",\n \"options\": [],\n \"query\": {\n \"qryType\": 1,\n \"query\": \"label_values(splice_perf_ingestion_total_items, test)\",\n \"refId\": \"PrometheusVariableQueryEditor-VariableQuery\"\n },\n \"refresh\": 1,\n \"regex\": \"\",\n \"type\": \"query\"\n }\n ]\n },\n \"time\": { \"from\": \"now-30d\", \"to\": \"now\" },\n \"timepicker\": {},\n \"timezone\": \"\",\n \"title\": \"Store Ingestion Performance\",\n \"uid\": \"splice-perf-ingestion\",\n \"version\": 1,\n \"weekStart\": \"\"\n}\n", diff --git a/cluster/pulumi/observability/grafana-dashboards/splice-stores/bulk-storage.json b/cluster/pulumi/observability/grafana-dashboards/splice-stores/bulk-storage.json index eacc594de0..012f483902 100644 --- a/cluster/pulumi/observability/grafana-dashboards/splice-stores/bulk-storage.json +++ b/cluster/pulumi/observability/grafana-dashboards/splice-stores/bulk-storage.json @@ -110,8 +110,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", - "legendFormat": "{{namespace}} ACS Snapshots", + "expr": "min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot_staging{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", + "legendFormat": "{{namespace}} ACS Snapshots - Staging", "range": true, "refId": "A" }, @@ -121,11 +121,34 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "min by(namespace)((splice_history_bulk_storage_latest_updates_segment{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", + "expr": "min by(namespace)((splice_history_bulk_storage_latest_updates_segment_staging{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", "instant": false, - "legendFormat": "{{namespace}} Updates", + "legendFormat": "{{namespace}} Updates - Staging", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min by(namespace)((splice_history_bulk_storage_latest_acs_snapshot_committed{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", + "legendFormat": "{{namespace}} ACS Snapshots - Committed", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "min by(namespace)((splice_history_bulk_storage_latest_updates_segment_committed{namespace=~\"$namespace\"} >-62135596800000) / 1000.0)", + "instant": false, + "legendFormat": "{{namespace}} Updates - Committed", + "range": true, + "refId": "D" } ], "title": "Last record time stored", @@ -136,6 +159,7 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Rate (compared to wallclock time) at which the latest persisted record time is advancing", "fieldConfig": { "defaults": { "color": { @@ -198,6 +222,353 @@ "x": 12, "y": 0 }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg_over_time((deriv(splice_history_bulk_storage_latest_acs_snapshot_staging{namespace=~\"$namespace\"}[$__rate_interval]) / 1000000)[1h:])", + "legendFormat": "{{namespace}} ACS Snapshots - Staging", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg_over_time((deriv(splice_history_bulk_storage_latest_updates_segment_staging{namespace=~\"$namespace\"}[$__rate_interval]) / 1000000)[1h:])", + "instant": false, + "legendFormat": "{{namespace}} Updates - Staging", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg_over_time((deriv(splice_history_bulk_storage_latest_acs_snapshot_committed{namespace=~\"$namespace\"}[$__rate_interval]) / 1000000)[1h:])", + "legendFormat": "{{namespace}} ACS Snapshots - Committed", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "avg_over_time((deriv(splice_history_bulk_storage_latest_updates_segment_committed{namespace=~\"$namespace\"}[$__rate_interval]) / 1000000)[1h:])", + "instant": false, + "legendFormat": "{{namespace}} Updates - Committed", + "range": true, + "refId": "D" + } + ], + "title": "Catchup rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Counts items from Scan DB processed into the staging bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "splice_history_bulk_storage_updates_count{namespace=~\"$namespace\"}", + "legendFormat": "Updates {{namespace}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "splice_history_bulk_storage_contracts_count{namespace=~\"$namespace\"}", + "instant": false, + "legendFormat": "ACS contracts {{namespace}}", + "range": true, + "refId": "B" + } + ], + "title": "Processed items", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Counts items from Scan DB processed into the staging bucket", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "rate(splice_history_bulk_storage_updates_count{namespace=~\"$namespace\"}[$__rate_interval])", + "legendFormat": "Updates {{namespace}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "rate(splice_history_bulk_storage_contracts_count{namespace=~\"$namespace\"}[$__rate_interval])", + "instant": false, + "legendFormat": "ACS contracts {{namespace}}", + "range": true, + "refId": "B" + } + ], + "title": "Rate of processing items", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, "id": 2, "options": { "legend": { @@ -217,13 +588,107 @@ { "editorMode": "code", "expr": "splice_history_bulk_storage_object_count{namespace=~\"$namespace\"}", - "legendFormat": "{{namespace}} {{object_type}}", + "legendFormat": "{{namespace}} {{object_type}} {{encoding}} - {{bucket}}", "range": true, "refId": "A" } ], "title": "Num S3 objects stored", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.4.0", + "targets": [ + { + "editorMode": "code", + "expr": "avg_over_time(rate(splice_history_bulk_storage_object_count{namespace=~\"$namespace\"}[$__rate_interval])[1h:])", + "legendFormat": "{{namespace}} {{object_type}} {{encoding}} - {{bucket}}", + "range": true, + "refId": "A" + } + ], + "title": "S3 objects storage rate", + "type": "timeseries" } ], "preload": false, @@ -262,13 +727,13 @@ ] }, "time": { - "from": "now-2d", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Bulk Storage", "uid": "cnxl9bn", - "version": 3, + "version": 13, "weekStart": "" }