diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json index 149dfacd857..169854a22cf 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-build.json @@ -316,7 +316,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", + "expr": "{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", "queryType": "range", "refId": "B" } @@ -414,7 +414,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", + "expr": "sum(count_over_time({cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", "legendFormat": "Error rate", "queryType": "range", "refId": "A" @@ -459,7 +459,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", + "expr": "{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", "queryType": "range", "refId": "B" } @@ -557,7 +557,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", + "expr": "sum(count_over_time({cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", "legendFormat": "Error rate", "queryType": "range", "refId": "A" @@ -759,7 +759,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Planned", @@ -772,7 +772,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Completed - {{status}}", @@ -785,7 +785,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "loki_bloomplanner_inflight_tasks{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", + "expr": "loki_bloomplanner_inflight_tasks{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "inflight p95", @@ -798,7 +798,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum(loki_bloomplanner_queue_length{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Queued", @@ -1008,7 +1008,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_connected_builders{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum(loki_bloomplanner_connected_builders{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "hide": false, "instant": false, "legendFormat": "Connected builders", @@ -1021,7 +1021,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", + "expr": "count(loki_bloombuilder_processing_task{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", "hide": false, "instant": false, "legendFormat": "Builders processing task", @@ -1034,7 +1034,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", + "expr": "count(loki_bloombuilder_processing_task{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", "hide": false, "instant": false, "legendFormat": "IDLE Builders", @@ -1132,7 +1132,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1145,7 +1145,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p90", @@ -1158,7 +1158,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p75", @@ -1171,7 +1171,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1268,7 +1268,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1281,7 +1281,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p90", @@ -1294,7 +1294,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1516,7 +1516,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Planned", @@ -1529,7 +1529,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Completed - {{status}}", @@ -1542,7 +1542,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", + "expr": "sum(loki_bloomplanner_queue_length{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Queued", @@ -1703,7 +1703,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}))", + "expr": "sum(rate(loki_bloom_tokens_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}))", "hide": false, "instant": false, "legendFormat": "Per core", @@ -1716,7 +1716,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -1818,7 +1818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", + "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1921,7 +1921,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "max", @@ -1934,7 +1934,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1947,7 +1947,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -2049,7 +2049,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", + "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -2161,7 +2161,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", + "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", "hide": false, "instant": false, "legendFormat": "Needed", @@ -2174,7 +2174,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Available", @@ -2973,7 +2973,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", "instant": false, "legendFormat": "Restarts", "range": true, @@ -3077,7 +3077,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", "instant": false, "legendFormat": "{{reason}} / {{pod}}", "range": true, @@ -3746,7 +3746,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", "instant": false, "legendFormat": "Restarts", "range": true, @@ -3850,7 +3850,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", "instant": false, "legendFormat": "{{reason}} / {{pod}}", "range": true, @@ -3960,7 +3960,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Metas", @@ -4028,7 +4028,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", "format": "table", "hide": false, "instant": true, @@ -4126,7 +4126,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Deleted during {{phase}}", @@ -4237,7 +4237,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Blocks", @@ -4305,7 +4305,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", "format": "time_series", "hide": false, "instant": true, @@ -4403,7 +4403,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Deleted during {{phase}}", @@ -4500,7 +4500,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Blocks", @@ -4603,7 +4603,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -4703,7 +4703,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "legendFormat": "__auto", "range": true, "refId": "A" @@ -4799,7 +4799,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true, "refId": "A" @@ -4968,7 +4968,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5066,7 +5066,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5079,7 +5079,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5092,7 +5092,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5190,7 +5190,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5288,7 +5288,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5301,7 +5301,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5314,7 +5314,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5441,7 +5441,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5539,7 +5539,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5552,7 +5552,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5565,7 +5565,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5663,7 +5663,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5761,7 +5761,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5774,7 +5774,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5787,7 +5787,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5914,7 +5914,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -6012,7 +6012,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -6025,7 +6025,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -6038,7 +6038,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -6136,7 +6136,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -6234,7 +6234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -6247,7 +6247,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -6260,7 +6260,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -6302,16 +6302,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -6367,7 +6368,7 @@ "multi": false, "name": "tenant", "options": [ ], - "query": "label_values(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", namespace=\"$namespace\"}, tenant)", + "query": "label_values(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", namespace=\"$namespace\"}, tenant)", "refresh": 0, "regex": "", "sort": 3, diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json index 0deb5e33b5d..71088161790 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-bloom-gateway.json @@ -140,7 +140,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Chunks", @@ -154,7 +154,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -236,7 +236,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -249,7 +249,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "expr": "kube_statefulset_status_replicas_ready{cluster=~\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", "hide": false, "instant": false, "legendFormat": "Ready", @@ -394,7 +394,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "expr": "kube_statefulset_replicas{cluster=~\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", "hide": false, "instant": false, "legendFormat": "Desired", @@ -509,7 +509,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "instant": false, "legendFormat": "Chunks", "range": true, @@ -522,7 +522,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -604,7 +604,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -617,7 +617,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", @@ -722,7 +722,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", + "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", "hide": false, "instant": false, "interval": "", @@ -838,7 +838,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", + "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", "instant": false, "legendFormat": "chunks {{ route}}", "range": true, @@ -953,7 +953,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "series", @@ -967,7 +967,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "chunks", @@ -1078,7 +1078,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, @@ -1090,7 +1090,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -1193,7 +1193,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1294,7 +1294,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, @@ -1396,7 +1396,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1457,7 +1457,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", "queryType": "range", "refId": "A" }, @@ -1467,7 +1467,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", "queryType": "range", "refId": "B" } @@ -1612,7 +1612,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", + "expr": "sum by (level) (count_over_time({cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", "queryType": "range", "refId": "A" }, @@ -1622,7 +1622,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "expr": "sum (count_over_time({cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", "queryType": "range", "refId": "B" } @@ -2126,7 +2126,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -2333,7 +2333,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, @@ -2345,7 +2345,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, @@ -2819,7 +2819,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3305,7 +3305,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})", + "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=~\"$cluster\", namespace=\"$namespace\"})", "hide": false, "instant": false, "legendFormat": "{{user}}", @@ -3403,7 +3403,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "p99", @@ -3416,7 +3416,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "p50", @@ -3527,7 +3527,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})", + "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=~\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3643,7 +3643,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "instant": false, "legendFormat": "{{status}}-p99", "range": true, @@ -3655,7 +3655,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p95", @@ -3668,7 +3668,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p90", @@ -3769,7 +3769,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "instant": false, "legendFormat": "{{status}}-p99", "range": true, @@ -3781,7 +3781,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p95", @@ -3794,7 +3794,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p90", @@ -3894,7 +3894,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", "instant": false, "legendFormat": "dequeued", "range": true, @@ -3906,7 +3906,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "processed {{status}}", @@ -4007,7 +4007,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", "instant": false, "legendFormat": "{{status}}", "range": true, @@ -4151,7 +4151,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4164,7 +4164,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -4311,7 +4311,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "expr": "sum(loki_bloom_blocks_cache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", "hide": false, "instant": false, "legendFormat": "Items", @@ -4324,7 +4324,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Added", @@ -4337,7 +4337,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)", + "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)", "hide": false, "instant": false, "legendFormat": "Evicted ({{reason}})", @@ -4350,7 +4350,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": true, "instant": false, "legendFormat": "Size", @@ -4483,7 +4483,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", + "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -4597,7 +4597,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})", "instant": false, "legendFormat": "Size", "range": true, @@ -4609,7 +4609,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", "hide": false, "instant": false, "legendFormat": "Size", @@ -4723,7 +4723,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=~\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "metas fetch rate", @@ -4736,7 +4736,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=~\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "blocks fetch rate", @@ -4749,7 +4749,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "legendFormat": "p90 blocks size", @@ -4762,7 +4762,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "legendFormat": "p90 metas size", @@ -4775,7 +4775,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4788,7 +4788,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4801,7 +4801,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4915,7 +4915,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "interval": "", @@ -5087,7 +5087,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5184,7 +5184,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5196,7 +5196,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5209,7 +5209,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5336,7 +5336,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5433,7 +5433,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5445,7 +5445,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5458,7 +5458,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5585,7 +5585,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5682,7 +5682,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5694,7 +5694,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5707,7 +5707,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5804,7 +5804,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", "format": "heatmap", "instant": false, "legendFormat": "__auto", @@ -5885,7 +5885,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", "format": "heatmap", "hide": false, "instant": false, @@ -5928,16 +5928,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-chunks.json b/production/loki-mixin-compiled-ssd/dashboards/loki-chunks.json index 10d0b09be0b..7a14609c919 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-chunks.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-chunks.json @@ -64,7 +64,7 @@ "span": 6, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"})", "format": "time_series", "legendFormat": "series", "legendLink": null @@ -111,7 +111,7 @@ "span": 6, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"})", "format": "time_series", "legendFormat": "chunks", "legendLink": null @@ -171,19 +171,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -249,19 +249,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -339,19 +339,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -416,7 +416,7 @@ "span": 6, "targets": [ { - "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Index Entries", "legendLink": null @@ -475,7 +475,7 @@ "span": 6, "targets": [ { - "expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}", + "expr": "loki_ingester_flush_queue_length{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null @@ -673,7 +673,7 @@ "stack": true, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "legendFormat": "{{status}}", "refId": "A" @@ -732,7 +732,7 @@ "span": 6, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null @@ -780,7 +780,7 @@ "stack": true, "targets": [ { - "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{reason}}", "legendLink": null @@ -843,7 +843,7 @@ "span": 12, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -905,7 +905,7 @@ "span": 12, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -981,19 +981,19 @@ "span": 12, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p99", "legendLink": null }, { - "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p90", "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p50", "legendLink": null @@ -1052,19 +1052,19 @@ "span": 12, "targets": [ { - "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p50", "legendLink": null }, { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p99", "legendLink": null }, { - "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=~\"$cluster\", job=~\"$namespace/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "avg", "legendLink": null @@ -1104,16 +1104,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json b/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json index b86882a69b8..85cddc6e4a9 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json @@ -657,16 +657,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-logs.json b/production/loki-mixin-compiled-ssd/dashboards/loki-logs.json index 476d4569013..3b0c2d94175 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-logs.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-logs.json @@ -63,7 +63,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })", + "expr": "sum by (pod)(go_goroutines{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })", "refId": "A" } ], @@ -154,7 +154,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}) by (quantile)", + "expr": "sum(go_gc_duration_seconds{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}) by (quantile)", "legendFormat": "{{quantile}}", "refId": "A" } @@ -241,7 +241,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", "refId": "A" } ], @@ -332,7 +332,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})", + "expr": "sum by (pod)(container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})", "refId": "A" } ], @@ -423,7 +423,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", "refId": "A" } ], @@ -514,7 +514,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -600,7 +600,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0", + "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0", "legendFormat": "{{reason}}", "refId": "A" } @@ -692,7 +692,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)", "legendFormat": "{{level}}", "refId": "A" } @@ -801,7 +801,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)", "intervalFactor": 3, "legendFormat": "{{level}}", "refId": "A" @@ -866,7 +866,7 @@ }, "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", "refId": "A" } ], @@ -900,16 +900,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -965,7 +966,7 @@ "multi": true, "name": "container", "options": [ ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\"}, container)", + "query": "label_values(kube_pod_container_info{cluster=~\"$cluster\", namespace=\"$namespace\"}, container)", "refresh": 0, "regex": "", "sort": 1, diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-mixin-recording-rules.json b/production/loki-mixin-compiled-ssd/dashboards/loki-mixin-recording-rules.json index f1f6c215f17..67495a1da0a 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-mixin-recording-rules.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-mixin-recording-rules.json @@ -610,16 +610,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -675,7 +676,7 @@ "multi": false, "name": "tenant", "options": [ ], - "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", + "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=~\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", "refresh": 0, "regex": "/\"([^\"]+)\"/", "sort": 1, diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json b/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json index 0ab69c6af91..d1956e581f1 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-operational.json @@ -87,7 +87,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", "legendFormat": "{{status}}", "refId": "A" } @@ -183,7 +183,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "legendFormat": "{{status}}", "refId": "A" } @@ -278,7 +278,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))", + "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))", "legendFormat": "{{tenant}}", "refId": "A" } @@ -375,7 +375,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024", + "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024", "legendFormat": "{{tenant}}", "refId": "A" } @@ -469,7 +469,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", "hide": false, "interval": "", "legendFormat": "{{container}}-{{pod}}", @@ -781,7 +781,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -988,7 +988,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1089,17 +1089,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.99", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.9", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.5", "refId": "C" } @@ -1196,17 +1196,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1302,7 +1302,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1402,17 +1402,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1508,7 +1508,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1609,7 +1609,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", "interval": "", "legendFormat": "{{ tenant }} - {{ reason }}", "refId": "A" @@ -1767,7 +1767,7 @@ "pluginVersion": "10.4.0", "targets": [ { - "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", + "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", "format": "table", "instant": true, "interval": "", @@ -1940,7 +1940,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*distributor.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*distributor.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -2035,7 +2035,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -2098,7 +2098,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"} |= \"level=error\"", "refId": "A" } ], @@ -2149,7 +2149,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -2244,7 +2244,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2338,7 +2338,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2432,7 +2432,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2539,7 +2539,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}))", + "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=~\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -2630,7 +2630,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]) > 0))", + "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]) > 0))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -2740,13 +2740,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "interval": "", "legendFormat": "Chunks", "refId": "A" }, { - "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) < 1", + "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) < 1", "interval": "", "legendFormat": "De-Dupe Ratio", "refId": "B" @@ -2824,7 +2824,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -2979,7 +2979,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(loki.*|enterprise-logs)-write\"}[$__rate_interval]))", "format": "heatmap", "instant": false, "interval": "", @@ -3167,7 +3167,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*querier.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*querier.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -3262,7 +3262,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -3325,7 +3325,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"} |= \"level=error\"", "refId": "A" } ], @@ -3376,7 +3376,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -3578,7 +3578,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*backend.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*backend.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -3673,7 +3673,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -3736,7 +3736,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} |= \"level=error\"", "refId": "A" } ], @@ -3787,7 +3787,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -3901,19 +3901,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "intervalFactor": 1, "legendFormat": "{{container}}: .99-{{method}}-{{name}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .9-{{method}}-{{name}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .5-{{method}}-{{name}}", "refId": "C" @@ -4005,7 +4005,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)", + "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)", "intervalFactor": 1, "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", "refId": "A" @@ -4118,19 +4118,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -4227,7 +4227,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)", + "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -4338,17 +4338,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4441,20 +4441,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4549,20 +4549,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4657,17 +4657,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4760,7 +4760,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4855,7 +4855,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4950,7 +4950,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5045,7 +5045,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5158,19 +5158,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5262,7 +5262,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5366,7 +5366,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5452,7 +5452,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5538,7 +5538,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_throttled_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5624,7 +5624,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5710,17 +5710,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".5", "refId": "C" } @@ -5815,19 +5815,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5918,7 +5918,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6030,19 +6030,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6133,7 +6133,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6245,19 +6245,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6348,7 +6348,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6460,19 +6460,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6563,7 +6563,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_index_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6650,16 +6650,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-reads.json b/production/loki-mixin-compiled-ssd/dashboards/loki-reads.json index 718e01f9022..6aa73172a90 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-reads.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-reads.json @@ -384,16 +384,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json index 3b3d7e7c0a4..bfc7cc0e610 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-resources-overview.json @@ -1193,16 +1193,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-retention.json b/production/loki-mixin-compiled-ssd/dashboards/loki-retention.json index 316a43298b3..03b043d8400 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-retention.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-retention.json @@ -1405,16 +1405,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-thanos-object-storage.json b/production/loki-mixin-compiled-ssd/dashboards/loki-thanos-object-storage.json index d81f10f9dd3..67343ff55b1 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-thanos-object-storage.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-thanos-object-storage.json @@ -64,7 +64,7 @@ }, "targets": [ { - "expr": "sum by(operation) (rate(loki_objstore_bucket_operations_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(operation) (rate(loki_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -111,7 +111,7 @@ }, "targets": [ { - "expr": "sum by(operation) (rate(loki_objstore_bucket_operation_failures_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) > 0", + "expr": "sum by(operation) (rate(loki_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) > 0", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -158,7 +158,7 @@ }, "targets": [ { - "expr": "sum by (method, status_code) (rate(loki_objstore_bucket_transport_requests_total{cluster=\"$cluster\", namespace=~\"$namespace\", status_code!~\"2..\"}[$__rate_interval])) > 0", + "expr": "sum by (method, status_code) (rate(loki_objstore_bucket_transport_requests_total{cluster=~\"$cluster\", namespace=~\"$namespace\", status_code!~\"2..\"}[$__rate_interval])) > 0", "format": "time_series", "legendFormat": "{{method}} - {{status_code}}", "legendLink": null @@ -219,19 +219,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -297,19 +297,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -375,19 +375,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -466,19 +466,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -544,19 +544,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -622,19 +622,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -693,16 +693,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json b/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json index e5fc0f35491..51498da74ae 100644 --- a/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json +++ b/production/loki-mixin-compiled-ssd/dashboards/loki-writes.json @@ -509,16 +509,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-build.json b/production/loki-mixin-compiled/dashboards/loki-bloom-build.json index 149dfacd857..169854a22cf 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-build.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-build.json @@ -316,7 +316,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", + "expr": "{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\"", "queryType": "range", "refId": "B" } @@ -414,7 +414,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", + "expr": "sum(count_over_time({cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} |= \"level=error\" |= \"component=bloom-planner\" [$__auto]))", "legendFormat": "Error rate", "queryType": "range", "refId": "A" @@ -459,7 +459,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", + "expr": "{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\"", "queryType": "range", "refId": "B" } @@ -557,7 +557,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum(count_over_time({cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", + "expr": "sum(count_over_time({cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} |= \"level=error\" |= \"component=bloom-builder\" [$__auto]))", "legendFormat": "Error rate", "queryType": "range", "refId": "A" @@ -759,7 +759,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Planned", @@ -772,7 +772,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Completed - {{status}}", @@ -785,7 +785,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "loki_bloomplanner_inflight_tasks{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", + "expr": "loki_bloomplanner_inflight_tasks{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", quantile=\"0.95\"}", "hide": false, "instant": false, "legendFormat": "inflight p95", @@ -798,7 +798,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", + "expr": "sum(loki_bloomplanner_queue_length{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}) > 0", "hide": false, "instant": false, "legendFormat": "Queued", @@ -1008,7 +1008,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_connected_builders{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum(loki_bloomplanner_connected_builders{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "hide": false, "instant": false, "legendFormat": "Connected builders", @@ -1021,7 +1021,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", + "expr": "count(loki_bloombuilder_processing_task{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} > 0)", "hide": false, "instant": false, "legendFormat": "Builders processing task", @@ -1034,7 +1034,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "count(loki_bloombuilder_processing_task{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", + "expr": "count(loki_bloombuilder_processing_task{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} == 0)", "hide": false, "instant": false, "legendFormat": "IDLE Builders", @@ -1132,7 +1132,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1145,7 +1145,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p90", @@ -1158,7 +1158,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.75, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p75", @@ -1171,7 +1171,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_series_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1268,7 +1268,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1281,7 +1281,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.9, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p90", @@ -1294,7 +1294,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "# series checked per compaction\nhistogram_quantile(\n 0.5, \n sum by (le) (\n rate(loki_bloombuilder_bytes_per_task_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -1516,7 +1516,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "expr": "sum(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Planned", @@ -1529,7 +1529,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", + "expr": "sum by (status) (loki_bloomplanner_tenant_tasks_completed{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", tenant=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Completed - {{status}}", @@ -1542,7 +1542,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloomplanner_queue_length{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", + "expr": "sum(loki_bloomplanner_queue_length{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", user=\"$tenant\"}) > 0", "hide": false, "instant": false, "legendFormat": "Queued", @@ -1703,7 +1703,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}))", + "expr": "sum(rate(loki_bloom_tokens_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))\n/\nsum(count(loki_bloom_tokens_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}))", "hide": false, "instant": false, "legendFormat": "Per core", @@ -1716,7 +1716,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Total", @@ -1818,7 +1818,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", + "expr": "# tokens/s by type+collision\nsum by (collision) (\n rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n) \n/ on () group_left\nsum (\n rate(loki_bloom_inserts_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1921,7 +1921,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 1.0,\n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "max", @@ -1934,7 +1934,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 0.99, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p99", @@ -1947,7 +1947,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(\n 0.50, \n sum by (le) (\n rate(loki_bloom_size_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])\n )\n)", "hide": false, "instant": false, "legendFormat": "p50", @@ -2049,7 +2049,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", + "expr": "# chunks indexed, by iteration or copied from a pre-existing bloom\nsum(rate(loki_bloom_chunks_indexed_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval])) by (type)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -2161,7 +2161,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", + "expr": "# This query shows the expected number of cpu cores we need to not fall behind\n# building blooms for data we're ingesting.\n# conceptually, the formula is:\n# (cell_bytes * space_amplification / bloom_bytes_processed_per_core)\n\n# number of replicas needed\nsum(avg_over_time(loki_cell:bytes:rate1m{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))\n*\n## Space amplification (how much data do we write compared to what we ingest?)\n(\n # rep factor\n 3 *\n sum(\n # 1 - dedupe_ratio\n 1 - \n sum(rate(loki_chunk_store_deduped_chunks_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n /\n sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (cluster, namespace)\n )\n)\n/\n(\nsum(rate(loki_bloombuilder_chunk_series_size_sum{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n/\nsum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))\n)", "hide": false, "instant": false, "legendFormat": "Needed", @@ -2174,7 +2174,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Available", @@ -2973,7 +2973,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n) > 0", "instant": false, "legendFormat": "Restarts", "range": true, @@ -3077,7 +3077,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-builder\"}\n )\n) > 0", "instant": false, "legendFormat": "{{reason}} / {{pod}}", "range": true, @@ -3746,7 +3746,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", + "expr": "sum (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n) > 0", "instant": false, "legendFormat": "Restarts", "range": true, @@ -3850,7 +3850,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", + "expr": "(\n sum by (pod) (\n increase(\n kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}[10m]\n )\n )\n * on (pod) group_right\n max by (pod, reason) (\n kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-planner\"}\n )\n) > 0", "instant": false, "legendFormat": "{{reason}} / {{pod}}", "range": true, @@ -3960,7 +3960,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Metas", @@ -4028,7 +4028,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "expr": "sum(increase(loki_bloombuilder_metas_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", "format": "table", "hide": false, "instant": true, @@ -4126,7 +4126,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "expr": "sum by (phase) (increase(loki_bloomplanner_metas_deleted_total{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Deleted during {{phase}}", @@ -4237,7 +4237,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Blocks", @@ -4305,7 +4305,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", + "expr": "sum(increase(loki_bloombuilder_blocks_created_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__range]))", "format": "time_series", "hide": false, "instant": true, @@ -4403,7 +4403,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", + "expr": "sum by (phase) (increase(loki_bloomplanner_blocks_deleted_total{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Deleted during {{phase}}", @@ -4500,7 +4500,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", + "expr": "sum(increase(loki_bloombuilder_blocks_reused_total{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Blocks", @@ -4603,7 +4603,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "instant": true, "legendFormat": "__auto", "range": false, @@ -4703,7 +4703,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"})", + "expr": "sum by (cluster, namespace) (loki_bloomplanner_retention_running{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"})", "legendFormat": "__auto", "range": true, "refId": "A" @@ -4799,7 +4799,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", + "expr": "histogram_quantile(0.9, \n sum by (status, le) (\n rate(loki_bloomplanner_retention_time_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"}[$__rate_interval])\n )\n)", "legendFormat": "__auto", "range": true, "refId": "A" @@ -4968,7 +4968,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5066,7 +5066,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5079,7 +5079,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5092,7 +5092,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5190,7 +5190,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (container, status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5288,7 +5288,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5301,7 +5301,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5314,7 +5314,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5441,7 +5441,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5539,7 +5539,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5552,7 +5552,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5565,7 +5565,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\", job=\"\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5663,7 +5663,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -5761,7 +5761,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -5774,7 +5774,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5787,7 +5787,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5914,7 +5914,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -6012,7 +6012,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -6025,7 +6025,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -6038,7 +6038,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-planner\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -6136,7 +6136,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "{{operation}} {{status_code}}", @@ -6234,7 +6234,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p99", @@ -6247,7 +6247,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -6260,7 +6260,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-builder\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -6302,16 +6302,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -6367,7 +6368,7 @@ "multi": false, "name": "tenant", "options": [ ], - "query": "label_values(loki_bloomplanner_tenant_tasks_planned{cluster=\"$cluster\", namespace=\"$namespace\"}, tenant)", + "query": "label_values(loki_bloomplanner_tenant_tasks_planned{cluster=~\"$cluster\", namespace=\"$namespace\"}, tenant)", "refresh": 0, "regex": "", "sort": 3, diff --git a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json index 0deb5e33b5d..71088161790 100644 --- a/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json +++ b/production/loki-mixin-compiled/dashboards/loki-bloom-gateway.json @@ -140,7 +140,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Chunks", @@ -154,7 +154,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_filtered_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_requested_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -236,7 +236,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -249,7 +249,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_filtered_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_requested_series_sum{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", @@ -381,7 +381,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kube_statefulset_status_replicas_ready{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "expr": "kube_statefulset_status_replicas_ready{cluster=~\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", "hide": false, "instant": false, "legendFormat": "Ready", @@ -394,7 +394,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "kube_statefulset_replicas{cluster=\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", + "expr": "kube_statefulset_replicas{cluster=~\"$cluster\", namespace=\"$namespace\", statefulset=\"bloom-gateway\"}", "hide": false, "instant": false, "legendFormat": "Desired", @@ -509,7 +509,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "instant": false, "legendFormat": "Chunks", "range": true, @@ -522,7 +522,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_series_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Series", @@ -604,7 +604,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_querier_chunks_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", "instant": true, "legendFormat": "Chunks", "range": false, @@ -617,7 +617,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", + "expr": "sum(increase(loki_bloom_gateway_querier_series_filtered_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))\n/\nsum(increase(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__range]))", "hide": false, "instant": true, "legendFormat": "Series", @@ -722,7 +722,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", + "expr": "(\n max by (pod, reason) (kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n * on (pod) group_left\n sum by (pod) (increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))\n) > 0", "hide": false, "instant": false, "interval": "", @@ -838,7 +838,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", + "expr": "(\n sum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n -\n sum(rate(loki_index_gateway_postfilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)\n)\n/\nsum(rate(loki_index_gateway_prefilter_chunks_sum{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval])) by (route)", "instant": false, "legendFormat": "chunks {{ route}}", "range": true, @@ -953,7 +953,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_series_skipped_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_series_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "series", @@ -967,7 +967,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_querier_chunks_skipped_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))\n/\nsum(rate(loki_bloom_gateway_querier_chunks_total{cluster=~\"$cluster\", job=\"$namespace/index-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "chunks", @@ -1078,7 +1078,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, @@ -1090,7 +1090,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -1193,7 +1193,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_chunks_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1294,7 +1294,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=~\"(found|skipped|missed)\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"requested\"}[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, @@ -1396,7 +1396,7 @@ "uid": "000000134" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"filtered\"}[$__rate_interval])) by (type)\n/ on () group_left\nsum(rate(loki_bloom_recorder_series_total{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\", type=\"found\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -1457,7 +1457,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=error\" or \"panic:\" | logfmt", "queryType": "range", "refId": "A" }, @@ -1467,7 +1467,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"level=warn\" | logfmt", "queryType": "range", "refId": "B" } @@ -1612,7 +1612,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum by (level) (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", + "expr": "sum by (level) (count_over_time({cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |~ \"level=(warn|error)\" | logfmt [$__auto]))", "queryType": "range", "refId": "A" }, @@ -1622,7 +1622,7 @@ "uid": "${loki_datasource}" }, "editorMode": "code", - "expr": "sum (count_over_time({cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", + "expr": "sum (count_over_time({cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"} |= \"panic:\" | logfmt [$__auto]))", "queryType": "range", "refId": "B" } @@ -2126,7 +2126,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (pod) (rate(go_gc_cycles_total_gc_cycles_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "__auto", @@ -2333,7 +2333,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(go_gc_pauses_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, @@ -2345,7 +2345,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(go_gc_pauses_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (le))", "hide": false, "legendFormat": "__auto", "range": true, @@ -2819,7 +2819,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", + "expr": "sum by (status_code) (\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=\"$namespace/bloom-gateway\", route=\"/logproto.BloomGateway/FilterChunkRefs\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3305,7 +3305,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=\"$cluster\", namespace=\"$namespace\"})", + "expr": "sum by (user) (loki_bloom_gateway_queue_length{cluster=~\"$cluster\", namespace=\"$namespace\"})", "hide": false, "instant": false, "legendFormat": "{{user}}", @@ -3403,7 +3403,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "p99", @@ -3416,7 +3416,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (le) (rate(loki_bloom_gateway_queue_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "p50", @@ -3527,7 +3527,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})", + "expr": "sum by (quantile) (loki_bloom_gateway_inflight_tasks{cluster=~\"$cluster\", namespace=\"$namespace\", quantile=\"0.99\"})", "hide": false, "instant": false, "legendFormat": "__auto", @@ -3643,7 +3643,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "instant": false, "legendFormat": "{{status}}-p99", "range": true, @@ -3655,7 +3655,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p95", @@ -3668,7 +3668,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_process_duration_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p90", @@ -3769,7 +3769,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.99, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "instant": false, "legendFormat": "{{status}}-p99", "range": true, @@ -3781,7 +3781,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p95", @@ -3794,7 +3794,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", + "expr": "histogram_quantile(0.90, sum(rate(loki_bloom_gateway_block_query_latency_seconds_bucket{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (le, status))", "hide": false, "instant": false, "legendFormat": "{{status}}-p90", @@ -3894,7 +3894,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_gateway_tasks_dequeued_total{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", "instant": false, "legendFormat": "dequeued", "range": true, @@ -3906,7 +3906,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (status) (rate(loki_bloom_gateway_tasks_processed_total{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "processed {{status}}", @@ -4007,7 +4007,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", + "expr": "sum(rate(loki_bloom_gateway_process_duration_seconds_count{cluster=~\"$cluster\",namespace=\"$namespace\",container=\"bloom-gateway\"}[$__rate_interval])) by (status)", "instant": false, "legendFormat": "{{status}}", "range": true, @@ -4151,7 +4151,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"}) by (pod)", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4164,7 +4164,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}) by (pod)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -4311,7 +4311,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "expr": "sum(loki_bloom_blocks_cache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", "hide": false, "instant": false, "legendFormat": "Items", @@ -4324,7 +4324,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_blocks_cache_added_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "Added", @@ -4337,7 +4337,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)", + "expr": "sum(rate(loki_bloom_blocks_cache_evicted_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])) by (reason)", "hide": false, "instant": false, "legendFormat": "Evicted ({{reason}})", @@ -4350,7 +4350,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": true, "instant": false, "legendFormat": "Size", @@ -4483,7 +4483,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", + "expr": "sum by (status) (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)\n/ ignoring(status) group_left\nsum (\n rate(loki_bloom_blocks_cache_fetched_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval])\n)", "hide": false, "instant": false, "legendFormat": "__auto", @@ -4597,7 +4597,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_embeddedcache_memory_bytes{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})", + "expr": "sum(loki_embeddedcache_memory_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})\n/\nsum(loki_embeddedcache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", cache=\"bloom-blocks-cache\", container=\"bloom-gateway\"})", "instant": false, "legendFormat": "Size", "range": true, @@ -4609,7 +4609,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", + "expr": "sum(loki_bloom_blocks_cache_usage_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})\n/\nsum(loki_bloom_blocks_cache_entries{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"})", "hide": false, "instant": false, "legendFormat": "Size", @@ -4723,7 +4723,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_metas_fetched_sum{cluster=~\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "metas fetch rate", @@ -4736,7 +4736,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum(rate(loki_bloom_store_blocks_fetched_sum{cluster=~\"$cluster\",namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "blocks fetch rate", @@ -4749,7 +4749,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_blocks_fetched_size_bytes_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "legendFormat": "p90 blocks size", @@ -4762,7 +4762,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.9, sum(rate(loki_bloom_store_metas_fetched_size_bytes_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": false, "instant": false, "legendFormat": "p90 metas size", @@ -4775,7 +4775,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(1.0, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4788,7 +4788,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4801,7 +4801,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_bloom_store_metas_fetched_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", "hide": true, "instant": false, "legendFormat": "__auto", @@ -4915,7 +4915,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", + "expr": "sum by (job)(rate(loki_bloom_store_download_queue_size_sum{cluster=~\"$cluster\", namespace=\"$namespace\", container=\"bloom-gateway\"}[$__rate_interval]))", "hide": false, "instant": false, "interval": "", @@ -5087,7 +5087,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5184,7 +5184,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5196,7 +5196,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5209,7 +5209,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5336,7 +5336,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5433,7 +5433,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5445,7 +5445,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5458,7 +5458,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5585,7 +5585,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", + "expr": "sum by (status_code, operation) (rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{status_code}}", "range": true, @@ -5682,7 +5682,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "instant": false, "legendFormat": "{{operation}} p99", "range": true, @@ -5694,7 +5694,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.90, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p90", @@ -5707,7 +5707,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", + "expr": "histogram_quantile(0.50, sum by (operation, le) (rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"} [$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{operation}} p50", @@ -5804,7 +5804,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "expr": "increase(loki_bloom_gateway_dequeue_duration_seconds_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", "format": "heatmap", "instant": false, "legendFormat": "__auto", @@ -5885,7 +5885,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", + "expr": "increase(loki_bloom_gateway_tasks_dequeued_bucket{cluster=~\"$cluster\", job=\"$namespace/bloom-gateway\"}[$__rate_interval])", "format": "heatmap", "hide": false, "instant": false, @@ -5928,16 +5928,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-chunks.json b/production/loki-mixin-compiled/dashboards/loki-chunks.json index 7105552025f..bde7ee1a5eb 100644 --- a/production/loki-mixin-compiled/dashboards/loki-chunks.json +++ b/production/loki-mixin-compiled/dashboards/loki-chunks.json @@ -64,7 +64,7 @@ "span": 6, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"})", "format": "time_series", "legendFormat": "series", "legendLink": null @@ -111,7 +111,7 @@ "span": 6, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}) / sum(loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"})", "format": "time_series", "legendFormat": "chunks", "legendLink": null @@ -171,19 +171,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -249,19 +249,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -339,19 +339,19 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -416,7 +416,7 @@ "span": 6, "targets": [ { - "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Index Entries", "legendLink": null @@ -475,7 +475,7 @@ "span": 6, "targets": [ { - "expr": "loki_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"} or cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}", + "expr": "loki_ingester_flush_queue_length{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"} or cortex_ingester_flush_queue_length{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null @@ -673,7 +673,7 @@ "stack": true, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "legendFormat": "{{status}}", "refId": "A" @@ -732,7 +732,7 @@ "span": 6, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{pod}}", "legendLink": null @@ -780,7 +780,7 @@ "stack": true, "targets": [ { - "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{reason}}", "legendLink": null @@ -843,7 +843,7 @@ "span": 12, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -905,7 +905,7 @@ "span": 12, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -981,19 +981,19 @@ "span": 12, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p99", "legendLink": null }, { - "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p90", "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p50", "legendLink": null @@ -1052,19 +1052,19 @@ "span": 12, "targets": [ { - "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p50", "legendLink": null }, { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) by (le))", "format": "time_series", "legendFormat": "p99", "legendLink": null }, { - "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=~\"$cluster\", job=~\"$namespace/(.*ingester.*)\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "avg", "legendLink": null @@ -1104,16 +1104,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-deletion.json b/production/loki-mixin-compiled/dashboards/loki-deletion.json index b86882a69b8..85cddc6e4a9 100644 --- a/production/loki-mixin-compiled/dashboards/loki-deletion.json +++ b/production/loki-mixin-compiled/dashboards/loki-deletion.json @@ -657,16 +657,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-logs.json b/production/loki-mixin-compiled/dashboards/loki-logs.json index 476d4569013..3b0c2d94175 100644 --- a/production/loki-mixin-compiled/dashboards/loki-logs.json +++ b/production/loki-mixin-compiled/dashboards/loki-logs.json @@ -63,7 +63,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })", + "expr": "sum by (pod)(go_goroutines{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\" })", "refId": "A" } ], @@ -154,7 +154,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}) by (quantile)", + "expr": "sum(go_gc_duration_seconds{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}) by (quantile)", "legendFormat": "{{quantile}}", "refId": "A" } @@ -241,7 +241,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", "refId": "A" } ], @@ -332,7 +332,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})", + "expr": "sum by (pod)(container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"})", "refId": "A" } ], @@ -423,7 +423,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval]))", "refId": "A" } ], @@ -514,7 +514,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (pod)(rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -600,7 +600,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0", + "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"}[30m]) > 0", "legendFormat": "{{reason}}", "refId": "A" } @@ -692,7 +692,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\", container=~\"$container\"}[$__rate_interval])) by (level)", "legendFormat": "{{level}}", "refId": "A" } @@ -801,7 +801,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" | __error__=\"\" [$__auto])) by (level)", "intervalFactor": 3, "legendFormat": "{{level}}", "refId": "A" @@ -866,7 +866,7 @@ }, "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", "refId": "A" } ], @@ -900,16 +900,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -965,7 +966,7 @@ "multi": true, "name": "container", "options": [ ], - "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\"}, container)", + "query": "label_values(kube_pod_container_info{cluster=~\"$cluster\", namespace=\"$namespace\"}, container)", "refresh": 0, "regex": "", "sort": 1, diff --git a/production/loki-mixin-compiled/dashboards/loki-mixin-recording-rules.json b/production/loki-mixin-compiled/dashboards/loki-mixin-recording-rules.json index f1f6c215f17..67495a1da0a 100644 --- a/production/loki-mixin-compiled/dashboards/loki-mixin-recording-rules.json +++ b/production/loki-mixin-compiled/dashboards/loki-mixin-recording-rules.json @@ -610,16 +610,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", @@ -675,7 +676,7 @@ "multi": false, "name": "tenant", "options": [ ], - "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", + "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=~\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", "refresh": 0, "regex": "/\"([^\"]+)\"/", "sort": 1, diff --git a/production/loki-mixin-compiled/dashboards/loki-operational.json b/production/loki-mixin-compiled/dashboards/loki-operational.json index fb27ec2eb56..f34cd72eb7a 100644 --- a/production/loki-mixin-compiled/dashboards/loki-operational.json +++ b/production/loki-mixin-compiled/dashboards/loki-operational.json @@ -87,7 +87,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", "legendFormat": "{{status}}", "refId": "A" } @@ -183,7 +183,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/distributor\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "legendFormat": "{{status}}", "refId": "A" } @@ -280,7 +280,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", + "expr": "topk(5, sum by (name,level) (rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\"}[$__interval])) - \nsum by (name,level) (rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\"}[$__interval] offset 1h)))", "legendFormat": "{{name}}-{{level}}", "refId": "A" } @@ -375,7 +375,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))", + "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant))", "legendFormat": "{{tenant}}", "refId": "A" } @@ -472,7 +472,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024", + "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (tenant)) / 1024 / 1024", "legendFormat": "{{tenant}}", "refId": "A" } @@ -566,7 +566,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", "hide": false, "interval": "", "legendFormat": "{{container}}-{{pod}}", @@ -878,7 +878,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1085,7 +1085,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1186,17 +1186,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.99", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.9", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.5", "refId": "C" } @@ -1293,17 +1293,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/querier\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1399,7 +1399,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1499,17 +1499,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1605,7 +1605,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1706,7 +1706,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])))", "interval": "", "legendFormat": "{{ tenant }} - {{ reason }}", "refId": "A" @@ -1864,7 +1864,7 @@ "pluginVersion": "10.4.0", "targets": [ { - "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", + "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[$__rate_interval])[$__range:$__rate_interval])))", "format": "table", "instant": true, "interval": "", @@ -2037,7 +2037,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*distributor.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*distributor.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -2132,7 +2132,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -2195,7 +2195,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"} |= \"level=error\"", "refId": "A" } ], @@ -2246,7 +2246,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/distributor\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -2341,7 +2341,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2435,7 +2435,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2529,7 +2529,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", + "expr": "sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2730,7 +2730,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*ingester.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*ingester.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -2825,7 +2825,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -2888,7 +2888,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"} |= \"level=error\"", "refId": "A" } ], @@ -2939,7 +2939,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -3047,7 +3047,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}))", + "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=~\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -3138,7 +3138,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]) > 0))", + "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=~\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]) > 0))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -3248,13 +3248,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))", "interval": "", "legendFormat": "Chunks", "refId": "A" }, { - "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) < 1", + "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) < 1", "interval": "", "legendFormat": "De-Dupe Ratio", "refId": "B" @@ -3332,7 +3332,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -3487,7 +3487,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ingester.*|partition-ingester.*)\"}[$__rate_interval]))", "format": "heatmap", "instant": false, "interval": "", @@ -3675,7 +3675,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", container=~\".*querier.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", container=~\".*querier.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -3770,7 +3770,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"[$__auto]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} | logfmt | level=\"error\"[$__auto]))", "refId": "A" } ], @@ -3833,7 +3833,7 @@ "panels": [ ], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"} |= \"level=error\"", "refId": "A" } ], @@ -3884,7 +3884,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -3998,19 +3998,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "intervalFactor": 1, "legendFormat": "{{container}}: .99-{{method}}-{{name}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .9-{{method}}-{{name}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", + "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .5-{{method}}-{{name}}", "refId": "C" @@ -4102,7 +4102,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)", + "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, method, name, container)", "intervalFactor": 1, "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", "refId": "A" @@ -4215,19 +4215,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -4324,7 +4324,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)", + "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, status_code, method)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -4435,17 +4435,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4538,20 +4538,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4646,20 +4646,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4754,17 +4754,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (operation, le))", "refId": "C" } ], @@ -4857,7 +4857,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4952,7 +4952,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5047,7 +5047,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5142,7 +5142,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[$__rate_interval])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -5255,19 +5255,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5359,7 +5359,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5463,7 +5463,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5549,7 +5549,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5635,7 +5635,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_throttled_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5721,7 +5721,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", "refId": "A" } ], @@ -5807,17 +5807,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])))", "legendFormat": ".5", "refId": "C" } @@ -5912,19 +5912,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6015,7 +6015,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6127,19 +6127,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6230,7 +6230,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6342,19 +6342,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6445,7 +6445,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6557,19 +6557,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_index_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_index_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6660,7 +6660,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_index_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", + "expr": "sum(rate(loki_index_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6747,16 +6747,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-reads-resources.json b/production/loki-mixin-compiled/dashboards/loki-reads-resources.json index af0444961d8..be608c198a2 100644 --- a/production/loki-mixin-compiled/dashboards/loki-reads-resources.json +++ b/production/loki-mixin-compiled/dashboards/loki-reads-resources.json @@ -2363,16 +2363,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-reads.json b/production/loki-mixin-compiled/dashboards/loki-reads.json index 60dbeced283..72b54664eaf 100644 --- a/production/loki-mixin-compiled/dashboards/loki-reads.json +++ b/production/loki-mixin-compiled/dashboards/loki-reads.json @@ -2757,16 +2757,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-retention.json b/production/loki-mixin-compiled/dashboards/loki-retention.json index ac719445d24..40e7f432995 100644 --- a/production/loki-mixin-compiled/dashboards/loki-retention.json +++ b/production/loki-mixin-compiled/dashboards/loki-retention.json @@ -1405,16 +1405,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-thanos-object-storage.json b/production/loki-mixin-compiled/dashboards/loki-thanos-object-storage.json index d81f10f9dd3..67343ff55b1 100644 --- a/production/loki-mixin-compiled/dashboards/loki-thanos-object-storage.json +++ b/production/loki-mixin-compiled/dashboards/loki-thanos-object-storage.json @@ -64,7 +64,7 @@ }, "targets": [ { - "expr": "sum by(operation) (rate(loki_objstore_bucket_operations_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by(operation) (rate(loki_objstore_bucket_operations_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -111,7 +111,7 @@ }, "targets": [ { - "expr": "sum by(operation) (rate(loki_objstore_bucket_operation_failures_total{cluster=\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) > 0", + "expr": "sum by(operation) (rate(loki_objstore_bucket_operation_failures_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) > 0", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -158,7 +158,7 @@ }, "targets": [ { - "expr": "sum by (method, status_code) (rate(loki_objstore_bucket_transport_requests_total{cluster=\"$cluster\", namespace=~\"$namespace\", status_code!~\"2..\"}[$__rate_interval])) > 0", + "expr": "sum by (method, status_code) (rate(loki_objstore_bucket_transport_requests_total{cluster=~\"$cluster\", namespace=~\"$namespace\", status_code!~\"2..\"}[$__rate_interval])) > 0", "format": "time_series", "legendFormat": "{{method}} - {{status_code}}", "legendLink": null @@ -219,19 +219,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -297,19 +297,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -375,19 +375,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -466,19 +466,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -544,19 +544,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -622,19 +622,19 @@ }, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_objstore_bucket_operation_duration_seconds_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))", + "expr": "sum(rate(loki_objstore_bucket_operation_duration_seconds_sum{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(loki_objstore_bucket_operation_duration_seconds_count{cluster=~\"$cluster\", namespace=~\"$namespace\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "legendFormat": "Average", "refId": "C" @@ -693,16 +693,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-writes-resources.json b/production/loki-mixin-compiled/dashboards/loki-writes-resources.json index 1188aea91dc..fa1269f4abe 100644 --- a/production/loki-mixin-compiled/dashboards/loki-writes-resources.json +++ b/production/loki-mixin-compiled/dashboards/loki-writes-resources.json @@ -764,16 +764,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin-compiled/dashboards/loki-writes.json b/production/loki-mixin-compiled/dashboards/loki-writes.json index f89c943696c..0fcc56d3ccd 100644 --- a/production/loki-mixin-compiled/dashboards/loki-writes.json +++ b/production/loki-mixin-compiled/dashboards/loki-writes.json @@ -1865,16 +1865,17 @@ "type": "datasource" }, { - "allValue": null, + "allValue": ".+", "current": { - "text": "prod", - "value": "prod" + "selected": true, + "text": "All", + "value": "$__all" }, "datasource": "$datasource", "hide": 0, - "includeAll": false, + "includeAll": true, "label": "cluster", - "multi": false, + "multi": true, "name": "cluster", "options": [ ], "query": "label_values(loki_build_info, cluster)", diff --git a/production/loki-mixin/dashboards/dashboard-utils.libsonnet b/production/loki-mixin/dashboards/dashboard-utils.libsonnet index 65bcad5620e..8cfd4a43146 100644 --- a/production/loki-mixin/dashboards/dashboard-utils.libsonnet +++ b/production/loki-mixin/dashboards/dashboard-utils.libsonnet @@ -27,7 +27,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; }, }, - addCluster(multi=false):: + addCluster(multi=true):: if multi then self.addMultiTemplate('cluster', 'loki_build_info', $._config.per_cluster_label) else @@ -87,6 +87,50 @@ local utils = import 'mixin-utils/utils.libsonnet'; namespaceMatcher():: $._config.per_cluster_label + '=~"$cluster", ' + $._config.per_namespace_label + '=~"$namespace"', + // replaceClusterMatchers normalizes cluster matchers in queries imported from + // exported dashboard JSON. + replaceClusterMatchers(expr, showMultiCluster=true):: + if showMultiCluster then + std.strReplace( + std.strReplace( + std.strReplace( + expr, + 'cluster=~"$cluster"', + $._config.per_cluster_label + '=~"$cluster"' + ), + 'cluster="$cluster"', + $._config.per_cluster_label + '=~"$cluster"' + ), + 'cluster_job', + $._config.per_cluster_label + '_job' + ) + else + std.strReplace( + std.strReplace( + std.strReplace( + std.strReplace( + std.strReplace( + std.strReplace( + expr, + 'cluster="$cluster", ', + '' + ), + 'cluster=~"$cluster", ', + '' + ), + ', cluster="$cluster"', + '' + ), + ', cluster=~"$cluster"', + '' + ), + 'cluster="$cluster",', + '' + ), + 'cluster=~"$cluster",', + '' + ), + logPanel(title, selector, datasource='$loki_datasource'):: { title: title, type: 'logs', diff --git a/production/loki-mixin/dashboards/loki-bloom-build.libsonnet b/production/loki-mixin/dashboards/loki-bloom-build.libsonnet index 30bb8566ae8..b543ade1786 100644 --- a/production/loki-mixin/dashboards/loki-bloom-build.libsonnet +++ b/production/loki-mixin/dashboards/loki-bloom-build.libsonnet @@ -12,7 +12,7 @@ local template = import 'grafonnet/template.libsonnet'; template.new( 'tenant', '$datasource', - 'label_values(loki_bloomplanner_tenant_tasks_planned{cluster="$cluster", ' + $._config.per_namespace_label + '="$namespace"}, tenant)', + 'label_values(loki_bloomplanner_tenant_tasks_planned{cluster=~"$cluster", ' + $._config.per_namespace_label + '="$namespace"}, tenant)', label='Tenant', sort=3, // numerical ascending includeAll=true, @@ -23,29 +23,11 @@ local template = import 'grafonnet/template.libsonnet'; 'loki-bloom-build.json': raw { - local replaceClusterMatchers(expr) = - // Replace the recording rules cluster label with the per-cluster label - std.strReplace( - // Replace the cluster label for equality matchers with the per-cluster label - std.strReplace( - // Replace the cluster label for regex matchers with the per-cluster label - std.strReplace( - expr, - 'cluster=~"$cluster"', - $._config.per_cluster_label + '=~"$cluster"' - ), - 'cluster="$cluster"', - $._config.per_cluster_label + '="$cluster"' - ), - 'cluster_job', - $._config.per_cluster_label + '_job' - ), - panels: [ p { targets: if std.objectHas(p, 'targets') then [ e { - expr: replaceClusterMatchers(e.expr), + expr: $.replaceClusterMatchers(e.expr), } for e in p.targets ] else [], @@ -53,7 +35,7 @@ local template = import 'grafonnet/template.libsonnet'; sp { targets: if std.objectHas(sp, 'targets') then [ spe { - expr: replaceClusterMatchers(spe.expr), + expr: $.replaceClusterMatchers(spe.expr), } for spe in sp.targets ] else [], @@ -61,7 +43,7 @@ local template = import 'grafonnet/template.libsonnet'; ssp { targets: if std.objectHas(ssp, 'targets') then [ sspe { - expr: replaceClusterMatchers(sspe.expr), + expr: $.replaceClusterMatchers(sspe.expr), } for sspe in ssp.targets ] else [], diff --git a/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet index 40e1644ac02..b5f2721d289 100644 --- a/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet +++ b/production/loki-mixin/dashboards/loki-bloom-gateway.libsonnet @@ -10,29 +10,11 @@ local raw = (import './dashboard-bloom-gateway.json'); 'loki-bloom-gateway.json': raw { - local replaceClusterMatchers(expr) = - // Replace the recording rules cluster label with the per-cluster label - std.strReplace( - // Replace the cluster label for equality matchers with the per-cluster label - std.strReplace( - // Replace the cluster label for regex matchers with the per-cluster label - std.strReplace( - expr, - 'cluster=~"$cluster"', - $._config.per_cluster_label + '=~"$cluster"' - ), - 'cluster="$cluster"', - $._config.per_cluster_label + '="$cluster"' - ), - 'cluster_job', - $._config.per_cluster_label + '_job' - ), - panels: [ p { targets: if std.objectHas(p, 'targets') then [ e { - expr: replaceClusterMatchers(e.expr), + expr: $.replaceClusterMatchers(e.expr), } for e in p.targets ] else [], @@ -40,7 +22,7 @@ local raw = (import './dashboard-bloom-gateway.json'); sp { targets: if std.objectHas(sp, 'targets') then [ spe { - expr: replaceClusterMatchers(spe.expr), + expr: $.replaceClusterMatchers(spe.expr), } for spe in sp.targets ] else [], @@ -48,7 +30,7 @@ local raw = (import './dashboard-bloom-gateway.json'); ssp { targets: if std.objectHas(ssp, 'targets') then [ sspe { - expr: replaceClusterMatchers(sspe.expr), + expr: $.replaceClusterMatchers(sspe.expr), } for sspe in ssp.targets ] else [], diff --git a/production/loki-mixin/dashboards/loki-chunks.libsonnet b/production/loki-mixin/dashboards/loki-chunks.libsonnet index b4d2783b582..2ae9e274e5c 100644 --- a/production/loki-mixin/dashboards/loki-chunks.libsonnet +++ b/production/loki-mixin/dashboards/loki-chunks.libsonnet @@ -6,7 +6,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; local dashboards = self, 'loki-chunks.json': { local cfg = self, - labelsSelector:: $._config.per_cluster_label + '="$cluster", job=~"$namespace/%s"' % ( + labelsSelector:: $._config.per_cluster_label + '=~"$cluster", job=~"$namespace/%s"' % ( if $._config.meta_monitoring.enabled then '(.*ingester.*|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher else if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else '(.*ingester.*)' diff --git a/production/loki-mixin/dashboards/loki-logs.libsonnet b/production/loki-mixin/dashboards/loki-logs.libsonnet index 2800ec33bc0..ee461a6a7c6 100644 --- a/production/loki-mixin/dashboards/loki-logs.libsonnet +++ b/production/loki-mixin/dashboards/loki-logs.libsonnet @@ -7,7 +7,7 @@ local template = import 'grafonnet/template.libsonnet'; template.new( 'container', '$datasource', - 'label_values(kube_pod_container_info{' + $._config.per_cluster_label + '="$cluster", ' + $._config.per_namespace_label + '="$namespace"}, container)', + 'label_values(kube_pod_container_info{' + $._config.per_cluster_label + '=~"$cluster", ' + $._config.per_namespace_label + '="$namespace"}, container)', sort=1, multi=true, ), @@ -45,9 +45,7 @@ local template = import 'grafonnet/template.libsonnet'; p { targets: [ e { - expr: if dashboards['loki-logs.json'].showMultiCluster - then std.strReplace(super.expr, 'cluster="$cluster"', $._config.per_cluster_label + '="$cluster"') - else std.strReplace(super.expr, 'cluster="$cluster", ', ''), + expr: $.replaceClusterMatchers(super.expr, dashboards['loki-logs.json'].showMultiCluster), } for e in p.targets ], diff --git a/production/loki-mixin/dashboards/loki-object-store.libsonnet b/production/loki-mixin/dashboards/loki-object-store.libsonnet index 002dd284e29..842f4658a0f 100644 --- a/production/loki-mixin/dashboards/loki-object-store.libsonnet +++ b/production/loki-mixin/dashboards/loki-object-store.libsonnet @@ -3,7 +3,7 @@ local row = grafana.row; { grafanaDashboards+:: if !$._config.thanos.enabled then {} else { - local cluster_namespace_matcher = 'cluster="$cluster", namespace=~"$namespace"', + local cluster_namespace_matcher = 'cluster=~"$cluster", namespace=~"$namespace"', local dashboard = ( (import 'dashboard-utils.libsonnet') + { _config+:: $._config, diff --git a/production/loki-mixin/dashboards/loki-operational.libsonnet b/production/loki-mixin/dashboards/loki-operational.libsonnet index 14e2ad474d5..36898c682e8 100644 --- a/production/loki-mixin/dashboards/loki-operational.libsonnet +++ b/production/loki-mixin/dashboards/loki-operational.libsonnet @@ -84,39 +84,6 @@ local utils = import 'mixin-utils/utils.libsonnet'; else error 'no pod matchers' else error 'matcher must be either job or container', - local replaceClusterMatchers(expr) = - if dashboards['loki-operational.json'].showMultiCluster - // Replace the recording rules cluster label with the per-cluster label - then std.strReplace( - // Replace the cluster label for equality matchers with the per-cluster label - std.strReplace( - // Replace the cluster label for regex matchers with the per-cluster label - std.strReplace( - expr, - 'cluster=~"$cluster"', - $._config.per_cluster_label + '=~"$cluster"' - ), - 'cluster="$cluster"', - $._config.per_cluster_label + '="$cluster"' - ), - 'cluster_job', - $._config.per_cluster_label + '_job' - ) - else - std.strReplace( - std.strReplace( - std.strReplace( - expr, - ', ' + $._config.per_cluster_label + '="$cluster"', - '' - ), - ', ' + $._config.per_cluster_label + '=~"$cluster"', - '' - ), - $._config.per_cluster_label + '="$cluster",', - '' - ), - local replaceBackendMatchers(expr) = std.strReplace( std.strReplace( @@ -239,7 +206,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; datasource: selectDatasource(super.datasource), targets: if std.objectHas(p, 'targets') then [ e { - expr: removeInternalComponents(p.title, replaceClusterMatchers(e.expr)), + expr: removeInternalComponents(p.title, $.replaceClusterMatchers(e.expr, dashboards['loki-operational.json'].showMultiCluster)), } for e in p.targets ] else [], @@ -248,7 +215,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; datasource: selectDatasource(super.datasource), targets: if std.objectHas(sp, 'targets') then [ e { - expr: removeInternalComponents(p.title, replaceClusterMatchers(e.expr)), + expr: removeInternalComponents(p.title, $.replaceClusterMatchers(e.expr, dashboards['loki-operational.json'].showMultiCluster)), } for e in sp.targets ] else [], @@ -257,7 +224,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; datasource: selectDatasource(super.datasource), targets: if std.objectHas(ssp, 'targets') then [ e { - expr: removeInternalComponents(p.title, replaceClusterMatchers(e.expr)), + expr: removeInternalComponents(p.title, $.replaceClusterMatchers(e.expr, dashboards['loki-operational.json'].showMultiCluster)), } for e in ssp.targets ] else [], diff --git a/production/loki-mixin/dashboards/recording-rules.libsonnet b/production/loki-mixin/dashboards/recording-rules.libsonnet index 8b47db7dfaa..efb298af997 100644 --- a/production/loki-mixin/dashboards/recording-rules.libsonnet +++ b/production/loki-mixin/dashboards/recording-rules.libsonnet @@ -7,7 +7,7 @@ local template = import 'grafonnet/template.libsonnet'; template.new( 'tenant', '$datasource', - 'query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{' + $._config.per_cluster_label + '="$cluster",' + $._config.per_namespace_label + '="$namespace"},"id","$1","tenant","(.*)")) by(id))', + 'query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{' + $._config.per_cluster_label + '=~"$cluster",' + $._config.per_namespace_label + '="$namespace"},"id","$1","tenant","(.*)")) by(id))', regex='/"([^"]+)"/', sort=1, includeAll=true,