From 41821e3b6e5641f687dbf932a262cd537e2aeef8 Mon Sep 17 00:00:00 2001
From: Thomas Jungblut <tjungblu@redhat.com>
Date: Thu, 9 Oct 2025 14:36:43 +0200
Subject: [PATCH 1/3] NO-JIRA: add recommended etcd threshold for alerts

---
 jsonnet/custom.libsonnet                      | 15 ++++++++-
 jsonnet/jsonnetfile.lock.json                 |  2 +-
 jsonnet/main.jsonnet                          |  2 +-
 ...00_90_etcd-operator_03_prometheusrule.yaml | 33 ++++++-------------
 4 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/jsonnet/custom.libsonnet b/jsonnet/custom.libsonnet
index 5cd4cb1d75..2527dbfe9a 100644
--- a/jsonnet/custom.libsonnet
+++ b/jsonnet/custom.libsonnet
@@ -61,7 +61,7 @@
           },
           {
             alert: 'etcdHighCommitDurations',
-            expr: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.5',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.025',
             'for': '10m',
             labels: {
               severity: 'warning',
@@ -71,6 +71,19 @@
               summary: 'etcd cluster 99th percentile commit durations are too high.',
             },
           },
+          {
+            alert: 'etcdHighFsyncDurations',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.010',
+            'for': '10m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
+              summary: 'etcd cluster 99th percentile fsync durations are too high.',
+              runbook_url: 'https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md'
+            },
+          },
           {
             alert: 'etcdHighNumberOfFailedGRPCRequests',
             expr: |||
diff --git a/jsonnet/jsonnetfile.lock.json b/jsonnet/jsonnetfile.lock.json
index 3ec8200a61..b8357e07ca 100644
--- a/jsonnet/jsonnetfile.lock.json
+++ b/jsonnet/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "e4d6a05f8f1ec972384e24a83c420f707a6644f2",
+      "version": "48f735d13e83dea5b1901ea627eddd75d05ee8d5",
       "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
diff --git a/jsonnet/main.jsonnet b/jsonnet/main.jsonnet
index 7e1ed0b6c8..69abf1cc29 100644
--- a/jsonnet/main.jsonnet
+++ b/jsonnet/main.jsonnet
@@ -6,7 +6,7 @@ local promRules = if std.objectHasAll(etcdMixin, 'prometheusRules') then etcdMix
 
 // Exclude rules that are either OpenShift specific or do not work for OpenShift.
 // List should be ordered!
-local excludedAlerts = ['etcdDatabaseQuotaLowSpace', 'etcdGRPCRequestsSlow', 'etcdHighCommitDurations', 'etcdHighNumberOfFailedGRPCRequests', 'etcdHighNumberOfLeaderChanges', 'etcdInsufficientMembers', 'etcdMembersDown'];
+local excludedAlerts = ['etcdDatabaseQuotaLowSpace', 'etcdGRPCRequestsSlow', 'etcdHighCommitDurations', 'etcdHighFsyncDurations', 'etcdHighNumberOfFailedGRPCRequests', 'etcdHighNumberOfLeaderChanges', 'etcdInsufficientMembers', 'etcdMembersDown'];
 local excludeRules = std.map(
   function(group) group {
     rules: std.filter(
diff --git a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
index 1db3073044..40f50d3f73 100644
--- a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
+++ b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
@@ -39,28 +39,6 @@ spec:
       for: 15m
       labels:
         severity: warning
-    - alert: etcdHighFsyncDurations
-      annotations:
-        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
-        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
-        summary: etcd cluster 99th percentile fsync durations are too high.
-      expr: |
-        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
-        > 0.5
-      for: 10m
-      labels:
-        severity: warning
-    - alert: etcdHighFsyncDurations
-      annotations:
-        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
-        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
-        summary: etcd cluster 99th percentile fsync durations are too high.
-      expr: |
-        histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m]))
-        > 1
-      for: 10m
-      labels:
-        severity: critical
     - alert: etcdExcessiveDatabaseGrowth
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": Predicting running out of disk space in the next four hours, based on write observations within the past four hours on etcd instance {{ $labels.instance }}, please check as it might be disruptive.'
@@ -128,10 +106,19 @@ spec:
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
         summary: etcd cluster 99th percentile commit durations are too high.
-      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.5
+      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.025
       for: 10m
       labels:
         severity: warning
+    - alert: etcdHighFsyncDurations
+      annotations:
+        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
+        summary: etcd cluster 99th percentile fsync durations are too high.
+      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.010
+      for: 10m
+      labels:
+        severity: critical
     - alert: etcdHighNumberOfFailedGRPCRequests
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": {{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}.'

From 1d496e6dc07cf4fbfbf73d32b6049a0fc902d9ef Mon Sep 17 00:00:00 2001
From: Devan Goodwin <dgoodwin@redhat.com>
Date: Thu, 6 Nov 2025 14:16:58 -0400
Subject: [PATCH 2/3] Update values for observed fleet and CI data

---
 jsonnet/custom.libsonnet                               | 4 ++--
 jsonnet/jsonnetfile.lock.json                          | 2 +-
 manifests/0000_90_etcd-operator_03_prometheusrule.yaml | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/jsonnet/custom.libsonnet b/jsonnet/custom.libsonnet
index 2527dbfe9a..e8ca14c601 100644
--- a/jsonnet/custom.libsonnet
+++ b/jsonnet/custom.libsonnet
@@ -61,7 +61,7 @@
           },
           {
             alert: 'etcdHighCommitDurations',
-            expr: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.025',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.08',
             'for': '10m',
             labels: {
               severity: 'warning',
@@ -73,7 +73,7 @@
           },
           {
             alert: 'etcdHighFsyncDurations',
-            expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.010',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.05',
             'for': '10m',
             labels: {
               severity: 'critical',
diff --git a/jsonnet/jsonnetfile.lock.json b/jsonnet/jsonnetfile.lock.json
index b8357e07ca..1b83e9b82a 100644
--- a/jsonnet/jsonnetfile.lock.json
+++ b/jsonnet/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "48f735d13e83dea5b1901ea627eddd75d05ee8d5",
+      "version": "9f1883643231ae949be4019f7f9a6ec07d2c34d1",
       "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
diff --git a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
index 40f50d3f73..e33227aa88 100644
--- a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
+++ b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
@@ -106,7 +106,7 @@ spec:
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
         summary: etcd cluster 99th percentile commit durations are too high.
-      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.025
+      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.08
       for: 10m
       labels:
         severity: warning
@@ -115,7 +115,7 @@ spec:
         description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
         runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
         summary: etcd cluster 99th percentile fsync durations are too high.
-      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.010
+      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.05
       for: 10m
       labels:
         severity: critical

From 90ac16ff43efd887f91f38264ca692d989066036 Mon Sep 17 00:00:00 2001
From: Devan Goodwin <dgoodwin@redhat.com>
Date: Fri, 7 Nov 2025 09:19:01 -0400
Subject: [PATCH 3/3] Reinstate warning levels for commit duration and wal
 fsync alerts

---
 jsonnet/custom.libsonnet                      | 25 +++++++++++++++++++
 jsonnet/jsonnetfile.lock.json                 |  2 +-
 ...00_90_etcd-operator_03_prometheusrule.yaml | 19 ++++++++++++++
 3 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/jsonnet/custom.libsonnet b/jsonnet/custom.libsonnet
index e8ca14c601..473636f8e1 100644
--- a/jsonnet/custom.libsonnet
+++ b/jsonnet/custom.libsonnet
@@ -71,10 +71,35 @@
               summary: 'etcd cluster 99th percentile commit durations are too high.',
             },
           },
+          {
+            alert: 'etcdHighCommitDurations',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.10',
+            'for': '10m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.',
+              summary: 'etcd cluster 99th percentile commit durations are too high.',
+            },
+          },
           {
             alert: 'etcdHighFsyncDurations',
             expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.05',
             'for': '10m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.',
+              summary: 'etcd cluster 99th percentile fsync durations are too high.',
+              runbook_url: 'https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md'
+            },
+          },
+          {
+            alert: 'etcdHighFsyncDurations',
+            expr: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.07',
+            'for': '10m',
             labels: {
               severity: 'critical',
             },
diff --git a/jsonnet/jsonnetfile.lock.json b/jsonnet/jsonnetfile.lock.json
index 1b83e9b82a..d9f4e1aea3 100644
--- a/jsonnet/jsonnetfile.lock.json
+++ b/jsonnet/jsonnetfile.lock.json
@@ -8,7 +8,7 @@
           "subdir": "contrib/mixin"
         }
       },
-      "version": "9f1883643231ae949be4019f7f9a6ec07d2c34d1",
+      "version": "c9184ab8b1264819ee39bd5e664a94f02a358882",
       "sum": "XmXkOCriQIZmXwlIIFhqlJMa0e6qGWdxZD+ZDYaN0Po="
     },
     {
diff --git a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
index e33227aa88..1b379b0715 100644
--- a/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
+++ b/manifests/0000_90_etcd-operator_03_prometheusrule.yaml
@@ -105,11 +105,21 @@ spec:
     - alert: etcdHighCommitDurations
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighCommitDurations.md
         summary: etcd cluster 99th percentile commit durations are too high.
       expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.08
       for: 10m
       labels:
         severity: warning
+    - alert: etcdHighCommitDurations
+      annotations:
+        description: 'etcd cluster "{{ $labels.job }}": 99th percentile commit durations {{ $value }}s on etcd instance {{ $labels.instance }}.'
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighCommitDurations.md
+        summary: etcd cluster 99th percentile commit durations are too high.
+      expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.10
+      for: 10m
+      labels:
+        severity: critical
     - alert: etcdHighFsyncDurations
       annotations:
         description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
@@ -117,6 +127,15 @@ spec:
         summary: etcd cluster 99th percentile fsync durations are too high.
       expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.05
       for: 10m
+      labels:
+        severity: warning
+    - alert: etcdHighFsyncDurations
+      annotations:
+        description: 'etcd cluster "{{ $labels.job }}": 99th percentile fsync durations are {{ $value }}s on etcd instance {{ $labels.instance }}.'
+        runbook_url: https://github.com/openshift/runbooks/blob/master/alerts/cluster-etcd-operator/etcdHighFsyncDurations.md
+        summary: etcd cluster 99th percentile fsync durations are too high.
+      expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=~".*etcd.*"}[5m])) > 0.07
+      for: 10m
       labels:
         severity: critical
     - alert: etcdHighNumberOfFailedGRPCRequests