From 7d19cc2fd11f256bc63bba4551753ffa5149cee1 Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Thu, 16 Jul 2026 15:39:52 -0500 Subject: [PATCH 1/7] init otel metrics scraper --- .../observability/02-cluster-role.yaml | 7 +++- .../opentelemetry-collector-large.yaml | 2 + .../opentelemetry-collector-medium.yaml | 8 ++++ .../opentelemetry-collector-small.yaml | 8 ++++ .../scrape.d/kube-state-metrics.yaml | 37 +++++++++++++++++++ .../scrape.d/metrics-server.yaml | 19 ++++++++++ .../observability/scrape.d/node-exporter.yaml | 19 ++++++++++ packaging/rpm/microshift.spec | 14 +++++++ test/assets/observability/otel_config.yaml | 8 ++++ test/suites/optional/observability.robot | 29 ++++++++++++++- 10 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 packaging/observability/scrape.d/kube-state-metrics.yaml create mode 100644 packaging/observability/scrape.d/metrics-server.yaml create mode 100644 packaging/observability/scrape.d/node-exporter.yaml diff --git a/assets/optional/observability/02-cluster-role.yaml b/assets/optional/observability/02-cluster-role.yaml index 17b8723acd..d9b83d9635 100644 --- a/assets/optional/observability/02-cluster-role.yaml +++ b/assets/optional/observability/02-cluster-role.yaml @@ -63,4 +63,9 @@ rules: verbs: - get - list - - watch \ No newline at end of file + - watch + # OpenTelemetry Component: prometheus receiver scraping metrics-server + - nonResourceURLs: + - /metrics + verbs: + - get \ No newline at end of file diff --git a/packaging/observability/opentelemetry-collector-large.yaml b/packaging/observability/opentelemetry-collector-large.yaml index 208dd4932c..6131ac5667 100644 --- a/packaging/observability/opentelemetry-collector-large.yaml +++ b/packaging/observability/opentelemetry-collector-large.yaml @@ -40,6 +40,8 @@ receivers: priority: info prometheus: config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml scrape_configs: - job_name: k8s scrape_interval: 10s diff --git a/packaging/observability/opentelemetry-collector-medium.yaml b/packaging/observability/opentelemetry-collector-medium.yaml index cc2b6f1677..56d21543c5 100644 --- a/packaging/observability/opentelemetry-collector-medium.yaml +++ b/packaging/observability/opentelemetry-collector-medium.yaml @@ -4,6 +4,10 @@ # - System journal for MicroShift service, priority < Info receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml kubeletstats: auth_type: tls ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt @@ -48,6 +52,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ batch ] + exporters: [ otlp ] metrics/kubeletstats: receivers: [ kubeletstats ] processors: [ batch ] diff --git a/packaging/observability/opentelemetry-collector-small.yaml b/packaging/observability/opentelemetry-collector-small.yaml index 588ea38707..88400e5d7d 100644 --- a/packaging/observability/opentelemetry-collector-small.yaml +++ b/packaging/observability/opentelemetry-collector-small.yaml @@ -3,6 +3,10 @@ # - Kubernetes Events receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml kubeletstats: auth_type: tls ca_file: /var/lib/microshift/certs/ca-bundle/client-ca.crt @@ -43,6 +47,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ batch ] + exporters: [ otlp ] metrics/kubeletstats: receivers: [ kubeletstats ] processors: [ batch ] diff --git a/packaging/observability/scrape.d/kube-state-metrics.yaml b/packaging/observability/scrape.d/kube-state-metrics.yaml new file mode 100644 index 0000000000..5840d6b2aa --- /dev/null +++ b/packaging/observability/scrape.d/kube-state-metrics.yaml @@ -0,0 +1,37 @@ +scrape_configs: + - job_name: kube-state-metrics + scrape_interval: 30s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: kube-state-metrics.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kube-state-metrics;https-main + - job_name: kube-state-metrics-self + scrape_interval: 30s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: kube-state-metrics.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kube-state-metrics;https-self diff --git a/packaging/observability/scrape.d/metrics-server.yaml b/packaging/observability/scrape.d/metrics-server.yaml new file mode 100644 index 0000000000..b6212dc239 --- /dev/null +++ b/packaging/observability/scrape.d/metrics-server.yaml @@ -0,0 +1,19 @@ +scrape_configs: + - job_name: metrics-server + scrape_interval: 30s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: metrics-server.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: metrics-server;https diff --git a/packaging/observability/scrape.d/node-exporter.yaml b/packaging/observability/scrape.d/node-exporter.yaml new file mode 100644 index 0000000000..4265813dbd --- /dev/null +++ b/packaging/observability/scrape.d/node-exporter.yaml @@ -0,0 +1,19 @@ +scrape_configs: + - job_name: node-exporter + scrape_interval: 30s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + cert_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.crt + key_file: /var/lib/microshift/certs/admin-kubeconfig-signer/openshift-observability-client/client.key + server_name: node-exporter.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: + - openshift-monitoring + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: node-exporter;https diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 0dbbc931a6..5aa6a9b564 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -623,6 +623,7 @@ install -p -m644 assets/optional/ai-model-serving/release-ai-model-serving-x86_6 # observability install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d install -p -m644 packaging/observability/*.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/ # Explicit copy of large config as default. Not using symlink to avoid accidental package upgrade overwriting user config if the user edits the config without copying (i.e. edits the target of symlink). install -p -m644 packaging/observability/opentelemetry-collector-large.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml @@ -681,6 +682,9 @@ cat assets/optional/metrics-server/kustomization.aarch64.yaml >> %{buildroot}/%{ cat assets/optional/metrics-server/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/metrics-server.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # metrics-server-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/metrics-server/release-metrics-server-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -705,6 +709,9 @@ cat assets/optional/node-exporter/kustomization.aarch64.yaml >> %{buildroot}/%{_ cat assets/optional/node-exporter/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/node-exporter.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # node-exporter-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/node-exporter/release-node-exporter-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -728,6 +735,9 @@ cat assets/optional/kube-state-metrics/kustomization.aarch64.yaml >> %{buildroot cat assets/optional/kube-state-metrics/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml %endif +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d +install -p -m644 packaging/observability/scrape.d/kube-state-metrics.yaml %{buildroot}/%{_sysconfdir}/microshift/observability/scrape.d/ + # kube-state-metrics-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/kube-state-metrics/release-kube-state-metrics-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ @@ -923,6 +933,7 @@ fi %files observability %dir %{_prefix}/lib/microshift/manifests.d/003-microshift-observability %dir %{_sysconfdir}/microshift/observability/ +%dir %{_sysconfdir}/microshift/observability/scrape.d/ %{_unitdir}/microshift-observability.service %config(noreplace) %{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml %{_sysconfdir}/microshift/observability/opentelemetry-collector-*.yaml @@ -938,6 +949,7 @@ fi %files metrics-server %dir %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/* +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/metrics-server.yaml %files metrics-server-release-info %{_datadir}/microshift/release/release-metrics-server-{x86_64,aarch64}.json @@ -945,12 +957,14 @@ fi %files metrics-node-exporter %dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/* +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/node-exporter.yaml %files metrics-node-exporter-release-info %{_datadir}/microshift/release/release-node-exporter-{x86_64,aarch64}.json %files metrics-kube-state %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* +%config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/kube-state-metrics.yaml %files metrics-kube-state-release-info %{_datadir}/microshift/release/release-kube-state-metrics-{x86_64,aarch64}.json diff --git a/test/assets/observability/otel_config.yaml b/test/assets/observability/otel_config.yaml index 4565f82077..9d3763eae3 100644 --- a/test/assets/observability/otel_config.yaml +++ b/test/assets/observability/otel_config.yaml @@ -1,5 +1,9 @@ # This OTEL config is only for testing purposes receivers: + prometheus: + config: + scrape_config_files: + - /etc/microshift/observability/scrape.d/*.yaml hostmetrics: root_path: / collection_interval: 10s @@ -67,6 +71,10 @@ extensions: service: extensions: [ file_storage ] pipelines: + metrics/scrape: + receivers: [ prometheus ] + processors: [ resourcedetection/system ] + exporters: [ prometheus, prometheusremotewrite ] metrics/hostmetrics: receivers: [ hostmetrics ] processors: [ resourcedetection/system ] diff --git a/test/suites/optional/observability.robot b/test/suites/optional/observability.robot index d1ffb51be0..718f798ef1 100644 --- a/test/suites/optional/observability.robot +++ b/test/suites/optional/observability.robot @@ -57,6 +57,29 @@ Kube Events Logs Are Exported Wait Until Keyword Succeeds 10x 5s ... Check Loki Query ${LOKI_HOST} ${LOKI_PORT} {service_name="kube_events"} +KSM Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape kube-state-metrics via the + ... scrape.d drop-in and export kube_node_info to the prometheus exporter. + + Wait Until Keyword Succeeds 60s 5s + ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} kube_node_info + +Node Exporter Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape node-exporter via the + ... scrape.d drop-in and export node_cpu_seconds_total to the prometheus exporter. + + Wait Until Keyword Succeeds 60s 5s + ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} node_cpu_seconds_total + +Metrics Server Metrics Are Exported Via Scrape Drop-In + [Documentation] The prometheus receiver should scrape metrics-server via the + ... scrape.d drop-in and export metrics to the prometheus exporter. + ... TODO: replace rest_client_requests_total with a metrics-server-specific metric + ... once confirmed against actual metrics-server /metrics output. + + Wait Until Keyword Succeeds 60s 5s + ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} rest_client_requests_total + Logs Should Not Contain Receiver Errors [Documentation] Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration ... and thus indicate the provided default config should be reviewed. @@ -71,7 +94,11 @@ Setup Suite And Prepare Test Host [Documentation] The service starts after MicroShift starts and thus will start generating pertinent log data ... right away. When the suite is executed, immediately get the cursor for the microshift-observability unit. Setup Suite - Setup MicroShift With Optionals 003-microshift-observability + Setup MicroShift With Optionals + ... 003-microshift-observability + ... 080-microshift-metrics-server + ... 081-microshift-kube-state-metrics + ... 082-microshift-node-exporter ${ns} Create Unique Namespace VAR ${NAMESPACE} ${ns} scope=SUITE Configure Firewall And Observability From beddc8da42a79c8287bd0cea94cf1739a0c12f6a Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Thu, 16 Jul 2026 21:17:26 -0500 Subject: [PATCH 2/7] address review findings: fix ms metric name, add scrape.d dir ownership, update profile comments --- packaging/observability/opentelemetry-collector-large.yaml | 1 + packaging/observability/opentelemetry-collector-medium.yaml | 1 + packaging/observability/opentelemetry-collector-small.yaml | 1 + packaging/rpm/microshift.spec | 3 +++ test/suites/optional/observability.robot | 4 +--- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packaging/observability/opentelemetry-collector-large.yaml b/packaging/observability/opentelemetry-collector-large.yaml index 6131ac5667..49fa54d2f3 100644 --- a/packaging/observability/opentelemetry-collector-large.yaml +++ b/packaging/observability/opentelemetry-collector-large.yaml @@ -4,6 +4,7 @@ # - Host's CPU, memory, disk, and network usage metrics # - System journals for selected MicroShift services and dependencies, priority < Info # - Metrics exposed by Pods that have "prometheus.io/scrape": "true" annotation +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: kubeletstats: diff --git a/packaging/observability/opentelemetry-collector-medium.yaml b/packaging/observability/opentelemetry-collector-medium.yaml index 56d21543c5..af1950bd08 100644 --- a/packaging/observability/opentelemetry-collector-medium.yaml +++ b/packaging/observability/opentelemetry-collector-medium.yaml @@ -2,6 +2,7 @@ # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node # - Kubernetes Events # - System journal for MicroShift service, priority < Info +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: prometheus: diff --git a/packaging/observability/opentelemetry-collector-small.yaml b/packaging/observability/opentelemetry-collector-small.yaml index 88400e5d7d..6f96b8ea40 100644 --- a/packaging/observability/opentelemetry-collector-small.yaml +++ b/packaging/observability/opentelemetry-collector-small.yaml @@ -1,6 +1,7 @@ # Following file contains OpenTelemetry Collector configuration that exports: # - Resource usage metrics of MicroShift's Containers, Pods, Volumes, and Node # - Kubernetes Events +# - Metrics from optional component scrape configs (scrape.d drop-ins) receivers: prometheus: diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 5aa6a9b564..710a57f041 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -949,6 +949,7 @@ fi %files metrics-server %dir %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/* +%dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/metrics-server.yaml %files metrics-server-release-info @@ -957,6 +958,7 @@ fi %files metrics-node-exporter %dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/* +%dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/node-exporter.yaml %files metrics-node-exporter-release-info @@ -964,6 +966,7 @@ fi %files metrics-kube-state %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* +%dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/kube-state-metrics.yaml %files metrics-kube-state-release-info diff --git a/test/suites/optional/observability.robot b/test/suites/optional/observability.robot index 718f798ef1..72b1bd2294 100644 --- a/test/suites/optional/observability.robot +++ b/test/suites/optional/observability.robot @@ -74,11 +74,9 @@ Node Exporter Metrics Are Exported Via Scrape Drop-In Metrics Server Metrics Are Exported Via Scrape Drop-In [Documentation] The prometheus receiver should scrape metrics-server via the ... scrape.d drop-in and export metrics to the prometheus exporter. - ... TODO: replace rest_client_requests_total with a metrics-server-specific metric - ... once confirmed against actual metrics-server /metrics output. Wait Until Keyword Succeeds 60s 5s - ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} rest_client_requests_total + ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} metrics_server_kubelet_request_total Logs Should Not Contain Receiver Errors [Documentation] Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration From 55e466904be75e48ac3343b3f62c78eccc31d49c Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Thu, 16 Jul 2026 22:54:57 -0500 Subject: [PATCH 3/7] address review findings: add endpoints RBAC, fix comments and pipeline name The prometheus receiver's kubernetes_sd role:endpoints requires the observability-client service account to list/watch endpoints. Without this permission all scrape.d drop-in configs fail with a 403. Also fixes a misleading comment on the nonResourceURLs /metrics rule, renames the large profile pipeline from metrics/pods to metrics/scrape for consistency with the other profiles, and adds missing blank lines between RPM spec sections. Co-Authored-By: Claude Opus 4.6 --- assets/optional/observability/02-cluster-role.yaml | 5 +++-- packaging/observability/opentelemetry-collector-large.yaml | 2 +- packaging/rpm/microshift.spec | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/assets/optional/observability/02-cluster-role.yaml b/assets/optional/observability/02-cluster-role.yaml index d9b83d9635..55472a81b7 100644 --- a/assets/optional/observability/02-cluster-role.yaml +++ b/assets/optional/observability/02-cluster-role.yaml @@ -7,10 +7,11 @@ rules: - apiGroups: [""] resources: ["nodes/stats"] verbs: ["get"] - # OpenTelemetry Component: k8s_event receiver + # OpenTelemetry Component: k8s_event receiver, prometheus receiver (endpoints SD) - apiGroups: - "" resources: + - endpoints - events - namespaces - namespaces/status @@ -64,7 +65,7 @@ rules: - get - list - watch - # OpenTelemetry Component: prometheus receiver scraping metrics-server + # OpenTelemetry Component: prometheus receiver (kube-rbac-proxy authorization) - nonResourceURLs: - /metrics verbs: diff --git a/packaging/observability/opentelemetry-collector-large.yaml b/packaging/observability/opentelemetry-collector-large.yaml index 49fa54d2f3..3757ae804e 100644 --- a/packaging/observability/opentelemetry-collector-large.yaml +++ b/packaging/observability/opentelemetry-collector-large.yaml @@ -112,7 +112,7 @@ service: receivers: [ journald ] processors: [ resourcedetection/system ] exporters: [ otlp ] - metrics/pods: + metrics/scrape: receivers: [ prometheus ] processors: [ batch ] exporters: [ otlp ] diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 710a57f041..cdbb9ed565 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -303,6 +303,7 @@ Requires: microshift-release-info = %{version} The microshift-metrics-node-exporter-release-info package provides release information files for this release. These files contain the list of container image references used by node-exporter and can be used to embed those images into osbuilder blueprints or bootc containerfiles. + %package metrics-kube-state Summary: Kubernetes kube-state-metrics for MicroShift ExclusiveArch: x86_64 aarch64 @@ -963,6 +964,7 @@ fi %files metrics-node-exporter-release-info %{_datadir}/microshift/release/release-node-exporter-{x86_64,aarch64}.json + %files metrics-kube-state %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* From c0a5a7a5e2d080728f18b11560c44255af31692a Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Thu, 16 Jul 2026 23:08:45 -0500 Subject: [PATCH 4/7] fix orphaned observability dir when metrics packages installed standalone Metrics subpackages use Recommends (not Requires) for microshift-observability, so they can be installed without it. Without owning the parent /etc/microshift/observability directory, rpm -e leaves an empty orphaned directory behind. Add %dir ownership so the full tree is cleaned up on removal. Co-Authored-By: Claude Opus 4.6 --- packaging/rpm/microshift.spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index cdbb9ed565..960d5769f7 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -950,6 +950,7 @@ fi %files metrics-server %dir %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server %{_prefix}/lib/microshift/manifests.d/080-microshift-metrics-server/* +%dir %{_sysconfdir}/microshift/observability %dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/metrics-server.yaml @@ -959,6 +960,7 @@ fi %files metrics-node-exporter %dir %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter %{_prefix}/lib/microshift/manifests.d/082-microshift-node-exporter/* +%dir %{_sysconfdir}/microshift/observability %dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/node-exporter.yaml @@ -968,6 +970,7 @@ fi %files metrics-kube-state %dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* +%dir %{_sysconfdir}/microshift/observability %dir %{_sysconfdir}/microshift/observability/scrape.d %config(noreplace) %{_sysconfdir}/microshift/observability/scrape.d/kube-state-metrics.yaml From 10f8b7bd9c8111a71d77554e988a9f82569a087a Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Fri, 17 Jul 2026 11:20:45 -0500 Subject: [PATCH 5/7] remove IPv4-only regex from metrics scrape test The Scrape Metrics From keyword validated the endpoint IP with an IPv4-only regex, which would reject IPv6 addresses and break IPv6-only CI. The jsonpath already returns a valid IP string from the Kubernetes API; the regex was unnecessary. Co-Authored-By: Claude Opus 4.6 Co-Authored-By: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- test/suites/optional/metrics.robot | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/suites/optional/metrics.robot b/test/suites/optional/metrics.robot index 68cfdef75a..96bd3d1cca 100644 --- a/test/suites/optional/metrics.robot +++ b/test/suites/optional/metrics.robot @@ -114,9 +114,6 @@ Scrape Metrics From ... ${METRICS_NS} ... ${metrics_service} ... .subsets[0].addresses[0].ip - ${ep_ip}= Get Regexp Matches ${ep_ip} \\d+\\.\\d+\\.\\d+\\.\\d+ - Length Should Be ${ep_ip} 1 - VAR ${ep_ip}= ${ep_ip}[0] VAR ${svc_host}= ${metrics_service}.${METRICS_NS}.svc ${stdout}= Command Should Work ... curl -s --resolve ${svc_host}:${port}:${ep_ip} --cacert ${SERVICE_CA} --cert ${CLIENT_CERT} --key ${CLIENT_KEY} https://${svc_host}:${port}/metrics From 7f9520c8d1b3c52e438088565315b36759902ac3 Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Fri, 17 Jul 2026 14:57:01 -0500 Subject: [PATCH 6/7] fix metrics test: avoid stderr pollution from oc get endpoints Run With Kubeconfig merges stderr into stdout. In Kubernetes 1.33+, oc get endpoints emits a deprecation warning to stderr that gets prepended to the IP address, breaking the curl --resolve argument. Switch to Command Should Work which separates stdout from stderr. --- test/suites/optional/metrics.robot | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/suites/optional/metrics.robot b/test/suites/optional/metrics.robot index 96bd3d1cca..e172c13508 100644 --- a/test/suites/optional/metrics.robot +++ b/test/suites/optional/metrics.robot @@ -109,11 +109,10 @@ Scrape Metrics From ... with mTLS client certs. Resolves the IP from the Endpoints object ... to validate that the headless Service selector matches the pod. [Arguments] ${metrics_service} ${port} - ${ep_ip}= Oc Get JsonPath - ... endpoints - ... ${METRICS_NS} - ... ${metrics_service} - ... .subsets[0].addresses[0].ip + # Oc Get JsonPath uses Run With Kubeconfig which merges stderr into stdout; + # the v1 Endpoints deprecation warning from K8s 1.33+ pollutes the IP value. + ${ep_ip}= Command Should Work + ... oc get endpoints -n ${METRICS_NS} ${metrics_service} -o jsonpath\='{.subsets[0].addresses[0].ip}' --kubeconfig\=/var/lib/microshift/resources/kubeadmin/kubeconfig VAR ${svc_host}= ${metrics_service}.${METRICS_NS}.svc ${stdout}= Command Should Work ... curl -s --resolve ${svc_host}:${port}:${ep_ip} --cacert ${SERVICE_CA} --cert ${CLIENT_CERT} --key ${CLIENT_KEY} https://${svc_host}:${port}/metrics From 405a7ce758c3da9342b8c94d16c41cae54a88d95 Mon Sep 17 00:00:00 2001 From: "Jonathan H. Cope" Date: Mon, 20 Jul 2026 10:34:25 -0500 Subject: [PATCH 7/7] address review findings: align scrape interval, add PromQL query checks Align scrape.d drop-in scrape_interval from 30s to 10s for consistency with the collector config profiles. Add Check Prometheus Query alongside Check Prometheus Exporter in the scrape drop-in test cases to match the pattern used by the other observability tests. Co-Authored-By: Claude Opus 4.6 --- .../scrape.d/kube-state-metrics.yaml | 4 ++-- .../observability/scrape.d/metrics-server.yaml | 2 +- .../observability/scrape.d/node-exporter.yaml | 2 +- test/suites/optional/observability.robot | 15 +++++++++------ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/packaging/observability/scrape.d/kube-state-metrics.yaml b/packaging/observability/scrape.d/kube-state-metrics.yaml index 5840d6b2aa..06a23f770c 100644 --- a/packaging/observability/scrape.d/kube-state-metrics.yaml +++ b/packaging/observability/scrape.d/kube-state-metrics.yaml @@ -1,6 +1,6 @@ scrape_configs: - job_name: kube-state-metrics - scrape_interval: 30s + scrape_interval: 10s scheme: https tls_config: ca_file: /var/lib/microshift/certs/service-ca/ca.crt @@ -18,7 +18,7 @@ scrape_configs: action: keep regex: kube-state-metrics;https-main - job_name: kube-state-metrics-self - scrape_interval: 30s + scrape_interval: 10s scheme: https tls_config: ca_file: /var/lib/microshift/certs/service-ca/ca.crt diff --git a/packaging/observability/scrape.d/metrics-server.yaml b/packaging/observability/scrape.d/metrics-server.yaml index b6212dc239..b653dac0c3 100644 --- a/packaging/observability/scrape.d/metrics-server.yaml +++ b/packaging/observability/scrape.d/metrics-server.yaml @@ -1,6 +1,6 @@ scrape_configs: - job_name: metrics-server - scrape_interval: 30s + scrape_interval: 10s scheme: https tls_config: ca_file: /var/lib/microshift/certs/service-ca/ca.crt diff --git a/packaging/observability/scrape.d/node-exporter.yaml b/packaging/observability/scrape.d/node-exporter.yaml index 4265813dbd..66626da522 100644 --- a/packaging/observability/scrape.d/node-exporter.yaml +++ b/packaging/observability/scrape.d/node-exporter.yaml @@ -1,6 +1,6 @@ scrape_configs: - job_name: node-exporter - scrape_interval: 30s + scrape_interval: 10s scheme: https tls_config: ca_file: /var/lib/microshift/certs/service-ca/ca.crt diff --git a/test/suites/optional/observability.robot b/test/suites/optional/observability.robot index 72b1bd2294..5952a6113a 100644 --- a/test/suites/optional/observability.robot +++ b/test/suites/optional/observability.robot @@ -61,22 +61,25 @@ KSM Metrics Are Exported Via Scrape Drop-In [Documentation] The prometheus receiver should scrape kube-state-metrics via the ... scrape.d drop-in and export kube_node_info to the prometheus exporter. - Wait Until Keyword Succeeds 60s 5s - ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} kube_node_info + VAR ${METRIC} kube_node_info scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} Node Exporter Metrics Are Exported Via Scrape Drop-In [Documentation] The prometheus receiver should scrape node-exporter via the ... scrape.d drop-in and export node_cpu_seconds_total to the prometheus exporter. - Wait Until Keyword Succeeds 60s 5s - ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} node_cpu_seconds_total + VAR ${METRIC} node_cpu_seconds_total scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} Metrics Server Metrics Are Exported Via Scrape Drop-In [Documentation] The prometheus receiver should scrape metrics-server via the ... scrape.d drop-in and export metrics to the prometheus exporter. - Wait Until Keyword Succeeds 60s 5s - ... Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} metrics_server_kubelet_request_total + VAR ${METRIC} metrics_server_kubelet_request_total scope=TEST + Check Prometheus Query ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} ${METRIC} + Check Prometheus Exporter ${USHIFT_HOST} ${PROM_EXPORTER_PORT} ${METRIC} Logs Should Not Contain Receiver Errors [Documentation] Internal receiver errors are not treated as fatal. Typically these are due to a misconfiguration