PMM-15248 oplatencies histogram.

exporter/fixtures_test.go

@@ -0,0 +1,118 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"os"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// diagnosticData83FixturePath is a getDiagnosticData reply captured from the MongoDB 8.3.2
+// instance of https://github.com/percona/mongodb_exporter/issues/1285, trimmed to the
+// subtrees the tests below need: serverStatus.opLatencies for the "histogram" bucket arrays
+// and serverStatus.metrics.query for the "histograms" nodes.
+const diagnosticData83FixturePath = "testdata/get_diagnostic_data_8.3.json"
+
+// loadDiagnosticData83Fixture reads the captured command reply. Extended JSON is used instead of
+// encoding/json because the driver decodes arrays into primitive.A while encoding/json
+// produces []any, which makeMetrics silently walks past. A fixture parsed the plain way
+// hides everything array shaped, histogram buckets included.
+func loadDiagnosticData83Fixture(t *testing.T) bson.M {
+	t.Helper()
+
+	buf, err := os.ReadFile(diagnosticData83FixturePath)
+	require.NoError(t, err)
+
+	var m bson.M
+	require.NoError(t, bson.UnmarshalExtJSON(buf, false, &m))
+
+	return m
+}
+
+// gatherFamilies exports the metrics the way the registry does at scrape time, so
+// duplicated series and inconsistent descriptors fail the test instead of the scrape.
+func gatherFamilies(t *testing.T, metrics []prometheus.Metric) []*dto.MetricFamily {
+	t.Helper()
+
+	reg := prometheus.NewPedanticRegistry()
+	require.NoError(t, reg.Register(staticCollector(metrics)), "descriptors must be consistent")
+
+	families, err := reg.Gather()
+	require.NoError(t, err, "metrics must be exported only once")
+
+	return families
+}
+
+// gatherMetrics exports the metrics and indexes the families by name.
+func gatherMetrics(t *testing.T, metrics []prometheus.Metric) map[string]*dto.MetricFamily {
+	t.Helper()
+
+	families := gatherFamilies(t, metrics)
+
+	metricsByName := make(map[string]*dto.MetricFamily, len(families))
+	for _, family := range families {
+		metricsByName[family.GetName()] = family
+	}
+
+	return metricsByName
+}
+
+// gatheredMetricNames returns the exported metric names, in the order Gather sorts them.
+func gatheredMetricNames(t *testing.T, metrics []prometheus.Metric) []string {
+	t.Helper()
+
+	families := gatherFamilies(t, metrics)
+
+	names := make([]string, 0, len(families))
+	for _, family := range families {
+		names = append(names, family.GetName())
+	}
+
+	return names
+}
+
+// labelValues returns the value of the given label for every series of a metric family.
+func labelValues(family *dto.MetricFamily, name string) []string {
+	values := make([]string, 0, len(family.GetMetric()))
+	for _, metric := range family.GetMetric() {
+		for _, label := range metric.GetLabel() {
+			if label.GetName() == name {
+				values = append(values, label.GetValue())
+			}
+		}
+	}
+
+	return values
+}
+
+// countsByLowerBound maps each bucket bound of a histogram family to the count of its series.
+func countsByLowerBound(family *dto.MetricFamily) map[string]float64 {
+	values := make(map[string]float64, len(family.GetMetric()))
+	for _, metric := range family.GetMetric() {
+		for _, label := range metric.GetLabel() {
+			if label.GetName() == histogramBoundLabel {
+				values[label.GetValue()] = metric.GetCounter().GetValue()
+			}
+		}
+	}
+
+	return values
+}

exporter/metrics.go

@@ -31,9 +31,17 @@ import (
 )
 
 const (
-	exporterPrefix = "mongodb_"
+	exporterPrefix         = "mongodb_"
+	histogramLowerBoundKey = "lowerBound"
+	histogramMicrosKey     = "micros"
+	histogramBoundLabel    = "lower_bound"
 )
 
+// Fields holding a histogram bucket boundary, sorted by precedence. See histogramBoundKey.
+//
+//nolint:gochecknoglobals
+var histogramBoundKeys = []string{histogramLowerBoundKey, histogramMicrosKey}
+
 type rawMetric struct {
 	// Full Qualified Name
 	fqName string
@@ -320,7 +328,7 @@ func makeMetricsWithHistograms(prefix string, m bson.M, labels map[string]string
 
 	for k, val := range m {
 		nextPrefix := prefix + k
-		if !includeHistograms && isHistogramPath(nextPrefix) {
+		if !includeHistograms && isHistogramBucketValue(nextPrefix, val) {
 			continue
 		}
 
@@ -433,18 +441,54 @@ func processHistogramSlice(prefix string, v []any, commonLabels map[string]strin
 			continue
 		}
 
+		boundKey, ok := histogramBoundKey(bucket)
+		if !ok {
+			continue
+		}
+
 		labels := make(map[string]string, len(commonLabels)+1)
 		for name, value := range commonLabels {
 			labels[name] = value
 		}
 
-		labels["lower_bound"] = fmt.Sprint(bucket["lowerBound"])
+		labels[histogramBoundLabel] = fmt.Sprint(bucket[boundKey])
 		metrics = appendMetricValue(metrics, prefix+".", "count", bucket["count"], labels, compatibleMode)
 	}
 
 	return metrics
 }
 
+// histogramBoundKey returns the field holding the bucket boundary. getDiagnosticData names it
+// two ways, both documented as the lower bound of the bucket: "lowerBound" under "histograms"
+// nodes and "micros" under the "histogram" arrays of serverStatus.opLatencies. Both are exposed
+// under histogramBoundLabel so that a query does not have to know which shape it came from.
+// The order decides which one wins should a bucket ever carry both.
+func histogramBoundKey(bucket map[string]any) (string, bool) {
+	for _, key := range histogramBoundKeys {
+		if _, ok := bucket[key]; ok {
+			return key, true
+		}
+	}
+
+	return "", false
+}
+
+// isHistogramBucketValue reports whether the value holds histogram buckets, so that
+// includeHistograms gates the buckets themselves rather than every node that happens to be
+// named "histogram" or "histograms". Only the diagnostic data collector can enable them,
+// every other collector passes includeHistograms=false; see the "latencyStats" request of
+// the collstats collector and PMM-9568.
+func isHistogramBucketValue(prefix string, val any) bool {
+	switch v := val.(type) {
+	case primitive.A:
+		return isHistogramBucketSlice(prefix, v)
+	case []any:
+		return isHistogramBucketSlice(prefix, v)
+	default:
+		return false
+	}
+}
+
 func isHistogramBucketSlice(prefix string, v []any) bool {
 	if len(v) == 0 {
 		return false
@@ -458,7 +502,7 @@ func isHistogramBucketSlice(prefix string, v []any) bool {
 		if !ok {
 			return false
 		}
-		if _, ok := bucket["lowerBound"]; !ok {
+		if _, ok := histogramBoundKey(bucket); !ok {
 			return false
 		}
 		if _, ok := bucket["count"]; !ok {
@@ -470,7 +514,11 @@ func isHistogramBucketSlice(prefix string, v []any) bool {
 }
 
 func isHistogramPath(prefix string) bool {
-	return prefix == "histograms" || strings.Contains(prefix, ".histograms.") || strings.HasSuffix(prefix, ".histograms") //nolint:goconst
+	return isPathNode(prefix, "histograms") || isPathNode(prefix, "histogram")
+}
+
+func isPathNode(prefix, node string) bool {
+	return prefix == node || strings.Contains(prefix, "."+node+".") || strings.HasSuffix(prefix, "."+node)
 }
 
 func asMetricMap(item any) (map[string]any, bool) {

exporter/metrics_histogram_fixture_test.go

@@ -0,0 +1,73 @@
+// mongodb_exporter
+// Copyright (C) 2017 Percona LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package exporter
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+// A MongoDB 8.3 reply carries both bucket shapes: the "histogram" arrays of
+// serverStatus.opLatencies and the "histograms" nodes under serverStatus.metrics.query.
+func TestServerStatusHistogramsFromFixture(t *testing.T) {
+	t.Parallel()
+
+	serverStatus, ok := loadDiagnosticData83Fixture(t)["serverStatus"].(bson.M)
+	require.True(t, ok)
+
+	labels := map[string]string{"cl_id": "", "cl_role": ""}
+	metricsByName := gatherMetrics(t, makeMetricsWithHistograms("serverStatus", serverStatus, labels, false, true))
+
+	// Every op type keeps its own bucket family.
+	for _, opType := range []string{"reads", "writes", "commands", "transactions"} {
+		assert.Contains(t, metricsByName, "mongodb_ss_opLatencies_"+opType+"_histogram_count", opType)
+	}
+
+	// Each bound keeps the count it was captured with, so a bound paired with the wrong count
+	// fails here rather than showing up as a plausible looking heatmap.
+	opLatencyBuckets, ok := metricsByName["mongodb_ss_opLatencies_reads_histogram_count"]
+	require.True(t, ok)
+	assert.Equal(t, map[string]float64{
+		"0": 0, "8": 0, "64": 16737, "512": 947, "3072": 351,
+		"8192": 77, "24576": 54, "65536": 13, "131072": 21,
+	}, countsByLowerBound(opLatencyBuckets))
+	assert.NotContains(t, metricsByName, "mongodb_ss_opLatencies_reads_histogram_micros")
+
+	plannerBuckets, ok := metricsByName["mongodb_ss_metrics_query_multiPlanner_histograms_classicWorks_count"]
+	require.True(t, ok)
+	assert.ElementsMatch(t, []string{"0", "128", "256", "512", "1024", "2048", "4096", "8192", "16384", "32768"},
+		labelValues(plannerBuckets, "lower_bound"))
+}
+
+func TestServerStatusHistogramsFromFixtureAreSkippedByDefault(t *testing.T) {
+	t.Parallel()
+
+	serverStatus, ok := loadDiagnosticData83Fixture(t)["serverStatus"].(bson.M)
+	require.True(t, ok)
+
+	labels := map[string]string{"cl_id": "", "cl_role": ""}
+	metricsByName := gatherMetrics(t, makeMetrics("serverStatus", serverStatus, labels, false))
+
+	for name := range metricsByName {
+		assert.NotContains(t, name, "histogram")
+	}
+
+	// Fields next to the buckets are still collected.
+	assert.Contains(t, metricsByName, "mongodb_ss_opLatencies_latency")
+}