From e848afb9dcd0a79b61f9561965ae14fbc3f25095 Mon Sep 17 00:00:00 2001 From: Timothy Coleman Date: Thu, 11 Jun 2026 13:40:27 +0000 Subject: [PATCH] Report disk usage metrics for index and log drives The sys-disk metric only reported total/used bytes for the database drive. Extend it to also report the index and log drives when they live on different physical drives. CreateDiskMetric now takes a list of paths and a drive-info resolver. It resolves each path to its drive and registers one total/used pair per distinct drive, deduplicating by disk name so paths sharing a mount (the common default layout) don't emit duplicate (kind, disk) series. When everything is on one drive the output is unchanged. The index-path expression is computed once in ClusterVNode and reused. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Metrics/SystemMetricsTests.cs | 69 ++++++++++++++++++- src/KurrentDB.Core/ClusterVNode.cs | 4 +- src/KurrentDB.Core/Metrics/SystemMetrics.cs | 20 ++++-- src/KurrentDB.Core/MetricsBootstrapper.cs | 5 +- 4 files changed, 88 insertions(+), 10 deletions(-) diff --git a/src/KurrentDB.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs b/src/KurrentDB.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs index 9b1f8588bd1..4d858de0000 100644 --- a/src/KurrentDB.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs +++ b/src/KurrentDB.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs @@ -3,7 +3,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.Metrics; +using System.Linq; using System.Runtime; using FluentAssertions; using KurrentDB.Common.Configuration; @@ -45,7 +47,7 @@ public SystemMetricsTests() { { MetricsConfiguration.SystemTracker.TotalMem, "total" }, }); - _sut.CreateDiskMetric("eventstore-sys-disk", ".", new() { + _sut.CreateDiskMetric("eventstore-sys-disk", ["."], DriveStats.GetDriveInfo, new() { { MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" }, { MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" }, }); @@ -148,6 +150,71 @@ public void can_collect_sys_mem() { }); } + private static SystemMetrics CreateSystemMetrics(Meter meter) { + var config = new Dictionary(); + foreach (var value in Enum.GetValues()) + config[value] = true; + + return new SystemMetrics(meter, TimeSpan.FromSeconds(42), config, legacyNames: false); + } + + [Fact] + public void deduplicates_disk_paths_on_the_same_drive() { + // Different paths that resolve to the same drive should only produce a single + // (kind, disk) series per kind, not one per path. + using var meter = new Meter($"{typeof(SystemMetricsTests)}.dedupe"); + using var listener = new TestMeterListener(meter); + + DriveData GetDriveInfo(string path) => new("the-only-disk", TotalBytes: 1000, AvailableBytes: 400); + + var sut = CreateSystemMetrics(meter); + sut.CreateDiskMetric("eventstore-sys-disk", ["/db", "/index", "/logs"], GetDriveInfo, new() { + { MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" }, + { MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" }, + }); + + listener.Observe(); + + // One "used" and one "total" measurement, despite three paths on the same disk. + listener.RetrieveMeasurements("eventstore-sys-disk-bytes").Should().HaveCount(2); + } + + [Fact] + public void separate_drives_produce_separate_stats() { + // Paths on different drives should each produce their own (kind, disk) series. + using var meter = new Meter($"{typeof(SystemMetricsTests)}.twodisks"); + using var listener = new TestMeterListener(meter); + + DriveData GetDriveInfo(string path) => path switch { + "/db" => new DriveData("disk-a", TotalBytes: 1000, AvailableBytes: 400), // used 600 + "/logs" => new DriveData("disk-b", TotalBytes: 5000, AvailableBytes: 1000), // used 4000 + _ => throw new ArgumentOutOfRangeException(nameof(path), path, null), + }; + + var sut = CreateSystemMetrics(meter); + sut.CreateDiskMetric("eventstore-sys-disk", ["/db", "/logs"], GetDriveInfo, new() { + { MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" }, + { MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" }, + }); + + listener.Observe(); + + var measurements = listener.RetrieveMeasurements("eventstore-sys-disk-bytes"); + + // used + total for each of the two distinct disks. + measurements.Should().HaveCount(4); + + long ValueFor(string disk, string kind) => measurements + .Single(m => m.Tags.Any(t => t.Key == "disk" && (string)t.Value == disk) && + m.Tags.Any(t => t.Key == "kind" && (string)t.Value == kind)) + .Value; + + ValueFor("disk-a", "total").Should().Be(1000); + ValueFor("disk-a", "used").Should().Be(600); + ValueFor("disk-b", "total").Should().Be(5000); + ValueFor("disk-b", "used").Should().Be(4000); + } + [Fact] public void can_collect_sys_disk() { Assert.Collection( diff --git a/src/KurrentDB.Core/ClusterVNode.cs b/src/KurrentDB.Core/ClusterVNode.cs index 25880edfc85..4b21ea9d782 100644 --- a/src/KurrentDB.Core/ClusterVNode.cs +++ b/src/KurrentDB.Core/ClusterVNode.cs @@ -339,8 +339,9 @@ public ClusterVNode(ClusterVNodeOptions options, out var readerThreadsCount); var trackers = new Trackers(); + var indexPath = options.Database.Index ?? Path.Combine(dbConfig.Path, ESConsts.DefaultIndexDirectoryName); var metricsConfiguration = MetricsConfiguration.Get(configuration); - MetricsBootstrapper.Bootstrap(metricsConfiguration, dbConfig, trackers); + MetricsBootstrapper.Bootstrap(metricsConfiguration, dbConfig, indexPath, options.Logging.Log, trackers); var namingStrategy = new VersionedPatternFileNamingStrategy(dbConfig.Path, "chunk-"); IChunkFileSystem fileSystem = new ChunkLocalFileSystem(namingStrategy); @@ -625,7 +626,6 @@ void StartSubsystems() { threadPoolQueueLengthMonitor.Start(); // Log Format - var indexPath = options.Database.Index ?? Path.Combine(Db.Config.Path, ESConsts.DefaultIndexDirectoryName); var pTableMaxReaderCount = GetPTableMaxReaderCount(readerThreadsCount); var tfReader = new TFChunkReader(Db, Db.Config.WriterCheckpoint.AsReadOnly()); diff --git a/src/KurrentDB.Core/Metrics/SystemMetrics.cs b/src/KurrentDB.Core/Metrics/SystemMetrics.cs index 42a79a4af14..d95b354ce33 100644 --- a/src/KurrentDB.Core/Metrics/SystemMetrics.cs +++ b/src/KurrentDB.Core/Metrics/SystemMetrics.cs @@ -49,13 +49,21 @@ public void CreateMemoryMetric(string metricName, Dictionary dimNames) { + public void CreateDiskMetric(string metricName, IReadOnlyList paths, Func getDriveInfo, Dictionary dimNames) { var dims = new Dimensions(config, dimNames, tag => new()); - var getDriveInfo = Functions.Debounce(() => DriveStats.GetDriveInfo(dbPath), timeout); + // Multiple paths (db, index, logs) may live on the same drive. Resolve each to its drive + // and register only once per distinct drive so we don't emit duplicate (kind, disk) series. + var seenDisks = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var path in paths) { + var diskName = getDriveInfo(path).DiskName; + if (!seenDisks.Add(diskName)) + continue; - dims.Register(SystemTracker.DriveUsedBytes, GenMeasure(info => info.UsedBytes)); - dims.Register(SystemTracker.DriveTotalBytes, GenMeasure(info => info.TotalBytes)); + var debouncedDriveInfo = Functions.Debounce(() => getDriveInfo(path), timeout); + dims.Register(SystemTracker.DriveUsedBytes, GenMeasure(debouncedDriveInfo, info => info.UsedBytes)); + dims.Register(SystemTracker.DriveTotalBytes, GenMeasure(debouncedDriveInfo, info => info.TotalBytes)); + } if (dims.AnyRegistered()) if (legacyNames) @@ -65,8 +73,8 @@ public void CreateDiskMetric(string metricName, string dbPath, Dictionary> GenMeasure(Func func) => tag => { - var info = getDriveInfo(); + static Func> GenMeasure(Func getInfo, Func func) => tag => { + var info = getInfo(); return new( func(info), diff --git a/src/KurrentDB.Core/MetricsBootstrapper.cs b/src/KurrentDB.Core/MetricsBootstrapper.cs index 145ee44337f..97b6c80728f 100644 --- a/src/KurrentDB.Core/MetricsBootstrapper.cs +++ b/src/KurrentDB.Core/MetricsBootstrapper.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Linq; using KurrentDB.Core.Bus; @@ -67,6 +68,8 @@ public static class MetricsBootstrapper { public static void Bootstrap( Conf conf, TFChunkDbConfig dbConfig, + string indexPath, + string logPath, Trackers trackers) { OptionsFormatter.LogConfig("Metrics", conf); @@ -282,7 +285,7 @@ public static void Bootstrap( { Conf.SystemTracker.TotalMem, "total" }, }); - systemMetrics.CreateDiskMetric($"{serviceName}-sys-disk", dbConfig.Path, new() { + systemMetrics.CreateDiskMetric($"{serviceName}-sys-disk", [dbConfig.Path, indexPath, logPath], DriveStats.GetDriveInfo, new() { { Conf.SystemTracker.DriveTotalBytes, "total" }, { Conf.SystemTracker.DriveUsedBytes, "used" }, });