Skip to content

[DB-2137] Report disk usage metrics for index and log drives #5640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
timothycoleman wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[DB-2137] Report disk usage metrics for index and log drives #5640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 68 additions & 1 deletion src/KurrentDB.Core.XUnit.Tests/Metrics/SystemMetricsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using System.Runtime;
using FluentAssertions;
using KurrentDB.Common.Configuration;
Expand Down Expand Up @@ -45,7 +47,7 @@ public SystemMetricsTests() {
{ MetricsConfiguration.SystemTracker.TotalMem, "total" },
});

_sut.CreateDiskMetric("eventstore-sys-disk", ".", new() {
_sut.CreateDiskMetric("eventstore-sys-disk", ["."], DriveStats.GetDriveInfo, new() {
{ MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" },
{ MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" },
});
Expand Down Expand Up @@ -148,6 +150,71 @@ public void can_collect_sys_mem() {
});
}

private static SystemMetrics CreateSystemMetrics(Meter meter) {
var config = new Dictionary<MetricsConfiguration.SystemTracker, bool>();
foreach (var value in Enum.GetValues<MetricsConfiguration.SystemTracker>())
config[value] = true;

return new SystemMetrics(meter, TimeSpan.FromSeconds(42), config, legacyNames: false);
}

[Fact]
public void deduplicates_disk_paths_on_the_same_drive() {
// Different paths that resolve to the same drive should only produce a single
// (kind, disk) series per kind, not one per path.
using var meter = new Meter($"{typeof(SystemMetricsTests)}.dedupe");
using var listener = new TestMeterListener<long>(meter);

DriveData GetDriveInfo(string path) => new("the-only-disk", TotalBytes: 1000, AvailableBytes: 400);

var sut = CreateSystemMetrics(meter);
sut.CreateDiskMetric("eventstore-sys-disk", ["/db", "/index", "/logs"], GetDriveInfo, new() {
{ MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" },
{ MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" },
});

listener.Observe();

// One "used" and one "total" measurement, despite three paths on the same disk.
listener.RetrieveMeasurements("eventstore-sys-disk-bytes").Should().HaveCount(2);
}

[Fact]
public void separate_drives_produce_separate_stats() {
// Paths on different drives should each produce their own (kind, disk) series.
using var meter = new Meter($"{typeof(SystemMetricsTests)}.twodisks");
using var listener = new TestMeterListener<long>(meter);

DriveData GetDriveInfo(string path) => path switch {
"/db" => new DriveData("disk-a", TotalBytes: 1000, AvailableBytes: 400), // used 600
"/logs" => new DriveData("disk-b", TotalBytes: 5000, AvailableBytes: 1000), // used 4000
_ => throw new ArgumentOutOfRangeException(nameof(path), path, null),
};

var sut = CreateSystemMetrics(meter);
sut.CreateDiskMetric("eventstore-sys-disk", ["/db", "/logs"], GetDriveInfo, new() {
{ MetricsConfiguration.SystemTracker.DriveTotalBytes, "total" },
{ MetricsConfiguration.SystemTracker.DriveUsedBytes, "used" },
});

listener.Observe();

var measurements = listener.RetrieveMeasurements("eventstore-sys-disk-bytes");

// used + total for each of the two distinct disks.
measurements.Should().HaveCount(4);

long ValueFor(string disk, string kind) => measurements
.Single(m => m.Tags.Any(t => t.Key == "disk" && (string)t.Value == disk) &&
m.Tags.Any(t => t.Key == "kind" && (string)t.Value == kind))
.Value;

ValueFor("disk-a", "total").Should().Be(1000);
ValueFor("disk-a", "used").Should().Be(600);
ValueFor("disk-b", "total").Should().Be(5000);
ValueFor("disk-b", "used").Should().Be(4000);
}

[Fact]
public void can_collect_sys_disk() {
Assert.Collection(
Expand Down
4 changes: 2 additions & 2 deletions src/KurrentDB.Core/ClusterVNode.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,8 +339,9 @@ public ClusterVNode(ClusterVNodeOptions options,
out var readerThreadsCount);

var trackers = new Trackers();
var indexPath = options.Database.Index ?? Path.Combine(dbConfig.Path, ESConsts.DefaultIndexDirectoryName);
var metricsConfiguration = MetricsConfiguration.Get(configuration);
MetricsBootstrapper.Bootstrap(metricsConfiguration, dbConfig, trackers);
MetricsBootstrapper.Bootstrap(metricsConfiguration, dbConfig, indexPath, options.Logging.Log, trackers);

var namingStrategy = new VersionedPatternFileNamingStrategy(dbConfig.Path, "chunk-");
IChunkFileSystem fileSystem = new ChunkLocalFileSystem(namingStrategy);
Expand Down Expand Up @@ -625,7 +626,6 @@ void StartSubsystems() {
threadPoolQueueLengthMonitor.Start();

// Log Format
var indexPath = options.Database.Index ?? Path.Combine(Db.Config.Path, ESConsts.DefaultIndexDirectoryName);

var pTableMaxReaderCount = GetPTableMaxReaderCount(readerThreadsCount);
var tfReader = new TFChunkReader(Db, Db.Config.WriterCheckpoint.AsReadOnly());
Expand Down
20 changes: 14 additions & 6 deletions src/KurrentDB.Core/Metrics/SystemMetrics.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,21 @@ public void CreateMemoryMetric(string metricName, Dictionary<SystemTracker, stri
meter.CreateObservableGauge(metricName, dims.GenObserve(), "bytes");
}

public void CreateDiskMetric(string metricName, string dbPath, Dictionary<SystemTracker, string> dimNames) {
public void CreateDiskMetric(string metricName, IReadOnlyList<string> paths, Func<string, DriveData> getDriveInfo, Dictionary<SystemTracker, string> dimNames) {
var dims = new Dimensions<SystemTracker, long>(config, dimNames, tag => new());
Comment on lines +52 to 53

var getDriveInfo = Functions.Debounce(() => DriveStats.GetDriveInfo(dbPath), timeout);
// Multiple paths (db, index, logs) may live on the same drive. Resolve each to its drive
// and register only once per distinct drive so we don't emit duplicate (kind, disk) series.
var seenDisks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var path in paths) {
var diskName = getDriveInfo(path).DiskName;
if (!seenDisks.Add(diskName))
continue;

dims.Register(SystemTracker.DriveUsedBytes, GenMeasure(info => info.UsedBytes));
dims.Register(SystemTracker.DriveTotalBytes, GenMeasure(info => info.TotalBytes));
var debouncedDriveInfo = Functions.Debounce(() => getDriveInfo(path), timeout);
dims.Register(SystemTracker.DriveUsedBytes, GenMeasure(debouncedDriveInfo, info => info.UsedBytes));
dims.Register(SystemTracker.DriveTotalBytes, GenMeasure(debouncedDriveInfo, info => info.TotalBytes));
}

if (dims.AnyRegistered())
if (legacyNames)
Expand All @@ -65,8 +73,8 @@ public void CreateDiskMetric(string metricName, string dbPath, Dictionary<System

return;

Func<string, Measurement<long>> GenMeasure(Func<DriveData, long> func) => tag => {
var info = getDriveInfo();
static Func<string, Measurement<long>> GenMeasure(Func<DriveData> getInfo, Func<DriveData, long> func) => tag => {
var info = getInfo();

return new(
func(info),
Expand Down
5 changes: 4 additions & 1 deletion src/KurrentDB.Core/MetricsBootstrapper.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.Metrics;
using System.Linq;
using KurrentDB.Core.Bus;
Expand Down Expand Up @@ -67,6 +68,8 @@ public static class MetricsBootstrapper {
public static void Bootstrap(
Conf conf,
TFChunkDbConfig dbConfig,
string indexPath,
string logPath,
Trackers trackers) {

OptionsFormatter.LogConfig("Metrics", conf);
Expand Down Expand Up @@ -282,7 +285,7 @@ public static void Bootstrap(
{ Conf.SystemTracker.TotalMem, "total" },
});

systemMetrics.CreateDiskMetric($"{serviceName}-sys-disk", dbConfig.Path, new() {
systemMetrics.CreateDiskMetric($"{serviceName}-sys-disk", [dbConfig.Path, indexPath, logPath], DriveStats.GetDriveInfo, new() {
{ Conf.SystemTracker.DriveTotalBytes, "total" },
{ Conf.SystemTracker.DriveUsedBytes, "used" },
});
Expand Down
Loading