From 81b2a254df3fa05006d741694ca8549dfd48f56c Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Fri, 5 Jun 2026 16:37:13 +0300 Subject: [PATCH 01/25] Initial design of Kontrol Plane --- src/Directory.Packages.props | 9 +- .../DatabaseSnapshot.cs | 21 +++ src/KurrentDB.KontrolPlane/IDatabase.cs | 14 ++ src/KurrentDB.KontrolPlane/IDatabaseNode.cs | 12 ++ .../IDatabaseReplicaSet.cs | 10 ++ src/KurrentDB.KontrolPlane/IKontroller.cs | 27 ++++ .../KurrentDB.KontrolPlane.csproj | 22 +++ .../LeadershipRequiredException.cs | 6 + .../RaftKontroller.Appointment.cs | 132 ++++++++++++++++ .../RaftKontroller.ConfigStorage.cs | 18 +++ .../RaftKontroller.Impl.cs | 93 +++++++++++ .../RaftKontroller.Leadership.cs | 24 +++ .../RaftKontroller.Options.cs | 34 ++++ src/KurrentDB.KontrolPlane/RaftKontroller.cs | 64 ++++++++ src/KurrentDB.KontrolPlane/ReplicaState.cs | 14 ++ .../StateMachine/ClusterState.cs | 102 ++++++++++++ .../ClusterStateMachine.Database.cs | 30 ++++ .../StateMachine/ClusterStateMachine.Node.cs | 41 +++++ .../StateMachine/ClusterStateMachine.cs | 147 ++++++++++++++++++ .../StateMachine/Database.cs | 30 ++++ .../StateMachine/DatabaseNode.cs | 31 ++++ .../StateMachine/EndPointExtensions.cs | 26 ++++ .../StateMachine/IProtobufSerializable.cs | 42 +++++ .../LogEntries/AddOrUpdateDatabase.cs | 10 ++ .../LogEntries/AddOrUpdateDatabaseNode.cs | 10 ++ .../StateMachine/LogEntries/AppointLeader.cs | 10 ++ .../StateMachine/LogEntries/ILogEntry.cs | 20 +++ .../LogEntries/ProtobufLogEntry.cs | 26 ++++ .../StateMachine/LogEntries/RemoveDatabase.cs | 10 ++ .../LogEntries/RemoveDatabaseNode.cs | 10 ++ .../LogEntries/ReplicationHelpers.cs | 55 +++++++ .../StateMachine/LogEntries/wal.proto | 38 +++++ .../StateMachine/state.proto | 30 ++++ src/KurrentDB.sln | 18 +++ 34 files changed, 1182 insertions(+), 4 deletions(-) create mode 100644 src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs create mode 100644 src/KurrentDB.KontrolPlane/IDatabase.cs create mode 100644 src/KurrentDB.KontrolPlane/IDatabaseNode.cs create mode 100644 src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs create mode 100644 src/KurrentDB.KontrolPlane/IKontroller.cs create mode 100644 src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj create mode 100644 src/KurrentDB.KontrolPlane/LeadershipRequiredException.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.ConfigStorage.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs create mode 100644 src/KurrentDB.KontrolPlane/RaftKontroller.cs create mode 100644 src/KurrentDB.KontrolPlane/ReplicaState.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Database.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabase.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabaseNode.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AppointLeader.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ILogEntry.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabase.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabaseNode.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/state.proto diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 91f073c1f12..efa3b9dc5d6 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -9,10 +9,11 @@ - - - - + + + + + diff --git a/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs b/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs new file mode 100644 index 00000000000..4a910865e1e --- /dev/null +++ b/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs @@ -0,0 +1,21 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane; + +/// +/// Represents instant state of the database. +/// +public readonly record struct DatabaseSnapshot { + /// + /// Gets the database leader. + /// + /// if the leader is not assigned. + public IDatabaseNode? Leader { get; init; } + + /// + /// Gets the entire database. + /// + /// if the database is removed. + public required IDatabase? Database { get; init; } +} diff --git a/src/KurrentDB.KontrolPlane/IDatabase.cs b/src/KurrentDB.KontrolPlane/IDatabase.cs new file mode 100644 index 00000000000..fafe974afb7 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/IDatabase.cs @@ -0,0 +1,14 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane; + +public interface IDatabase { + public const string MainDatabaseId = "main"; + + string Description { get; } + + ulong Epoch { get; } + + IReadOnlyList Nodes { get; } +} diff --git a/src/KurrentDB.KontrolPlane/IDatabaseNode.cs b/src/KurrentDB.KontrolPlane/IDatabaseNode.cs new file mode 100644 index 00000000000..3cd739e15c9 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/IDatabaseNode.cs @@ -0,0 +1,12 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; + +namespace KurrentDB.KontrolPlane; + +public interface IDatabaseNode { + EndPoint Address { get; } + + bool IsReadOnlyReplica { get; } +} diff --git a/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs b/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs new file mode 100644 index 00000000000..982883eea55 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; + +namespace KurrentDB.KontrolPlane; + +public interface IDatabaseReplicaSet { + ValueTask GetReplicaStateAsync(EndPoint address, CancellationToken token); +} diff --git a/src/KurrentDB.KontrolPlane/IKontroller.cs b/src/KurrentDB.KontrolPlane/IKontroller.cs new file mode 100644 index 00000000000..47ad90dd544 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/IKontroller.cs @@ -0,0 +1,27 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; + +namespace KurrentDB.KontrolPlane; + +public interface IKontroller { + /// + /// Gets a list of the databases registered in the Kontrol Plane. + /// + IReadOnlyDictionary Databases { get; } + + IDatabaseReplicaSet ReplicaSet { get; init; } + + ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default); + + ValueTask AddOrUpdateDatabaseAsync(string databaseId, string description = "", CancellationToken token = default); + + ValueTask RemoveDatabaseAsync(string databaseId, CancellationToken token = default); + + ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, IDatabaseNode node, CancellationToken token = default); + + ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint address, CancellationToken token = default); + + IAsyncEnumerable ListenDatabaseAsync(string databaseId, CancellationToken token = default); +} diff --git a/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj new file mode 100644 index 00000000000..53af0942fc9 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj @@ -0,0 +1,22 @@ + + + net10.0 + enable + enable + true + true + false + KurrentDB.KontrolPlane + + + + + + + + + + + + + diff --git a/src/KurrentDB.KontrolPlane/LeadershipRequiredException.cs b/src/KurrentDB.KontrolPlane/LeadershipRequiredException.cs new file mode 100644 index 00000000000..2c8d67eaf4b --- /dev/null +++ b/src/KurrentDB.KontrolPlane/LeadershipRequiredException.cs @@ -0,0 +1,6 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane; + +public sealed class LeadershipRequiredException(Exception e) : InvalidOperationException("Current Kontroller is not leader", e); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs new file mode 100644 index 00000000000..2ed75126c92 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -0,0 +1,132 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Collections.Concurrent; +using System.Net; +using System.Runtime.InteropServices; +using DotNext.Diagnostics; + +namespace KurrentDB.KontrolPlane; + +using StateMachine; + +partial class RaftKontroller { + // key is database ID, value is the time when the leadership was updated for the particular database + private readonly ConcurrentDictionary _appointmentState = new(); + + // Spin in the loop and process appointments for every database + private async ValueTask ProcessAppointmentsAsync(CancellationToken token) { + var tasks = new List(17); + var deletedDatabases = new HashSet(); + var timer = new PeriodicTimer(_appointmentExpiration); + try { + do { + var databases = _state.CurrentState.Databases; + StartAppointments(databases, tasks, token); + RemoveDeletedDatabases(databases, deletedDatabases); + + await Task.WhenAll(tasks); + tasks.Clear(); + deletedDatabases.Clear(); + } while (await timer.WaitForNextTickAsync(token)); + } finally { + timer.Dispose(); + _appointmentState.Clear(); + } + } + + private void RemoveDeletedDatabases( + IReadOnlyDictionary existingDatabases, + HashSet deletedDatabases) { + // Remove deleted databases from the appointment state + foreach (var databaseId in _appointmentState.Keys) { + if (!existingDatabases.ContainsKey(databaseId)) + deletedDatabases.Add(databaseId); + } + + foreach (var databaseId in deletedDatabases) { + _appointmentState.TryRemove(databaseId, out _); + } + } + + private void StartAppointments( + IReadOnlyDictionary databases, + List tasks, + CancellationToken token) { + // Process appointment for every database in parallel + foreach (var (databaseId, database) in databases) { + if (IsAppointmentRequired(databaseId, database)) + tasks.Add(AppointLeaderAsync(databaseId, database.Nodes, database.Version, token)); + } + } + + private bool IsAppointmentRequired(string databaseId, Database database) + => database.Nodes is not [] + && (!_appointmentState.TryGetValue(databaseId, out var appointment) + || appointment.IsExpired(_appointmentExpiration)); + + private async Task AppointLeaderAsync(string databaseId, IReadOnlyList nodes, ulong expectedVersion, CancellationToken token) { + var responses = new Dictionary(nodes.Count); + + await foreach (var task in Task.WhenEach(GetReplicaState(ReplicaSet, nodes, token))) { + try { + var pair = await task; + responses.Add(pair.Key, pair.Value); + } catch (OperationCanceledException e) when (e.CancellationToken == token) { + return; // cancellation requested, abort appointment + } catch (Exception) { + // member is unavailable, don't add it to a collection of successful responses + } + } + + // Appoint leader only if we have a quorum + var quorum = nodes.Count / 2 + 1; + if (responses.Count < quorum) + return; + + // Find the node with the max Epoch + var maxEpoch = responses.Values.Max(static state => state.Epoch); + + // Find the node with the max offset + var candidate = responses + .Where(pair => pair.Value.Epoch == maxEpoch) + .MaxBy(static pair => pair.Value.UncommittedOffset) + .Key; + + // Appoint the leader + if (await AppointLeaderAsync(databaseId, candidate, expectedVersion, token)) { + _appointmentState[databaseId] = new LeaderAppointment(candidate); + } + + static IEnumerable>> GetReplicaState( + IDatabaseReplicaSet replicas, + IEnumerable nodes, + CancellationToken token) + => nodes + .Where(static node => !node.IsReadOnlyReplica) // r/o replicas cannot contribute to the qorum + .Select(node => GetReplicaStateAsync(replicas, node.Address, token)); + + static async Task> GetReplicaStateAsync( + IDatabaseReplicaSet replicas, + EndPoint address, + CancellationToken token) + => new(address, await replicas.GetReplicaStateAsync(address, token)); + } + + private bool RenewLeaderAppointment(string databaseId, EndPoint leaderAddress) { + if (!_appointmentState.TryGetValue(databaseId, out var expectedAppointment)) + return false; + + var newAppointment = new LeaderAppointment(leaderAddress); + return _appointmentState.TryUpdate(databaseId, newAppointment, expectedAppointment); + } + + [StructLayout(LayoutKind.Auto)] + private readonly record struct LeaderAppointment(EndPoint Address, Timestamp UpdatedAt) { + public LeaderAppointment(EndPoint address) + : this(address, new()) { + } + + public bool IsExpired(TimeSpan expiration) => UpdatedAt.Elapsed >= expiration; + } +} diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.ConfigStorage.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.ConfigStorage.cs new file mode 100644 index 00000000000..fe6a67d98cb --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.ConfigStorage.cs @@ -0,0 +1,18 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using DotNext.Buffers; +using DotNext.Net; +using DotNext.Net.Cluster.Consensus.Raft.Membership; + +namespace KurrentDB.KontrolPlane; + +partial class RaftKontroller { + private sealed class PersistentConfigurationStorage(string fileName) : PersistentClusterConfigurationStorage(fileName) { + protected override void Encode(EndPoint address, ref BufferWriterSlim writer) + => writer.WriteEndPoint(address); + + protected override EndPoint Decode(ref SequenceReader reader) => reader.ReadEndPoint(); + } +} diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs new file mode 100644 index 00000000000..d849c0e3c7f --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -0,0 +1,93 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using System.Runtime.CompilerServices; +using DotNext.Net.Cluster.Consensus.Raft; + +namespace KurrentDB.KontrolPlane; + +using static StateMachine.LogEntries.ReplicationHelpers; + +partial class RaftKontroller : IKontroller { + public IReadOnlyDictionary Databases => _state.CurrentState; + + public async ValueTask AddOrUpdateDatabaseAsync(string databaseId, string description = "", CancellationToken token = default) { + var completion = new TaskCompletionSource(); + try { + await _raft.AddOrUpdateDatabaseAsync(databaseId, description, completion, token); + } catch (NotLeaderException e) { + throw new LeadershipRequiredException(e); + } + + return await completion.Task.WaitAsync(token); + } + + public async ValueTask RemoveDatabaseAsync(string databaseId, CancellationToken token = default) { + var completion = new TaskCompletionSource(); + try { + await _raft.RemoveDatabaseAsync(databaseId, completion, token); + } catch (NotLeaderException e) { + throw new LeadershipRequiredException(e); + } + + return await completion.Task.WaitAsync(token); + } + + public async ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, IDatabaseNode node, CancellationToken token = default) { + var completion = new TaskCompletionSource(); + try { + await _raft.AddOrUpdateDatabaseNodeAsync(databaseId, node, completion, token); + } catch (NotLeaderException e) { + throw new LeadershipRequiredException(e); + } + + return await completion.Task.WaitAsync(token); + } + + public async ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint address, CancellationToken token = default) { + var completion = new TaskCompletionSource(); + try { + await _raft.RemoveDatabaseNodeAsync(databaseId, address, completion, token); + } catch (NotLeaderException e) { + throw new LeadershipRequiredException(e); + } + + return await completion.Task.WaitAsync(token); + } + + private async ValueTask AppointLeaderAsync(string databaseId, + EndPoint address, + ulong expectedVersion, + CancellationToken token = default) { + var completion = new TaskCompletionSource(); + await _raft.AppointLeaderAsync(databaseId, address, expectedVersion, completion, token); + return await completion.Task.WaitAsync(token); + } + + public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default) { + ValueTask task; + try { + task = new(RenewLeaderAppointment(databaseId, leaderAddress)); + } catch (Exception e) { + task = ValueTask.FromException(e); + } + + return task; + } + + public async IAsyncEnumerable ListenDatabaseAsync(string databaseId, [EnumeratorCancellation] CancellationToken token = default) { + yield break; + // var leadershiptToken = _raft.LeadershipToken; + // var tokenSource = _multiplexer.Combine(_raft.LeadershipToken, token); + // try { + // yield return null; + // } catch (OperationCanceledException e) when (e.CausedBy(tokenSource, leadershiptToken)) { + // throw new LeadershipRequiredException(e); + // } catch (OperationCanceledException e) when (e.CancellationToken == tokenSource.CancellationOrigin) { + // throw new OperationCanceledException(e.Message, e, e.CancellationToken); + // } finally { + // await tokenSource.DisposeAsync(); + // } + } +} diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs new file mode 100644 index 00000000000..543b2b942c6 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs @@ -0,0 +1,24 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane; + +partial class RaftKontroller { + private async Task HandleLeadershipAsync() { + for (;;) { + CancellationToken leadershipToken; + try { + leadershipToken = await _raft.WaitForLeadershipAsync(CancellationToken.None); + } catch (ObjectDisposedException) { + break; + } + + // the local node is elected as Kontrol Plane leader + try { + await ProcessAppointmentsAsync(leadershipToken); + } catch (OperationCanceledException e) when (e.CancellationToken == leadershipToken) { + // the local node is not a leader anymore + } + } + } +} diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs new file mode 100644 index 00000000000..4fcc162d5c7 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs @@ -0,0 +1,34 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; + +namespace KurrentDB.KontrolPlane; + +partial class RaftKontroller { + /// + /// Represents Kontroller options. + /// + public readonly struct Options { + public required WriteAheadLog.Options WalOptions { + get; + init; + } + + public required IPEndPoint ListenAddress { + get; + init; + } + + public EndPoint PublicAddress { + get => field ?? ListenAddress; + init; + } + + public required TimeSpan AppointmentExpiration { + get; + init => field = value > TimeSpan.Zero ? value : throw new ArgumentOutOfRangeException(nameof(value)); + } + } +} diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs new file mode 100644 index 00000000000..16903b744af --- /dev/null +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -0,0 +1,64 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using DotNext.Net.Cluster.Consensus.Raft; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; +using DotNext.Threading; + +namespace KurrentDB.KontrolPlane; + +using StateMachine; + +/// +/// Represents Raft-based implementation of interface. +/// +public partial class RaftKontroller : IAsyncDisposable { + private readonly WriteAheadLog _wal; + private readonly ClusterStateMachine _state; + private readonly RaftCluster _raft; + private readonly CancellationTokenMultiplexer _multiplexer; + private readonly TimeSpan _appointmentExpiration; + private Task _leadershipTask; + + public RaftKontroller(in Options options) { + var stateLocation = new DirectoryInfo(Path.Combine(options.WalOptions.Location, "db")); + var configStorageLocation = Path.Combine(options.WalOptions.Location, "members.list"); + _state = new(stateLocation); + _wal = new WriteAheadLog(options.WalOptions, _state); + + var config = new RaftCluster.TcpConfiguration(options.ListenAddress) { + PublicEndPoint = options.PublicAddress, + ConfigurationStorage = new PersistentConfigurationStorage(configStorageLocation), + }; + + _raft = new RaftCluster(config) { + AuditTrail = _wal + }; + + _leadershipTask = Task.CompletedTask; + _multiplexer = new() { MaximumRetained = 128 }; + _appointmentExpiration = options.AppointmentExpiration; + _appointmentState = new(); + } + + public required IDatabaseReplicaSet ReplicaSet { + get; + init => field = value ?? throw new ArgumentNullException(nameof(value)); + } + + public async Task StartAsync(CancellationToken token) { + await _raft.StartAsync(token); + _leadershipTask = HandleLeadershipAsync(); + } + + public async Task StopAsync(CancellationToken token) { + await _raft.StopAsync(token); + await _leadershipTask.ConfigureAwait(false); + } + + public async ValueTask DisposeAsync() { + await _raft.DisposeAsync(); + await _wal.DisposeAsync(); + await _state.DisposeAsync(); + } +} diff --git a/src/KurrentDB.KontrolPlane/ReplicaState.cs b/src/KurrentDB.KontrolPlane/ReplicaState.cs new file mode 100644 index 00000000000..207a22516d0 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/ReplicaState.cs @@ -0,0 +1,14 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; + +namespace KurrentDB.KontrolPlane; + +/// +/// Represents replication state of the node. +/// +/// The latest known offset of the uncommitted log record. +/// The epoch of the database node. +[StructLayout(LayoutKind.Auto)] +public readonly record struct ReplicaState(long UncommittedOffset, ulong Epoch); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs new file mode 100644 index 00000000000..2e13259ff0b --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs @@ -0,0 +1,102 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class ClusterState : IReadOnlyDictionary, IProtobufSerializable { + public ClusterState AddMainDatabase(out Database database) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + database = new Database { Version = copy.Version }; + copy.Databases.Add(IDatabase.MainDatabaseId, database); + return copy; + } + + public ClusterState AddDatabase(string databaseId, Database database) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + database.Version = copy.Version; + copy.Databases.Add(databaseId, database); + return copy; + } + + public ClusterState RemoveDatabase(string databaseId) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + return copy.Databases.Remove(databaseId) ? copy : this; + } + + public ClusterState AddDatabaseNode(string databaseId, DatabaseNode databaseNode) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + + if (!copy.Databases.TryGetValue(databaseId, out var database)) + return this; + + copy.Databases[databaseId] = database.AddNode(databaseNode); + return copy; + } + + public ClusterState UpdateDatabaseNode(string databaseId, DatabaseNode databaseNode, int index) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + + if (!copy.Databases.TryGetValue(databaseId, out var database)) + return this; + + copy.Databases[databaseId] = database.UpdateNode(databaseNode, index); + return copy; + } + + public ClusterState RemoveDatabaseNode(string databaseId, ByteString address) { + var copy = new ClusterState(this) { Version = Version + 1UL }; + + if (!copy.Databases.TryGetValue(databaseId, out var database) || + database.Nodes.Find(address, out var index) is null) + return this; + + database = new(database) { Version = copy.Version }; + database.Nodes.RemoveAt(index); + copy.Databases[databaseId] = database; + return copy; + } + + public ClusterState AppointLeader(string databaseId, ulong expectedVersion, ByteString address) { + if (!Databases.TryGetValue(databaseId, out var database) + || database.Version != expectedVersion + || database.Nodes.Find(address, out _) is null) + return this; + + return new(this) { + Version = Version + 1UL, + Databases = { [databaseId] = database.AppointLeader(address) }, + }; + } + + IEnumerator> IEnumerable>.GetEnumerator() { + foreach (var (key, value) in Databases) { + yield return new(key, value); + } + } + + IEnumerator IEnumerable.GetEnumerator() => Databases.GetEnumerator(); + + int IReadOnlyCollection>.Count => Databases.Count; + + bool IReadOnlyDictionary.ContainsKey(string key) => Databases.ContainsKey(key); + + bool IReadOnlyDictionary.TryGetValue(string key, [MaybeNullWhen(false)] out IDatabase value) { + if (Databases.TryGetValue(key, out var typedResult)) { + value = typedResult; + return true; + } + + value = null; + return false; + } + + IDatabase IReadOnlyDictionary.this[string key] => Databases[key]; + + IEnumerable IReadOnlyDictionary.Keys => Databases.Keys; + + IEnumerable IReadOnlyDictionary.Values => Databases.Values; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs new file mode 100644 index 00000000000..3012f9202b2 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs @@ -0,0 +1,30 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class ClusterStateMachine { + private bool AddDatabase(LogEntries.AddOrUpdateDatabase entry) { + var stateCopy = CurrentState; + + if (stateCopy.Databases.ContainsKey(entry.DatabaseId)) + return false; + + CurrentState = stateCopy.AddDatabase(entry.DatabaseId, new() { Description = entry.Description }); + return true; + } + + private bool RemoveDatabase(LogEntries.RemoveDatabase entry) { + var stateCopy = CurrentState; + + if (!stateCopy.Databases.ContainsKey(entry.DatabaseId)) + return false; + + CurrentState = stateCopy.RemoveDatabase(entry.DatabaseId); + return true; + } + + private bool AppointLeader(LogEntries.AppointLeader entry) { + return TrySetCurrentState(CurrentState.AppointLeader(entry.DatabaseId, entry.ExpectedVersion, entry.Address)); + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs new file mode 100644 index 00000000000..205615dfd87 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs @@ -0,0 +1,41 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class ClusterStateMachine { + private bool AddDatabaseNode(LogEntries.AddOrUpdateDatabaseNode entry) { + var stateCopy = CurrentState; + + if (!stateCopy.Databases.TryGetValue(entry.DatabaseId, out var database)) { + if (entry.DatabaseId is not IDatabase.MainDatabaseId) + return false; + + // Add main database automatically + stateCopy = stateCopy.AddMainDatabase(out database); + } + + if (database.Nodes.Find(entry.Address, out var index) is { } node) { + // update existing node + node = new DatabaseNode(node) { IsReadOnlyReplica = entry.IsReadOnlyReplica }; + stateCopy = stateCopy.UpdateDatabaseNode(entry.DatabaseId, node, index); + } else { + // create new node + node = new DatabaseNode { Address = entry.Address, IsReadOnlyReplica = entry.IsReadOnlyReplica }; + stateCopy = stateCopy.AddDatabaseNode(entry.DatabaseId, node); + } + + CurrentState = stateCopy; + return true; + } + + private bool RemoveDatabaseNode(LogEntries.RemoveDatabaseNode entry) { + var stateCopy = CurrentState; + + if (!stateCopy.Databases.ContainsKey(entry.DatabaseId)) + return false; + + CurrentState = stateCopy.RemoveDatabaseNode(entry.DatabaseId, entry.Address); + return true; + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs new file mode 100644 index 00000000000..6dba63807d7 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -0,0 +1,147 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using DotNext; +using DotNext.IO; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; +using DotNext.Threading; + +namespace KurrentDB.KontrolPlane.StateMachine; + +/// +/// Represents internal Kontrol Plane database. +/// +/// +internal sealed partial class ClusterStateMachine(DirectoryInfo location) : SimpleStateMachine(location) { + private readonly AsyncTrigger _stateChanged = new(); + private ClusterState _state = new(); + + public ClusterState CurrentState { + get => Volatile.Read(in _state); + private set => TrySetCurrentState(value); + } + + private bool TrySetCurrentState(ClusterState newState) { + if (ReferenceEquals(Volatile.Read(in _state), newState)) + return false; + + _state = newState; + _stateChanged.Signal(resumeAll: true); // acts as a barrier + return true; + } + + /// + /// Gets or sets the snapshot depth. + /// + /// is less than or equal to zero. + public int SnapshotDepth { + get; + init => field = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value)); + } = 100; + + /// + /// Tracks cluster state changes as a stream of state snapshots. + /// + /// The token that can be used to cancel the operation. + /// A stream over cluster state snapshots. + public async IAsyncEnumerable> TrackChangesAsync( + [EnumeratorCancellation] CancellationToken token) { + do { + var actual = CurrentState; + yield return actual; + await _stateChanged.SpinWaitAsync(new DatabaseChangeTracker(this) { ExpectedVersion = actual.Version }, token); + } while (!token.IsCancellationRequested); + } + + // Restore state from the snapshot + protected override ValueTask RestoreAsync(FileInfo snapshotFile, CancellationToken token) { + var task = ValueTask.CompletedTask; + var fs = default(FileStream); + try { + fs = snapshotFile.Open(new FileStreamOptions { + Mode = FileMode.Open, + Access = FileAccess.Read, + Share = FileShare.Read, + BufferSize = 4096, + Options = FileOptions.SequentialScan + }); + + _state = ClusterState.Parser.ParseFrom(fs); + _stateChanged.Signal(resumeAll: true); + } catch (Exception e) { + task = ValueTask.FromException(e); + } finally { + fs?.Dispose(); + } + + return task; + } + + // Persist current state + protected override ValueTask PersistAsync(IAsyncBinaryWriter writer, CancellationToken token) + => _state.WriteToAsync(writer, token); + + // Apply log entry from the WAL to the current state + protected override async ValueTask ApplyAsync(LogEntry entry, CancellationToken token) { + switch (entry.CommandId) { + case LogEntries.AddOrUpdateDatabase.TypeId: + var result = AddDatabase(await DeserializeAsync(entry, token)); + (entry.Context as TaskCompletionSource)?.TrySetResult(result); + break; + case LogEntries.RemoveDatabase.TypeId: + result = RemoveDatabase(await DeserializeAsync(entry, token)); + (entry.Context as TaskCompletionSource)?.TrySetResult(result); + break; + case LogEntries.AddOrUpdateDatabaseNode.TypeId: + result = AddDatabaseNode(await DeserializeAsync(entry, token)); + (entry.Context as TaskCompletionSource)?.TrySetResult(result); + break; + case LogEntries.RemoveDatabaseNode.TypeId: + result = RemoveDatabaseNode(await DeserializeAsync(entry, token)); + (entry.Context as TaskCompletionSource)?.TrySetResult(result); + break; + case LogEntries.AppointLeader.TypeId: + result = AppointLeader(await DeserializeAsync(entry, token)); + break; + default: + Debug.Fail($"Unexpected entry type {entry.CommandId}"); + break; + } + + return entry.Index % SnapshotDepth is 0; + } + + private static ValueTask DeserializeAsync(in LogEntry entry, CancellationToken token) + where T : class, IProtobufSerializable { + ValueTask task; + if (entry.TryGetPayload(out var sequence)) { + // fast path, deserialize from memory + try { + task = new(T.Parser.ParseFrom(sequence)); + } catch (Exception e) { + task = ValueTask.FromException(e); + } + } else { + task = DeserializeSlowAsync(entry, token); + } + + return task; + + static async ValueTask DeserializeSlowAsync(LogEntry entry, CancellationToken token) { + using var owner = await entry.ToMemoryAsync(token: token); + return T.Parser.ParseFrom(owner.Memory.Span); + } + } + + [StructLayout(LayoutKind.Auto)] + private readonly struct DatabaseChangeTracker(ClusterStateMachine stateMachine) : ISupplier { + private ulong ActualVersion => stateMachine.CurrentState.Version; + + public required ulong ExpectedVersion { get; init; } + + bool ISupplier.Invoke() => ActualVersion == ExpectedVersion; + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/Database.cs new file mode 100644 index 00000000000..aed31f443af --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Database.cs @@ -0,0 +1,30 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class Database : IDatabase { + IReadOnlyList IDatabase.Nodes => Nodes; + + public Database AddNode(DatabaseNode node) { + var copy = new Database(this) { Version = Version + 1UL }; + node.Version = copy.Version; + copy.Nodes.Add(node); + return copy; + } + + public Database UpdateNode(DatabaseNode node, int index) { + var copy = new Database(this) { Version = Version + 1UL }; + node.Version = copy.Version; + copy.Nodes[index] = node; + return copy; + } + + public Database AppointLeader(ByteString address) => new(this) { + Version = Version + 1UL, + Epoch = Epoch + 1UL, + LeaderAddress = address, + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs b/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs new file mode 100644 index 00000000000..1e94a131caf --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs @@ -0,0 +1,31 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class DatabaseNode : IDatabaseNode { + EndPoint IDatabaseNode.Address => field ??= Address.ToEndPoint(); +} + +internal static class DatabaseNodeExtensions { + public static DatabaseNode? Find(this IReadOnlyList nodes, ByteString address, out int index) + => nodes.Find(address.AreEqual, out index); + + private static DatabaseNode? Find(this IReadOnlyList nodes, Predicate predicate, out int index) { + for (var i = 0; i < nodes.Count; i++) { + var result = nodes[i]; + if (predicate(result)) { + index = i; + return result; + } + } + + index = -1; + return null; + } + + private static bool AreEqual(this ByteString address, DatabaseNode node) => address.Span.SequenceEqual(node.Address.Span); +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs b/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs new file mode 100644 index 00000000000..0b8746adce6 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using DotNext.Buffers; +using DotNext.Net; +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine; + +internal static class EndPointExtensions { + public static EndPoint ToEndPoint(this ByteString bytes) { + var reader = new SequenceReader(bytes.Memory); + return reader.ReadEndPoint(); + } + + public static ByteString ToByteString(this EndPoint ep) { + var writer = new BufferWriterSlim(stackalloc byte[256]); + try { + writer.WriteEndPoint(ep); + return ByteString.CopyFrom(writer.WrittenSpan); + } finally { + writer.Dispose(); + } + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs b/src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs new file mode 100644 index 00000000000..57f37052433 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs @@ -0,0 +1,42 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using DotNext.Buffers; +using DotNext.IO; +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine; + +internal interface IProtobufSerializable : IMessage + where TSelf : class, IProtobufSerializable { + + public static abstract MessageParser Parser { get; } +} + +internal static class ProtobufSerializer { + public static ValueTask WriteToAsync(this T obj, TWriter writer, CancellationToken token) + where T : class, IProtobufSerializable + where TWriter : IAsyncBinaryWriter{ + ValueTask task; + if (writer.TryGetBufferWriter() is { } buffer) { + // fast path + task = ValueTask.CompletedTask; + try { + obj.WriteTo(buffer); + } catch (Exception e) { + task = ValueTask.FromException(e); + } + } else { + // slow path + task = SerializeSlowAsync(obj, writer, token); + } + + return task; + + static async ValueTask SerializeSlowAsync(T state, IAsyncBinaryWriter writer, CancellationToken token) { + using var buffer = new PoolingBufferWriter { Capacity = 4096 }; + state.WriteTo(buffer); + await writer.WriteAsync(buffer.WrittenMemory, token: token).ConfigureAwait(false); + } + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabase.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabase.cs new file mode 100644 index 00000000000..6eb715df0c1 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabase.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +partial class AddOrUpdateDatabase : ILogEntry { + public const int TypeId = 2; + + static int ILogEntry.TypeId => TypeId; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabaseNode.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabaseNode.cs new file mode 100644 index 00000000000..90f18a7cbfc --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AddOrUpdateDatabaseNode.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +partial class AddOrUpdateDatabaseNode : ILogEntry { + public const int TypeId = 0; + + static int ILogEntry.TypeId => TypeId; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AppointLeader.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AppointLeader.cs new file mode 100644 index 00000000000..a633be1fc52 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/AppointLeader.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +partial class AppointLeader : ILogEntry { + public const int TypeId = 4; + + static int ILogEntry.TypeId => TypeId; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ILogEntry.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ILogEntry.cs new file mode 100644 index 00000000000..7f00cbea1bc --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ILogEntry.cs @@ -0,0 +1,20 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using Google.Protobuf; + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +/// +/// Root interface for all log entries. +/// +internal interface ILogEntry : IMessage { + + /// + /// Type identifier that is used to distinguish log entry types. + /// + public static abstract int TypeId { get; } +} + +internal interface ILogEntry : ILogEntry, IProtobufSerializable + where TSelf : class, ILogEntry; diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs new file mode 100644 index 00000000000..fdec8fb0028 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs @@ -0,0 +1,26 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using DotNext.IO; +using DotNext.Net.Cluster.Consensus.Raft; + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +[StructLayout(LayoutKind.Auto)] +internal readonly struct ProtobufLogEntry(T entry) : IInputLogEntry + where T : class, ILogEntry { + + ValueTask IDataTransferObject.WriteToAsync(TWriter writer, CancellationToken token) + => entry.WriteToAsync(writer, token); + + bool IDataTransferObject.IsReusable => true; + + long? IDataTransferObject.Length => null; + + int? IRaftLogEntry.CommandId => T.TypeId; + + public required long Term { get; init; } + + public object? Context { get; init; } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabase.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabase.cs new file mode 100644 index 00000000000..d74faec3c28 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabase.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +partial class RemoveDatabase : ILogEntry { + public const int TypeId = 3; + + static int ILogEntry.TypeId => TypeId; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabaseNode.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabaseNode.cs new file mode 100644 index 00000000000..1d5dd06dd4c --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/RemoveDatabaseNode.cs @@ -0,0 +1,10 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +partial class RemoveDatabaseNode : ILogEntry { + public const int TypeId = 1; + + static int ILogEntry.TypeId => TypeId; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs new file mode 100644 index 00000000000..5aa50f5067f --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs @@ -0,0 +1,55 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using DotNext.Net.Cluster.Consensus.Raft; + +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; + +internal static class ReplicationHelpers { + extension(IRaftCluster raft) + { + public ValueTask AddOrUpdateDatabaseAsync(string databaseId, + string description, + TaskCompletionSource completion, + CancellationToken token) + => raft.ReplicateAsync( + new ProtobufLogEntry(new() { DatabaseId = databaseId, Description = description }) + { Term = raft.Term, Context = completion }, token); + + public ValueTask RemoveDatabaseAsync(string databaseId, + TaskCompletionSource completion, + CancellationToken token) + => raft.ReplicateAsync( + new ProtobufLogEntry(new() { DatabaseId = databaseId }) + { Term = raft.Term, Context = completion }, token); + + public ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, + IDatabaseNode node, + TaskCompletionSource completion, + CancellationToken token) + => raft.ReplicateAsync( + new ProtobufLogEntry(new() + { Address = node.Address.ToByteString(), DatabaseId = databaseId, IsReadOnlyReplica = node.IsReadOnlyReplica }) + { Term = raft.Term, Context = completion }, token); + + public ValueTask RemoveDatabaseNodeAsync(string databaseId, + EndPoint address, + TaskCompletionSource completion, + CancellationToken token) + => raft.ReplicateAsync( + new ProtobufLogEntry(new() + { Address = address.ToByteString(), DatabaseId = databaseId }) + { Term = raft.Term, Context = completion }, token); + + public ValueTask AppointLeaderAsync(string databaseId, + EndPoint address, + ulong expectedVersion, + TaskCompletionSource completion, + CancellationToken token) + => raft.ReplicateAsync( + new ProtobufLogEntry(new() + { Address = address.ToByteString(), DatabaseId = databaseId, ExpectedVersion = expectedVersion }) + { Term = raft.Term, Context = completion }, token); + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto new file mode 100644 index 00000000000..67f7011dddc --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto @@ -0,0 +1,38 @@ +// WAL log entries + +syntax = "proto3"; + +package kurrent.kontrol_plane.log_entries; + +option csharp_namespace = "KurrentDB.KontrolPlane.StateMachine.LogEntries"; + +message AddOrUpdateDatabaseNode +{ + string databaseId = 1; + bytes address = 2; + bool isReadOnlyReplica = 3; +} + +message RemoveDatabaseNode +{ + string databaseId = 1; + bytes address = 2; +} + +message AddOrUpdateDatabase +{ + string databaseId = 1; + string description = 2; +} + +message RemoveDatabase +{ + string databaseId = 1; +} + +message AppointLeader +{ + string databaseId = 1; + uint64 expectedVersion = 2; + bytes address = 3; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/state.proto b/src/KurrentDB.KontrolPlane/StateMachine/state.proto new file mode 100644 index 00000000000..ab2eab9dba0 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/state.proto @@ -0,0 +1,30 @@ +// Kontroller Database Format + +syntax = "proto3"; + +package kurrent.kontrol_plane.cluster_state; + +option csharp_namespace = "KurrentDB.KontrolPlane.StateMachine"; + +// Represents entire state of the Kontroller internal database +message ClusterState +{ + uint64 version = 1; + map Databases = 2; +} + +message Database +{ + uint64 version = 1; + string description = 2; + uint64 epoch = 3; + repeated DatabaseNode Nodes = 4; + bytes leaderAddress = 5; +} + +message DatabaseNode +{ + uint64 version = 1; + bytes address = 2; + bool isReadOnlyReplica = 3; +} diff --git a/src/KurrentDB.sln b/src/KurrentDB.sln index 7c5859d390f..fca682aa1a7 100644 --- a/src/KurrentDB.sln +++ b/src/KurrentDB.sln @@ -208,6 +208,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.Projections.Share EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.Projections.Management.Tests", "KurrentDB.Projections.Management.Tests\KurrentDB.Projections.Management.Tests.csproj", "{7EC3CEA5-F843-EEF2-AD47-B41941AA3978}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.KontrolPlane", "KurrentDB.KontrolPlane\KurrentDB.KontrolPlane.csproj", "{C4FA3E05-81FA-4151-8BF8-D1B54D486627}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1636,6 +1638,22 @@ Global {7EC3CEA5-F843-EEF2-AD47-B41941AA3978}.Release|x64.Build.0 = Release|x64 {7EC3CEA5-F843-EEF2-AD47-B41941AA3978}.Release|x86.ActiveCfg = Release|Any CPU {7EC3CEA5-F843-EEF2-AD47-B41941AA3978}.Release|x86.Build.0 = Release|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|ARM64.Build.0 = Debug|ARM64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|x64.ActiveCfg = Debug|x64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|x64.Build.0 = Debug|x64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|x86.ActiveCfg = Debug|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Debug|x86.Build.0 = Debug|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|Any CPU.Build.0 = Release|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|ARM64.ActiveCfg = Release|ARM64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|ARM64.Build.0 = Release|ARM64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x64.ActiveCfg = Release|x64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x64.Build.0 = Release|x64 + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x86.ActiveCfg = Release|Any CPU + {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 68fbf276eaf38c5bcb6591e6a97f8d1d7ea9413d Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 11:21:00 +0300 Subject: [PATCH 02/25] Finalized migration to DuckDB --- src/Directory.Packages.props | 2 +- .../{IDatabase.cs => Database.cs} | 14 +- src/KurrentDB.KontrolPlane/DatabaseCluster.cs | 18 ++ src/KurrentDB.KontrolPlane/DatabaseLeader.cs | 13 + src/KurrentDB.KontrolPlane/DatabaseNode.cs | 17 ++ .../DatabaseSnapshot.cs | 21 -- .../IDatabaseReplicaSet.cs | 9 + src/KurrentDB.KontrolPlane/IKontroller.cs | 23 +- .../{IDatabaseNode.cs => KontrollerEntity.cs} | 9 +- .../KurrentDB.KontrolPlane.csproj | 3 +- .../RaftKontroller.Appointment.cs | 42 ++-- .../RaftKontroller.Impl.cs | 152 ++++++++---- .../RaftKontroller.Options.cs | 5 + src/KurrentDB.KontrolPlane/RaftKontroller.cs | 4 +- .../StateMachine/ClusterState.cs | 102 -------- .../ClusterStateMachine.Database.cs | 30 --- .../ClusterStateMachine.Handlers.cs | 128 ++++++++++ .../StateMachine/ClusterStateMachine.Node.cs | 41 ---- .../ClusterStateMachine.Snapshot.cs | 111 +++++++++ .../StateMachine/ClusterStateMachine.cs | 223 ++++++++++-------- .../StateMachine/CommandInfo.cs | 14 ++ .../StateMachine/Database.cs | 30 --- .../StateMachine/DatabaseNode.cs | 31 --- .../StateMachine/EndPointExtensions.cs | 5 +- .../{ => LogEntries}/IProtobufSerializable.cs | 2 +- .../LogEntries/ReplicationHelpers.cs | 45 ++-- .../StateMachine/LogEntries/wal.proto | 18 +- .../StateMachine/Queries/AllDatabasesQuery.cs | 18 ++ .../StateMachine/Queries/AllNodesQuery.cs | 23 ++ .../StateMachine/Queries/DatabaseQuery.cs | 21 ++ .../StateMachine/Queries/LeaderQuery.cs | 28 +++ .../StateMachine/Queries/QueryHelpers.cs | 28 +++ .../StateMachine/Snapshot.Database.cs | 61 +++++ .../StateMachine/Snapshot.Node.cs | 102 ++++++++ .../StateMachine/Snapshot.RefCounter.cs | 38 +++ .../StateMachine/Snapshot.Schema.cs | 144 +++++++++++ .../StateMachine/Snapshot.Versioning.cs | 13 + .../StateMachine/Snapshot.cs | 92 ++++++++ 38 files changed, 1215 insertions(+), 465 deletions(-) rename src/KurrentDB.KontrolPlane/{IDatabase.cs => Database.cs} (51%) create mode 100644 src/KurrentDB.KontrolPlane/DatabaseCluster.cs create mode 100644 src/KurrentDB.KontrolPlane/DatabaseLeader.cs create mode 100644 src/KurrentDB.KontrolPlane/DatabaseNode.cs delete mode 100644 src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs rename src/KurrentDB.KontrolPlane/{IDatabaseNode.cs => KontrollerEntity.cs} (65%) delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/CommandInfo.cs delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Database.cs delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs rename src/KurrentDB.KontrolPlane/StateMachine/{ => LogEntries}/IProtobufSerializable.cs (95%) create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/DatabaseQuery.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 4fbc0badee1..e23e99d6f25 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -15,7 +15,7 @@ - + diff --git a/src/KurrentDB.KontrolPlane/IDatabase.cs b/src/KurrentDB.KontrolPlane/Database.cs similarity index 51% rename from src/KurrentDB.KontrolPlane/IDatabase.cs rename to src/KurrentDB.KontrolPlane/Database.cs index fafe974afb7..e80ff0563c0 100644 --- a/src/KurrentDB.KontrolPlane/IDatabase.cs +++ b/src/KurrentDB.KontrolPlane/Database.cs @@ -3,12 +3,18 @@ namespace KurrentDB.KontrolPlane; -public interface IDatabase { +/// +/// Describes Kurrent database. +/// +public class Database : KontrollerEntity { public const string MainDatabaseId = "main"; - string Description { get; } + public required string Id { get; init; } - ulong Epoch { get; } + public ulong Epoch { get; init; } - IReadOnlyList Nodes { get; } + public string Description { + get => field ?? string.Empty; + init; + } } diff --git a/src/KurrentDB.KontrolPlane/DatabaseCluster.cs b/src/KurrentDB.KontrolPlane/DatabaseCluster.cs new file mode 100644 index 00000000000..a74ef718db1 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/DatabaseCluster.cs @@ -0,0 +1,18 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; + +namespace KurrentDB.KontrolPlane; + +/// +/// Represents instant state of the database. +/// +public sealed class DatabaseCluster : Database { + public IReadOnlyList Nodes { + get => field ?? []; + init; + } + + public EndPoint? LeaderAddress { get; init; } +} diff --git a/src/KurrentDB.KontrolPlane/DatabaseLeader.cs b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs new file mode 100644 index 00000000000..720cc36d16a --- /dev/null +++ b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs @@ -0,0 +1,13 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane; + +/// +/// Represents database leader. +/// +public sealed class DatabaseLeader : KontrollerEntity { + public required DatabaseNode Node { get; init; } + + public required ulong Epoch { get; init; } +} diff --git a/src/KurrentDB.KontrolPlane/DatabaseNode.cs b/src/KurrentDB.KontrolPlane/DatabaseNode.cs new file mode 100644 index 00000000000..729e8aa84f0 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/DatabaseNode.cs @@ -0,0 +1,17 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; + +namespace KurrentDB.KontrolPlane; + +/// +/// Describes database node. +/// +public sealed class DatabaseNode : KontrollerEntity { + public required string DatabaseId { get; init; } + + public required EndPoint Address { get; init; } + + public bool IsReadOnlyReplica { get; init; } +} diff --git a/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs b/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs deleted file mode 100644 index 4a910865e1e..00000000000 --- a/src/KurrentDB.KontrolPlane/DatabaseSnapshot.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -namespace KurrentDB.KontrolPlane; - -/// -/// Represents instant state of the database. -/// -public readonly record struct DatabaseSnapshot { - /// - /// Gets the database leader. - /// - /// if the leader is not assigned. - public IDatabaseNode? Leader { get; init; } - - /// - /// Gets the entire database. - /// - /// if the database is removed. - public required IDatabase? Database { get; init; } -} diff --git a/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs b/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs index 982883eea55..5eed74298bf 100644 --- a/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs +++ b/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs @@ -5,6 +5,15 @@ namespace KurrentDB.KontrolPlane; +/// +/// Manages communication with the member in the replica set. +/// public interface IDatabaseReplicaSet { + /// + /// Gets the replication state for the specified member. + /// + /// The address of the member. + /// The token that can be used to cancel the operation. + /// The replication state of the member. ValueTask GetReplicaStateAsync(EndPoint address, CancellationToken token); } diff --git a/src/KurrentDB.KontrolPlane/IKontroller.cs b/src/KurrentDB.KontrolPlane/IKontroller.cs index 47ad90dd544..d1daaec5a6e 100644 --- a/src/KurrentDB.KontrolPlane/IKontroller.cs +++ b/src/KurrentDB.KontrolPlane/IKontroller.cs @@ -6,22 +6,29 @@ namespace KurrentDB.KontrolPlane; public interface IKontroller { - /// - /// Gets a list of the databases registered in the Kontrol Plane. - /// - IReadOnlyDictionary Databases { get; } - IDatabaseReplicaSet ReplicaSet { get; init; } + ValueTask> GetDatabasesAsync(CancellationToken token = default); + + ValueTask> GetDatabaseNodesAsync(string databaseId, CancellationToken token = default); + + ValueTask GetDatabaseLeaderAsync(string databaseId, CancellationToken token = default); + ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default); - ValueTask AddOrUpdateDatabaseAsync(string databaseId, string description = "", CancellationToken token = default); + ValueTask AddOrUpdateDatabaseAsync(Database database, CancellationToken token = default); ValueTask RemoveDatabaseAsync(string databaseId, CancellationToken token = default); - ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, IDatabaseNode node, CancellationToken token = default); + ValueTask AddOrUpdateDatabaseNodeAsync(DatabaseNode node, CancellationToken token = default); ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint address, CancellationToken token = default); - IAsyncEnumerable ListenDatabaseAsync(string databaseId, CancellationToken token = default); + /// + /// Listens for database changes. + /// + /// The identifier of the database. + /// The token that can be used to cancel the operation. + /// A stream of full database snapshot. The stream finishes if becomes deleted. + IAsyncEnumerable ListenDatabaseAsync(string databaseId, CancellationToken token = default); } diff --git a/src/KurrentDB.KontrolPlane/IDatabaseNode.cs b/src/KurrentDB.KontrolPlane/KontrollerEntity.cs similarity index 65% rename from src/KurrentDB.KontrolPlane/IDatabaseNode.cs rename to src/KurrentDB.KontrolPlane/KontrollerEntity.cs index 3cd739e15c9..82dbabaee55 100644 --- a/src/KurrentDB.KontrolPlane/IDatabaseNode.cs +++ b/src/KurrentDB.KontrolPlane/KontrollerEntity.cs @@ -1,12 +1,9 @@ // Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). -using System.Net; - namespace KurrentDB.KontrolPlane; -public interface IDatabaseNode { - EndPoint Address { get; } - - bool IsReadOnlyReplica { get; } +public abstract class KontrollerEntity { + private protected KontrollerEntity() { + } } diff --git a/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj index 53af0942fc9..5051791b853 100644 --- a/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj +++ b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj @@ -11,12 +11,13 @@ + + - diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 2ed75126c92..8fd9724a10a 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -9,6 +9,8 @@ namespace KurrentDB.KontrolPlane; using StateMachine; +using StateMachine.LogEntries; +using StateMachine.Queries; partial class RaftKontroller { // key is database ID, value is the time when the leadership was updated for the particular database @@ -17,17 +19,19 @@ partial class RaftKontroller { // Spin in the loop and process appointments for every database private async ValueTask ProcessAppointmentsAsync(CancellationToken token) { var tasks = new List(17); + var databases = new HashSet(17); var deletedDatabases = new HashSet(); var timer = new PeriodicTimer(_appointmentExpiration); try { do { - var databases = _state.CurrentState.Databases; - StartAppointments(databases, tasks, token); + var snapshot = await _state.CaptureCurrentStateAsync(token); + StartAppointments(snapshot, tasks, databases, token); RemoveDeletedDatabases(databases, deletedDatabases); await Task.WhenAll(tasks); tasks.Clear(); deletedDatabases.Clear(); + databases.Clear(); } while (await timer.WaitForNextTickAsync(token)); } finally { timer.Dispose(); @@ -36,11 +40,11 @@ private async ValueTask ProcessAppointmentsAsync(CancellationToken token) { } private void RemoveDeletedDatabases( - IReadOnlyDictionary existingDatabases, + IReadOnlySet existingDatabases, HashSet deletedDatabases) { // Remove deleted databases from the appointment state foreach (var databaseId in _appointmentState.Keys) { - if (!existingDatabases.ContainsKey(databaseId)) + if (!existingDatabases.Contains(databaseId)) deletedDatabases.Add(databaseId); } @@ -50,22 +54,31 @@ private void RemoveDeletedDatabases( } private void StartAppointments( - IReadOnlyDictionary databases, + Snapshot snapshot, List tasks, + HashSet databases, CancellationToken token) { // Process appointment for every database in parallel - foreach (var (databaseId, database) in databases) { - if (IsAppointmentRequired(databaseId, database)) - tasks.Add(AppointLeaderAsync(databaseId, database.Nodes, database.Version, token)); + using (snapshot.RentConnection(out var connection)) { + foreach (var database in connection.GetDatabases()) { + databases.Add(database.Id); + + IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes = connection + .GetDatabaseNodes(database.Id) + .ToList(); + + if (IsAppointmentRequired(database.Id, nodes)) + tasks.Add(AppointLeaderAsync(database.Id, nodes, token)); + } } } - private bool IsAppointmentRequired(string databaseId, Database database) - => database.Nodes is not [] + private bool IsAppointmentRequired(string databaseId, IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes) + => nodes is not [] && (!_appointmentState.TryGetValue(databaseId, out var appointment) || appointment.IsExpired(_appointmentExpiration)); - private async Task AppointLeaderAsync(string databaseId, IReadOnlyList nodes, ulong expectedVersion, CancellationToken token) { + private async Task AppointLeaderAsync(string databaseId, IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) { var responses = new Dictionary(nodes.Count); await foreach (var task in Task.WhenEach(GetReplicaState(ReplicaSet, nodes, token))) { @@ -94,13 +107,12 @@ private async Task AppointLeaderAsync(string databaseId, IReadOnlyList>> GetReplicaState( IDatabaseReplicaSet replicas, - IEnumerable nodes, + IEnumerable<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) => nodes .Where(static node => !node.IsReadOnlyReplica) // r/o replicas cannot contribute to the qorum diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index d849c0e3c7f..3f3b160d334 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -4,65 +4,101 @@ using System.Net; using System.Runtime.CompilerServices; using DotNext.Net.Cluster.Consensus.Raft; +using Kurrent.Quack; +using KurrentDB.KontrolPlane.StateMachine; namespace KurrentDB.KontrolPlane; +using StateMachine.Queries; using static StateMachine.LogEntries.ReplicationHelpers; partial class RaftKontroller : IKontroller { - public IReadOnlyDictionary Databases => _state.CurrentState; + public async ValueTask> GetDatabasesAsync(CancellationToken token = default) { + var snapshot = await _state.CaptureCurrentStateAsync(token); + var result = new List(); + try { + using (snapshot.RentConnection(out var connection)) { + foreach (var database in connection.GetDatabases()) { + result.Add(new Database { Id = database.Id, Description = database.Description, Epoch = database.Epoch }); + } + } + } finally { + snapshot.Release(); + } + + return result; + } + + public async ValueTask> GetDatabaseNodesAsync(string databaseId, CancellationToken token = default) { + var snapshot = await _state.CaptureCurrentStateAsync(token); + var result = new List(); + try { + using (snapshot.RentConnection(out var connection)) { + foreach (var database in connection.GetDatabaseNodes(databaseId)) { + result.Add(new DatabaseNode + { DatabaseId = databaseId, Address = database.Address, IsReadOnlyReplica = database.IsReadOnlyReplica }); + } + } + } finally { + snapshot.Release(); + } - public async ValueTask AddOrUpdateDatabaseAsync(string databaseId, string description = "", CancellationToken token = default) { - var completion = new TaskCompletionSource(); + return result; + } + + public async ValueTask GetDatabaseLeaderAsync(string databaseId, CancellationToken token = default) { + var snapshot = await _state.CaptureCurrentStateAsync(token); try { - await _raft.AddOrUpdateDatabaseAsync(databaseId, description, completion, token); + using (snapshot.RentConnection(out var connection)) { + return connection + .GetDatabaseLeader(databaseId) + .FirstOrDefault() + .TryGet(out var leader) + ? new DatabaseLeader { + Epoch = leader.Epoch, + Node = new() { + Address = leader.Address, + DatabaseId = databaseId, + IsReadOnlyReplica = leader.IsReadOnlyReplica + } + } + : null; + } + } finally { + snapshot.Release(); + } + } + + public async ValueTask AddOrUpdateDatabaseAsync(Database database, CancellationToken token = default) { + try { + await _raft.AddOrUpdateDatabaseAsync(database.Id, database.Description, token); } catch (NotLeaderException e) { throw new LeadershipRequiredException(e); } - - return await completion.Task.WaitAsync(token); } public async ValueTask RemoveDatabaseAsync(string databaseId, CancellationToken token = default) { - var completion = new TaskCompletionSource(); try { - await _raft.RemoveDatabaseAsync(databaseId, completion, token); + return await _raft.RemoveDatabaseAsync(databaseId, token); } catch (NotLeaderException e) { throw new LeadershipRequiredException(e); } - - return await completion.Task.WaitAsync(token); } - public async ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, IDatabaseNode node, CancellationToken token = default) { - var completion = new TaskCompletionSource(); + public async ValueTask AddOrUpdateDatabaseNodeAsync(DatabaseNode node, CancellationToken token = default) { try { - await _raft.AddOrUpdateDatabaseNodeAsync(databaseId, node, completion, token); + await _raft.AddOrUpdateDatabaseNodeAsync(node.DatabaseId, node.Address, node.IsReadOnlyReplica, token); } catch (NotLeaderException e) { throw new LeadershipRequiredException(e); } - - return await completion.Task.WaitAsync(token); } public async ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint address, CancellationToken token = default) { - var completion = new TaskCompletionSource(); try { - await _raft.RemoveDatabaseNodeAsync(databaseId, address, completion, token); + return await _raft.RemoveDatabaseNodeAsync(databaseId, address, token); } catch (NotLeaderException e) { throw new LeadershipRequiredException(e); } - - return await completion.Task.WaitAsync(token); - } - - private async ValueTask AppointLeaderAsync(string databaseId, - EndPoint address, - ulong expectedVersion, - CancellationToken token = default) { - var completion = new TaskCompletionSource(); - await _raft.AppointLeaderAsync(databaseId, address, expectedVersion, completion, token); - return await completion.Task.WaitAsync(token); } public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default) { @@ -76,18 +112,52 @@ public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint l return task; } - public async IAsyncEnumerable ListenDatabaseAsync(string databaseId, [EnumeratorCancellation] CancellationToken token = default) { - yield break; - // var leadershiptToken = _raft.LeadershipToken; - // var tokenSource = _multiplexer.Combine(_raft.LeadershipToken, token); - // try { - // yield return null; - // } catch (OperationCanceledException e) when (e.CausedBy(tokenSource, leadershiptToken)) { - // throw new LeadershipRequiredException(e); - // } catch (OperationCanceledException e) when (e.CancellationToken == tokenSource.CancellationOrigin) { - // throw new OperationCanceledException(e.Message, e, e.CancellationToken); - // } finally { - // await tokenSource.DisposeAsync(); - // } + public async IAsyncEnumerable ListenDatabaseAsync(string databaseId, [EnumeratorCancellation] CancellationToken token = default) { + await foreach (var snapshot in _state.TrackChangesAsync(databaseId, token)) { + try { + if (GetDatabaseCluster(snapshot, databaseId) is { } cluster) { + yield return cluster; + } else { + break; + } + } finally { + snapshot.Release(); + } + } + + static DatabaseCluster? GetDatabaseCluster(Snapshot snapshot, + string databaseId) { + using (snapshot.RentConnection(out var connection)) { + return connection.GetDatabase(databaseId).FirstOrDefault().TryGet(out var database) + ? new() { + Nodes = GetDatabaseNodes(connection, databaseId, out var leaderAddress), + LeaderAddress = leaderAddress, + Id = databaseId, + Epoch = database.Epoch, + Description = database.Description + } + : null; + } + } + + static IReadOnlyList GetDatabaseNodes(DuckDBAdvancedConnection connection, + string databaseId, + out EndPoint? leader) { + var nodes = new List(); + leader = null; + + foreach (var node in connection.GetDatabaseNodes(databaseId)) { + nodes.Add(new() { + Address = node.Address, + DatabaseId = databaseId, + IsReadOnlyReplica = node.IsReadOnlyReplica + }); + + if (node.IsLeader) + leader = node.Address; + } + + return nodes; + } } } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs index 4fcc162d5c7..7028b51706b 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs @@ -26,6 +26,11 @@ public EndPoint PublicAddress { init; } + public int ConnectionPoolCapacity { + get => field > 0 ? field : 10; + init => field = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value)); + } + public required TimeSpan AppointmentExpiration { get; init => field = value > TimeSpan.Zero ? value : throw new ArgumentOutOfRangeException(nameof(value)); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 16903b744af..2afa2c2e771 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -23,7 +23,7 @@ public partial class RaftKontroller : IAsyncDisposable { public RaftKontroller(in Options options) { var stateLocation = new DirectoryInfo(Path.Combine(options.WalOptions.Location, "db")); var configStorageLocation = Path.Combine(options.WalOptions.Location, "members.list"); - _state = new(stateLocation); + _state = new(stateLocation, options.ConnectionPoolCapacity); _wal = new WriteAheadLog(options.WalOptions, _state); var config = new RaftCluster.TcpConfiguration(options.ListenAddress) { @@ -59,6 +59,6 @@ public async Task StopAsync(CancellationToken token) { public async ValueTask DisposeAsync() { await _raft.DisposeAsync(); await _wal.DisposeAsync(); - await _state.DisposeAsync(); + _state.Dispose(); } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs deleted file mode 100644 index 2e13259ff0b..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using Google.Protobuf; - -namespace KurrentDB.KontrolPlane.StateMachine; - -partial class ClusterState : IReadOnlyDictionary, IProtobufSerializable { - public ClusterState AddMainDatabase(out Database database) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - database = new Database { Version = copy.Version }; - copy.Databases.Add(IDatabase.MainDatabaseId, database); - return copy; - } - - public ClusterState AddDatabase(string databaseId, Database database) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - database.Version = copy.Version; - copy.Databases.Add(databaseId, database); - return copy; - } - - public ClusterState RemoveDatabase(string databaseId) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - return copy.Databases.Remove(databaseId) ? copy : this; - } - - public ClusterState AddDatabaseNode(string databaseId, DatabaseNode databaseNode) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - - if (!copy.Databases.TryGetValue(databaseId, out var database)) - return this; - - copy.Databases[databaseId] = database.AddNode(databaseNode); - return copy; - } - - public ClusterState UpdateDatabaseNode(string databaseId, DatabaseNode databaseNode, int index) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - - if (!copy.Databases.TryGetValue(databaseId, out var database)) - return this; - - copy.Databases[databaseId] = database.UpdateNode(databaseNode, index); - return copy; - } - - public ClusterState RemoveDatabaseNode(string databaseId, ByteString address) { - var copy = new ClusterState(this) { Version = Version + 1UL }; - - if (!copy.Databases.TryGetValue(databaseId, out var database) || - database.Nodes.Find(address, out var index) is null) - return this; - - database = new(database) { Version = copy.Version }; - database.Nodes.RemoveAt(index); - copy.Databases[databaseId] = database; - return copy; - } - - public ClusterState AppointLeader(string databaseId, ulong expectedVersion, ByteString address) { - if (!Databases.TryGetValue(databaseId, out var database) - || database.Version != expectedVersion - || database.Nodes.Find(address, out _) is null) - return this; - - return new(this) { - Version = Version + 1UL, - Databases = { [databaseId] = database.AppointLeader(address) }, - }; - } - - IEnumerator> IEnumerable>.GetEnumerator() { - foreach (var (key, value) in Databases) { - yield return new(key, value); - } - } - - IEnumerator IEnumerable.GetEnumerator() => Databases.GetEnumerator(); - - int IReadOnlyCollection>.Count => Databases.Count; - - bool IReadOnlyDictionary.ContainsKey(string key) => Databases.ContainsKey(key); - - bool IReadOnlyDictionary.TryGetValue(string key, [MaybeNullWhen(false)] out IDatabase value) { - if (Databases.TryGetValue(key, out var typedResult)) { - value = typedResult; - return true; - } - - value = null; - return false; - } - - IDatabase IReadOnlyDictionary.this[string key] => Databases[key]; - - IEnumerable IReadOnlyDictionary.Keys => Databases.Keys; - - IEnumerable IReadOnlyDictionary.Values => Databases.Values; -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs deleted file mode 100644 index 3012f9202b2..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Database.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -namespace KurrentDB.KontrolPlane.StateMachine; - -partial class ClusterStateMachine { - private bool AddDatabase(LogEntries.AddOrUpdateDatabase entry) { - var stateCopy = CurrentState; - - if (stateCopy.Databases.ContainsKey(entry.DatabaseId)) - return false; - - CurrentState = stateCopy.AddDatabase(entry.DatabaseId, new() { Description = entry.Description }); - return true; - } - - private bool RemoveDatabase(LogEntries.RemoveDatabase entry) { - var stateCopy = CurrentState; - - if (!stateCopy.Databases.ContainsKey(entry.DatabaseId)) - return false; - - CurrentState = stateCopy.RemoveDatabase(entry.DatabaseId); - return true; - } - - private bool AppointLeader(LogEntries.AppointLeader entry) { - return TrySetCurrentState(CurrentState.AppointLeader(entry.DatabaseId, entry.ExpectedVersion, entry.Address)); - } -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs new file mode 100644 index 00000000000..782b07e1460 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs @@ -0,0 +1,128 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using DotNext.IO; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; +using DotNext.Threading; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class ClusterStateMachine { + // Apply log entry from the WAL to the current state + private async ValueTask ApplyAsync(LogEntry entry, CancellationToken token) { + var currentState = _snapshot; + var commandInfo = new CommandInfo(entry); + switch (entry.CommandId) { + case LogEntries.AddOrUpdateDatabase.TypeId: + Apply(currentState, + await DeserializeAsync(entry, token), + in commandInfo); + break; + case LogEntries.RemoveDatabase.TypeId: + Apply(currentState, + await DeserializeAsync(entry, token), + in commandInfo, + entry.Context as StrongBox); + break; + case LogEntries.AddOrUpdateDatabaseNode.TypeId: + Apply(currentState, + await DeserializeAsync(entry, token), + in commandInfo); + break; + case LogEntries.RemoveDatabaseNode.TypeId: + Apply(currentState, + await DeserializeAsync(entry, token), + in commandInfo, + entry.Context as StrongBox); + break; + case LogEntries.AppointLeader.TypeId: + Apply(currentState, + await DeserializeAsync(entry, token), + in commandInfo); + break; + default: + Debug.Fail($"Unexpected entry type {entry.CommandId}"); + break; + } + + // Produce snapshot in the background if needed + if (entry.Index % SnapshotDepth is 0 && _snapshotTask is null or { IsCompleted: true }) { + currentState.ReclaimGarbage(); + _snapshotTask = SaveSnapshotAsync(currentState, currentState.LastAppliedCommand, token); + } + + return entry.Index; + } + + private void Apply(Snapshot currentState, LogEntries.AddOrUpdateDatabase command, in CommandInfo commandInfo) { + currentState.Update(command, in commandInfo); + + _databases.GetOrAdd(command.DatabaseId, static _ => new AsyncStateTracker()).TryAdvance(); + } + + private void Apply(Snapshot currentState, + LogEntries.RemoveDatabase command, + in CommandInfo commandInfo, + StrongBox? resultContainer) { + var result = currentState.Update(command, in commandInfo); + + if (_databases.TryRemove(command.DatabaseId, out var tracker)) { + tracker.TryComplete(); + } + + resultContainer?.Value = result; + } + + private void Apply(Snapshot currentState, LogEntries.AddOrUpdateDatabaseNode command, in CommandInfo commandInfo) { + currentState.Update(command, in commandInfo); + + if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { + tracker.TryAdvance(); + } + } + + private void Apply(Snapshot currentState, + LogEntries.RemoveDatabaseNode command, + in CommandInfo commandInfo, + StrongBox? resultContainer) { + var result = currentState.Update(command, in commandInfo); + + if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { + tracker.TryAdvance(); + } + + resultContainer?.Value = result; + } + + private void Apply(Snapshot currentState, LogEntries.AppointLeader command, in CommandInfo commandInfo) { + currentState.Update(command, in commandInfo); + + if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { + tracker.TryAdvance(); + } + } + + private static ValueTask DeserializeAsync(in LogEntry entry, CancellationToken token) + where T : class, LogEntries.IProtobufSerializable { + ValueTask task; + if (entry.TryGetPayload(out var sequence)) { + // fast path, deserialize from memory + try { + task = new(T.Parser.ParseFrom(sequence)); + } catch (Exception e) { + task = ValueTask.FromException(e); + } + } else { + task = DeserializeSlowAsync(entry, token); + } + + return task; + + static async ValueTask DeserializeSlowAsync(LogEntry entry, CancellationToken token) { + using var owner = await entry.ToMemoryAsync(token: token); + return T.Parser.ParseFrom(owner.Memory.Span); + } + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs deleted file mode 100644 index 205615dfd87..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Node.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -namespace KurrentDB.KontrolPlane.StateMachine; - -partial class ClusterStateMachine { - private bool AddDatabaseNode(LogEntries.AddOrUpdateDatabaseNode entry) { - var stateCopy = CurrentState; - - if (!stateCopy.Databases.TryGetValue(entry.DatabaseId, out var database)) { - if (entry.DatabaseId is not IDatabase.MainDatabaseId) - return false; - - // Add main database automatically - stateCopy = stateCopy.AddMainDatabase(out database); - } - - if (database.Nodes.Find(entry.Address, out var index) is { } node) { - // update existing node - node = new DatabaseNode(node) { IsReadOnlyReplica = entry.IsReadOnlyReplica }; - stateCopy = stateCopy.UpdateDatabaseNode(entry.DatabaseId, node, index); - } else { - // create new node - node = new DatabaseNode { Address = entry.Address, IsReadOnlyReplica = entry.IsReadOnlyReplica }; - stateCopy = stateCopy.AddDatabaseNode(entry.DatabaseId, node); - } - - CurrentState = stateCopy; - return true; - } - - private bool RemoveDatabaseNode(LogEntries.RemoveDatabaseNode entry) { - var stateCopy = CurrentState; - - if (!stateCopy.Databases.ContainsKey(entry.DatabaseId)) - return false; - - CurrentState = stateCopy.RemoveDatabaseNode(entry.DatabaseId, entry.Address); - return true; - } -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs new file mode 100644 index 00000000000..6bba0356488 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs @@ -0,0 +1,111 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using DotNext.IO; +using DotNext.Net.Cluster.Consensus.Raft; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; +using static System.Globalization.CultureInfo; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class ClusterStateMachine { + private volatile SnapshotFile? _persistentSnapshot; + + ISnapshot? ISnapshotManager.Snapshot => _persistentSnapshot; + + ValueTask ISnapshotManager.ReclaimGarbageAsync(long watermark, CancellationToken token) { + var task = ValueTask.CompletedTask; + try { + ReclaimGarbage(watermark); + } catch (Exception e) { + task = ValueTask.FromException(e); + } + + return task; + } + + private void ReclaimGarbage(long watermark) { + var snapshots = new List(); + foreach (var snapshotFile in _location.EnumerateFiles()) { + if (long.TryParse(snapshotFile.Name, out var snapshotIndex) && snapshotIndex < watermark) { + snapshots.Add(snapshotFile); + } + } + + foreach (var snapshotFile in snapshots) { + snapshotFile.Delete(); + } + } + + private async ValueTask InstallSnapshotAsync(LogEntry entry, CancellationToken token) { + var fileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + var fs = new FileStream(fileName, new FileStreamOptions { + Access = FileAccess.Write, + Mode = FileMode.CreateNew, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + PreallocationSize = entry.Length.GetValueOrDefault(), + Share = FileShare.None, + }); + try { + // save snapshot to the file + await entry.WriteToAsync(fs, token: token); + await fs.FlushAsync(token); + + InstallSnapshot(fileName); + } finally { + await fs.DisposeAsync(); + File.Delete(fileName); + } + + return entry.Index; + } + + private Snapshot InstallSnapshot(string fileName) { + var newSnapshot = new Snapshot(_poolCapacity); + newSnapshot.LoadFromFile(fileName); + + // swap current state + Interlocked.Exchange(ref _snapshot, newSnapshot).Release(); + RefreshDatabaseTrackers(newSnapshot); + return newSnapshot; + } + + private Task SaveSnapshotAsync(Snapshot snapshot, CommandInfo info, CancellationToken token) + => Task.Run(() => SaveSnapshot(snapshot, info), token); + + private void SaveSnapshot(Snapshot snapshot, in CommandInfo info) { + var snapshotFileName = Path.Combine(_location.FullName, info.Index.ToString(InvariantCulture)); + try { + snapshot.SaveToFile(snapshotFileName); + } catch when (File.Exists(snapshotFileName)) { + File.Delete(snapshotFileName); + throw; + } + + _persistentSnapshot = new(snapshotFileName, info); + } + + private sealed class SnapshotFile(string fileName, in CommandInfo info) : ISnapshot { + private readonly FileInfo _file = new(fileName); + private readonly CommandInfo _info = info; + + async ValueTask IDataTransferObject.WriteToAsync(TWriter writer, CancellationToken token) { + await using var fs = _file.Open(new FileStreamOptions { + Access = FileAccess.Read, + Mode = FileMode.Open, + Share = FileShare.Read, + Options = FileOptions.Asynchronous | FileOptions.SequentialScan, + }); + + await writer.CopyFromAsync(fs, token: token); + } + + bool IDataTransferObject.IsReusable => true; + + long? IDataTransferObject.Length => _file.Length; + + long IRaftLogEntry.Term => _info.Term; + + long ISnapshot.Index => _info.Index; + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs index 6dba63807d7..c6521c9f2d8 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -1,36 +1,86 @@ // Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). -using System.Diagnostics; +using System.Collections.Concurrent; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using DotNext; -using DotNext.IO; using DotNext.Net.Cluster.Consensus.Raft.StateMachine; using DotNext.Threading; namespace KurrentDB.KontrolPlane.StateMachine; +using Queries; + /// /// Represents internal Kontrol Plane database. /// -/// -internal sealed partial class ClusterStateMachine(DirectoryInfo location) : SimpleStateMachine(location) { - private readonly AsyncTrigger _stateChanged = new(); - private ClusterState _state = new(); - - public ClusterState CurrentState { - get => Volatile.Read(in _state); - private set => TrySetCurrentState(value); +internal sealed partial class ClusterStateMachine : Disposable, IStateMachine { + private readonly DirectoryInfo _location; + private readonly int _poolCapacity; + private readonly ConcurrentDictionary _databases; + private volatile Snapshot _snapshot; + private Task? _snapshotTask; + + public ClusterStateMachine(DirectoryInfo location, int connectionPoolCapacity) { + if (!location.Exists) + location.Create(); + + _location = location; + _snapshot = new(connectionPoolCapacity); + _databases = new(); + _poolCapacity = connectionPoolCapacity; } - private bool TrySetCurrentState(ClusterState newState) { - if (ReferenceEquals(Volatile.Read(in _state), newState)) - return false; + /// + /// Recovers the internal state from the last known persisted snapshot. + /// + public void Recover() { + // Attempt to open the latest persisted snapshot + var snapshots = new SortedDictionary(); + foreach (var snapshotFile in _location.EnumerateFiles()) { + if (long.TryParse(snapshotFile.Name, out var snapshotIndex)) { + snapshots[snapshotIndex] = snapshotFile; + } + } + + if (snapshots.Count > 0) { + var latestSnapshotFile = snapshots.MaxBy(static pair => pair.Key).Value; - _state = newState; - _stateChanged.Signal(resumeAll: true); // acts as a barrier - return true; + var newSnapshot = InstallSnapshot(latestSnapshotFile.FullName); + _persistentSnapshot = new(latestSnapshotFile.FullName, newSnapshot.LastAppliedCommand); + } + + snapshots.Clear(); // help GC + } + + private void RefreshDatabaseTrackers(Snapshot snapshot) { + var loadedTrackers = new HashSet(); + using (snapshot.RentConnection(out var connection)) { + foreach (var database in connection.GetDatabases()) { + loadedTrackers.Add(database.Id); + } + } + + // add missing trackers + foreach (var databaseId in loadedTrackers) { + if (!_databases.ContainsKey(databaseId)) { + var tracker = new AsyncStateTracker(); + if (!_databases.TryAdd(databaseId, tracker)) { + tracker.TryComplete(); + } + } + } + + // remove deleted trackers + foreach (var databaseId in _databases.Keys + .Where(databaseId => !loadedTrackers.Contains(databaseId)) + .ToHashSet()) { + if (_databases.TryRemove(databaseId, out var tracker)) { + tracker.TryComplete(); + } + } + + loadedTrackers.Clear(); // help GC } /// @@ -43,105 +93,82 @@ public int SnapshotDepth { } = 100; /// - /// Tracks cluster state changes as a stream of state snapshots. + /// Tracks database changes as a stream of state snapshots. /// + /// + /// The caller must release the snapshot with method. + /// + /// The identifier of the database. /// The token that can be used to cancel the operation. /// A stream over cluster state snapshots. - public async IAsyncEnumerable> TrackChangesAsync( + public async IAsyncEnumerable TrackChangesAsync(string databaseId, [EnumeratorCancellation] CancellationToken token) { - do { - var actual = CurrentState; - yield return actual; - await _stateChanged.SpinWaitAsync(new DatabaseChangeTracker(this) { ExpectedVersion = actual.Version }, token); - } while (!token.IsCancellationRequested); - } - - // Restore state from the snapshot - protected override ValueTask RestoreAsync(FileInfo snapshotFile, CancellationToken token) { - var task = ValueTask.CompletedTask; - var fs = default(FileStream); - try { - fs = snapshotFile.Open(new FileStreamOptions { - Mode = FileMode.Open, - Access = FileAccess.Read, - Share = FileShare.Read, - BufferSize = 4096, - Options = FileOptions.SequentialScan - }); - - _state = ClusterState.Parser.ParseFrom(fs); - _stateChanged.Signal(resumeAll: true); - } catch (Exception e) { - task = ValueTask.FromException(e); - } finally { - fs?.Dispose(); - } + for (AsyncStateTracker.Token currentState;; token.ThrowIfCancellationRequested()) { + if (!_databases.TryGetValue(databaseId, out var tracker) || IsDisposingOrDisposed) + break; - return task; - } + currentState = tracker.CurrentState; + var snapshotCopy = _snapshot; - // Persist current state - protected override ValueTask PersistAsync(IAsyncBinaryWriter writer, CancellationToken token) - => _state.WriteToAsync(writer, token); + // The current snapshot cannot be acquired, which means that it's no longer available. Retry the operation + // and do Yield() to increase a chance to get latest snapshot copy from '_snapshot' field + if (!snapshotCopy.TryAcquire()) { + await Task.Yield(); + continue; + } - // Apply log entry from the WAL to the current state - protected override async ValueTask ApplyAsync(LogEntry entry, CancellationToken token) { - switch (entry.CommandId) { - case LogEntries.AddOrUpdateDatabase.TypeId: - var result = AddDatabase(await DeserializeAsync(entry, token)); - (entry.Context as TaskCompletionSource)?.TrySetResult(result); - break; - case LogEntries.RemoveDatabase.TypeId: - result = RemoveDatabase(await DeserializeAsync(entry, token)); - (entry.Context as TaskCompletionSource)?.TrySetResult(result); + yield return snapshotCopy; + if (!await tracker.WaitNextAsync(currentState, token)) break; - case LogEntries.AddOrUpdateDatabaseNode.TypeId: - result = AddDatabaseNode(await DeserializeAsync(entry, token)); - (entry.Context as TaskCompletionSource)?.TrySetResult(result); - break; - case LogEntries.RemoveDatabaseNode.TypeId: - result = RemoveDatabaseNode(await DeserializeAsync(entry, token)); - (entry.Context as TaskCompletionSource)?.TrySetResult(result); - break; - case LogEntries.AppointLeader.TypeId: - result = AppointLeader(await DeserializeAsync(entry, token)); - break; - default: - Debug.Fail($"Unexpected entry type {entry.CommandId}"); + } + } + + /// + /// Captures the current database state asynchronously. + /// + /// + /// The caller must release the snapshot with method. + /// + /// The token that can be used to cancel the operation. + /// The acquired snapshot. + public async ValueTask CaptureCurrentStateAsync(CancellationToken token) { + Snapshot snapshotCopy; + for (;; token.ThrowIfCancellationRequested()) { + snapshotCopy = _snapshot; + if (snapshotCopy.TryAcquire()) break; + + // The current snapshot cannot be acquired, which means that it's no longer available. Retry the operation + // and do Yield() to increase a chance to get latest snapshot copy from '_snapshot' field + await Task.Yield(); } - return entry.Index % SnapshotDepth is 0; + return snapshotCopy; } - private static ValueTask DeserializeAsync(in LogEntry entry, CancellationToken token) - where T : class, IProtobufSerializable { - ValueTask task; - if (entry.TryGetPayload(out var sequence)) { - // fast path, deserialize from memory - try { - task = new(T.Parser.ParseFrom(sequence)); - } catch (Exception e) { - task = ValueTask.FromException(e); - } - } else { - task = DeserializeSlowAsync(entry, token); - } + ValueTask IStateMachine.ApplyAsync(LogEntry entry, CancellationToken token) { + var lastAppliedIndex = _snapshot.LastAppliedCommand.Index; + if (entry.Index <= lastAppliedIndex) + return ValueTask.FromResult(lastAppliedIndex); - return task; + return entry.IsSnapshot + ? InstallSnapshotAsync(entry, token) + : ApplyAsync(entry, token); + } - static async ValueTask DeserializeSlowAsync(LogEntry entry, CancellationToken token) { - using var owner = await entry.ToMemoryAsync(token: token); - return T.Parser.ParseFrom(owner.Memory.Span); + private void CompleteTrackers() { + foreach (var tracker in _databases.Values) { + tracker.TryComplete(); } } - [StructLayout(LayoutKind.Auto)] - private readonly struct DatabaseChangeTracker(ClusterStateMachine stateMachine) : ISupplier { - private ulong ActualVersion => stateMachine.CurrentState.Version; - - public required ulong ExpectedVersion { get; init; } + protected override void Dispose(bool disposing) { + if (disposing) { + _snapshot.Release(); + CompleteTrackers(); + _databases.Clear(); + } - bool ISupplier.Invoke() => ActualVersion == ExpectedVersion; + base.Dispose(disposing); } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/CommandInfo.cs b/src/KurrentDB.KontrolPlane/StateMachine/CommandInfo.cs new file mode 100644 index 00000000000..4ad6ca3d29c --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/CommandInfo.cs @@ -0,0 +1,14 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using DotNext.Net.Cluster.Consensus.Raft.StateMachine; + +namespace KurrentDB.KontrolPlane.StateMachine; + +[StructLayout(LayoutKind.Auto)] +internal readonly record struct CommandInfo(long Index, long Term) { + public CommandInfo(in LogEntry entry) + : this(entry.Index, entry.Term) { + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/Database.cs deleted file mode 100644 index aed31f443af..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/Database.cs +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -using Google.Protobuf; - -namespace KurrentDB.KontrolPlane.StateMachine; - -partial class Database : IDatabase { - IReadOnlyList IDatabase.Nodes => Nodes; - - public Database AddNode(DatabaseNode node) { - var copy = new Database(this) { Version = Version + 1UL }; - node.Version = copy.Version; - copy.Nodes.Add(node); - return copy; - } - - public Database UpdateNode(DatabaseNode node, int index) { - var copy = new Database(this) { Version = Version + 1UL }; - node.Version = copy.Version; - copy.Nodes[index] = node; - return copy; - } - - public Database AppointLeader(ByteString address) => new(this) { - Version = Version + 1UL, - Epoch = Epoch + 1UL, - LeaderAddress = address, - }; -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs b/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs deleted file mode 100644 index 1e94a131caf..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/DatabaseNode.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -using System.Net; -using Google.Protobuf; - -namespace KurrentDB.KontrolPlane.StateMachine; - -partial class DatabaseNode : IDatabaseNode { - EndPoint IDatabaseNode.Address => field ??= Address.ToEndPoint(); -} - -internal static class DatabaseNodeExtensions { - public static DatabaseNode? Find(this IReadOnlyList nodes, ByteString address, out int index) - => nodes.Find(address.AreEqual, out index); - - private static DatabaseNode? Find(this IReadOnlyList nodes, Predicate predicate, out int index) { - for (var i = 0; i < nodes.Count; i++) { - var result = nodes[i]; - if (predicate(result)) { - index = i; - return result; - } - } - - index = -1; - return null; - } - - private static bool AreEqual(this ByteString address, DatabaseNode node) => address.Span.SequenceEqual(node.Address.Span); -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs b/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs index 0b8746adce6..d41dae5aa4e 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/EndPointExtensions.cs @@ -5,12 +5,13 @@ using DotNext.Buffers; using DotNext.Net; using Google.Protobuf; +using Kurrent.Quack; namespace KurrentDB.KontrolPlane.StateMachine; internal static class EndPointExtensions { - public static EndPoint ToEndPoint(this ByteString bytes) { - var reader = new SequenceReader(bytes.Memory); + public static EndPoint ToEndPoint(this Blob bytes) { + var reader = new SequenceReader(bytes.Reference.AsMemory()); return reader.ReadEndPoint(); } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs similarity index 95% rename from src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs rename to src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs index 57f37052433..0e826fb235b 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/IProtobufSerializable.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs @@ -5,7 +5,7 @@ using DotNext.IO; using Google.Protobuf; -namespace KurrentDB.KontrolPlane.StateMachine; +namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; internal interface IProtobufSerializable : IMessage where TSelf : class, IProtobufSerializable { diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs index 5aa50f5067f..0d6015ac33b 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs @@ -2,54 +2,55 @@ // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). using System.Net; +using System.Runtime.CompilerServices; using DotNext.Net.Cluster.Consensus.Raft; namespace KurrentDB.KontrolPlane.StateMachine.LogEntries; internal static class ReplicationHelpers { - extension(IRaftCluster raft) - { + extension(IRaftCluster raft) { public ValueTask AddOrUpdateDatabaseAsync(string databaseId, string description, - TaskCompletionSource completion, CancellationToken token) => raft.ReplicateAsync( new ProtobufLogEntry(new() { DatabaseId = databaseId, Description = description }) - { Term = raft.Term, Context = completion }, token); + { Term = raft.Term }, token); - public ValueTask RemoveDatabaseAsync(string databaseId, - TaskCompletionSource completion, - CancellationToken token) - => raft.ReplicateAsync( + public async ValueTask RemoveDatabaseAsync(string databaseId, + CancellationToken token) { + var box = new StrongBox(); + await raft.ReplicateAsync( new ProtobufLogEntry(new() { DatabaseId = databaseId }) - { Term = raft.Term, Context = completion }, token); + { Term = raft.Term, Context = box }, token); + return box.Value; + } public ValueTask AddOrUpdateDatabaseNodeAsync(string databaseId, - IDatabaseNode node, - TaskCompletionSource completion, + EndPoint address, + bool isReadOnlyReplica, CancellationToken token) => raft.ReplicateAsync( new ProtobufLogEntry(new() - { Address = node.Address.ToByteString(), DatabaseId = databaseId, IsReadOnlyReplica = node.IsReadOnlyReplica }) - { Term = raft.Term, Context = completion }, token); + { Address = address.ToByteString(), DatabaseId = databaseId, IsReadOnlyReplica = isReadOnlyReplica }) + { Term = raft.Term }, token); - public ValueTask RemoveDatabaseNodeAsync(string databaseId, + public async ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint address, - TaskCompletionSource completion, - CancellationToken token) - => raft.ReplicateAsync( + CancellationToken token) { + var box = new StrongBox(); + await raft.ReplicateAsync( new ProtobufLogEntry(new() { Address = address.ToByteString(), DatabaseId = databaseId }) - { Term = raft.Term, Context = completion }, token); + { Term = raft.Term, Context = box }, token); + return box.Value; + } public ValueTask AppointLeaderAsync(string databaseId, EndPoint address, - ulong expectedVersion, - TaskCompletionSource completion, CancellationToken token) => raft.ReplicateAsync( new ProtobufLogEntry(new() - { Address = address.ToByteString(), DatabaseId = databaseId, ExpectedVersion = expectedVersion }) - { Term = raft.Term, Context = completion }, token); + { Address = address.ToByteString(), DatabaseId = databaseId }) + { Term = raft.Term }, token); } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto index 67f7011dddc..7e47d7889b4 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto @@ -6,33 +6,33 @@ package kurrent.kontrol_plane.log_entries; option csharp_namespace = "KurrentDB.KontrolPlane.StateMachine.LogEntries"; -message AddOrUpdateDatabaseNode +message AddOrUpdateDatabase { string databaseId = 1; - bytes address = 2; - bool isReadOnlyReplica = 3; + string description = 2; } -message RemoveDatabaseNode +message RemoveDatabase { string databaseId = 1; - bytes address = 2; } -message AddOrUpdateDatabase +message AddOrUpdateDatabaseNode { string databaseId = 1; - string description = 2; + bytes address = 2; + bool isReadOnlyReplica = 3; + uint64 version = 4; } -message RemoveDatabase +message RemoveDatabaseNode { string databaseId = 1; + bytes address = 2; } message AppointLeader { string databaseId = 1; - uint64 expectedVersion = 2; bytes address = 3; } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs new file mode 100644 index 00000000000..5112fe2ab21 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs @@ -0,0 +1,18 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +[StructLayout(LayoutKind.Auto)] +internal readonly struct AllDatabasesQuery : IQuery<(string Id, string Description, ulong Epoch)> { + public static ReadOnlySpan CommandText => "SELECT * FROM database;"u8; + + public static (string Id, string Description, ulong Epoch) Parse(ref DataChunk.Row row) => new() { + Id = row.ReadString(), + Description = row.ReadString(), + Epoch = row.ReadUInt64(), + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs new file mode 100644 index 00000000000..c269ea86f0a --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs @@ -0,0 +1,23 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using System.Runtime.InteropServices; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +[StructLayout(LayoutKind.Auto)] +internal readonly struct AllNodesQuery : IQuery, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> { + public static ReadOnlySpan CommandText => "SELECT (address, is_read_only_replica, is_leader) FROM node WHERE database_id=?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1, + }; + + public static (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader) Parse(ref DataChunk.Row row) => new() { + Address = row.ReadBlob().ToEndPoint(), + IsReadOnlyReplica = row.ReadBoolean(), + IsLeader = row.ReadBoolean(), + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/DatabaseQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/DatabaseQuery.cs new file mode 100644 index 00000000000..88ab6276482 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/DatabaseQuery.cs @@ -0,0 +1,21 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +[StructLayout(LayoutKind.Auto)] +internal readonly record struct DatabaseQuery : IQuery, (string Description, ulong Epoch)> { + public static ReadOnlySpan CommandText => "SELECT description, epoch FROM database WHERE id = ?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1, + }; + + public static (string Description, ulong Epoch) Parse(ref DataChunk.Row row) => new() { + Description = row.ReadString(), + Epoch = row.ReadUInt64(), + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs new file mode 100644 index 00000000000..4b9345eca16 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs @@ -0,0 +1,28 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using System.Runtime.InteropServices; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +[StructLayout(LayoutKind.Auto)] +internal readonly struct LeaderQuery : IQuery, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica)> { + public static ReadOnlySpan CommandText => """ + SELECT n.address, d.epoch, n.is_read_only_replica + FROM node n + JOIN database d ON n.database_id = d.id + WHERE n.is_leader=true AND d.id=? + """u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1 + }; + + public static (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica) Parse(ref DataChunk.Row row) => new() { + Address = row.ReadBlob().ToEndPoint(), + Epoch = row.ReadUInt64(), + IsReadOnlyReplica = row.ReadBoolean(), + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs new file mode 100644 index 00000000000..fb3a2536192 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs @@ -0,0 +1,28 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +internal static class QueryHelpers { + public static QueryResult<(string Id, string Description, ulong Epoch), AllDatabasesQuery> GetDatabases( + this DuckDBAdvancedConnection connection) + => connection.ExecuteQuery<(string, string, ulong), AllDatabasesQuery>(); + + public static QueryResult, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader), AllNodesQuery> GetDatabaseNodes( + this DuckDBAdvancedConnection connection, + string databaseId) + => connection.ExecuteQuery, (EndPoint, bool, bool), AllNodesQuery>(new(databaseId)); + + public static QueryResult, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica), LeaderQuery> GetDatabaseLeader( + this DuckDBAdvancedConnection connection, + string databaseId) + => connection.ExecuteQuery, (EndPoint, ulong, bool), LeaderQuery>(new(databaseId)); + + public static QueryResult, (string Description, ulong Epoch), DatabaseQuery> GetDatabase( + this DuckDBAdvancedConnection connection, + string databaseId) + => connection.ExecuteQuery, (string, ulong), DatabaseQuery>(new(databaseId)); +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs new file mode 100644 index 00000000000..48916825b4f --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs @@ -0,0 +1,61 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using DotNext; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine; + +using LogEntries; + +partial class Snapshot { + public void Update(AddOrUpdateDatabase command, in CommandInfo info) + => Update(new(command), info); + + public bool Update(RemoveDatabase command, in CommandInfo info) + => Update(new(command), info); +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct AddOrUpdateDatabaseStmt(AddOrUpdateDatabase command) : IPreparedStatement<(string DatabaseId, string Description)>, IConsumer { + public static ReadOnlySpan CommandText => """ + INSERT INTO database (id, description) + VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE + SET description = $2 + WHERE database.id = $3; + """u8; + + public static StatementBindingResult Bind(in (string DatabaseId, string Description) args, PreparedStatement source) + => new(source) { + args.DatabaseId, + args.Description, + }; + + public void Invoke(DuckDBAdvancedConnection connection) + => connection.ExecuteNonQuery<(string, string), AddOrUpdateDatabaseStmt>(new(command.DatabaseId, command.Description)); +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct RemoveDatabaseNodesStmt : IPreparedStatement> { + public static ReadOnlySpan CommandText => "DELETE FROM node WHERE database_id = ?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1, + }; +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct RemoveDatabaseStmt(RemoveDatabase command) : IPreparedStatement>, ISupplier { + public static ReadOnlySpan CommandText => "DELETE FROM database WHERE id = ?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1, + }; + + public bool Invoke(DuckDBAdvancedConnection connection) { + connection.ExecuteNonQuery, RemoveDatabaseNodesStmt>(new(command.DatabaseId)); + return connection.ExecuteNonQuery, RemoveDatabaseStmt>(new(command.DatabaseId)) > 0L; + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs new file mode 100644 index 00000000000..0254ed1fc49 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs @@ -0,0 +1,102 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using DotNext; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine; + +using LogEntries; + +partial class Snapshot { + public void Update(AddOrUpdateDatabaseNode command, in CommandInfo info) + => Update(new(command), info); + + public bool Update(RemoveDatabaseNode command, in CommandInfo info) + => Update(new(command), info); + + public void Update(AppointLeader command, in CommandInfo info) + => Update(new(command), info); +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct AddOrUpdateDatabaseNodeStmt(AddOrUpdateDatabaseNode command) : + IPreparedStatement<(string DatabaseId, ReadOnlyMemory Address, bool IsReadOnlyReplica)>, + IConsumer { + public static ReadOnlySpan CommandText => """ + INSERT INTO node (database_id, address, is_read_only_replica) + VALUES ($1, $2, $3) + ON CONFLICT (database_id, address) DO UPDATE + SET is_read_only_replica=$3 + WHERE node.database_id=$1 AND node.address=$2; + """u8; + + public static StatementBindingResult Bind( + in (string DatabaseId, ReadOnlyMemory Address, bool IsReadOnlyReplica) args, + PreparedStatement source) => new(source) { + args.DatabaseId, + args.Address.Span, + args.IsReadOnlyReplica, + }; + + public void Invoke(DuckDBAdvancedConnection connection) + => connection.ExecuteNonQuery<(string, ReadOnlyMemory), RemoveDatabaseNodeStmt>( + new(command.DatabaseId, command.Address.Memory)); +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct RemoveDatabaseNodeStmt(RemoveDatabaseNode command) : + IPreparedStatement<(string DatabaseId, ReadOnlyMemory Address)>, + ISupplier { + public static ReadOnlySpan CommandText => "DELETE FROM node WHERE databaseId=$1 AND address=$2;"u8; + + public static StatementBindingResult Bind(in (string DatabaseId, ReadOnlyMemory Address) args, PreparedStatement source) + => new(source) { + args.DatabaseId, + args.Address.Span + }; + + public bool Invoke(DuckDBAdvancedConnection connection) + => connection.ExecuteNonQuery<(string, ReadOnlyMemory), RemoveDatabaseNodeStmt>( + new(command.DatabaseId, command.Address.Memory)) > 0L; +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct UnsetLeaderNodeStmt : IPreparedStatement> { + public static ReadOnlySpan CommandText => "UPDATE node SET is_leader=false WHERE database_id=?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1 + }; +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct IncrementEpochStmt : IPreparedStatement> { + public static ReadOnlySpan CommandText => "UPDATE database SET epoch = epoch + 1 WHERE id = ?;"u8; + + public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { + args.Item1 + }; +} + +[StructLayout(LayoutKind.Auto)] +file readonly struct AppointLeaderNodeStmt(AppointLeader command) + : IPreparedStatement<(string DatabaseId, ReadOnlyMemory Address)>, + IConsumer { + public static ReadOnlySpan CommandText => "UPDATE node SET is_leader=true WHERE database_id=$1 AND address=$2;"u8; + + public static StatementBindingResult Bind(in (string DatabaseId, ReadOnlyMemory Address) args, + PreparedStatement source) + => new(source) { + args.DatabaseId, + args.Address.Span, + }; + + public void Invoke(DuckDBAdvancedConnection connection) { + connection.ExecuteNonQuery, UnsetLeaderNodeStmt>(new(command.DatabaseId)); + connection.ExecuteNonQuery, IncrementEpochStmt>(new(command.DatabaseId)); + connection.ExecuteNonQuery<(string, ReadOnlyMemory), AppointLeaderNodeStmt>( + new(command.DatabaseId, command.Address.Memory)); + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs new file mode 100644 index 00000000000..27e27080f90 --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs @@ -0,0 +1,38 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class Snapshot { + private ulong _referenceCounter; + + public bool TryAcquire() { + var current = _referenceCounter; + for (ulong tmp;; current = tmp) { + if (current is 0UL) + break; + + tmp = Interlocked.CompareExchange(ref _referenceCounter, current + 1UL, current); + if (tmp == current) + break; + } + + return current > 0UL; + } + + public void Release() { + var current = _referenceCounter; + for (ulong tmp;; current = tmp) { + if (current is 0UL) + break; + + tmp = Interlocked.CompareExchange(ref _referenceCounter, current - 1UL, current); + if (tmp == current) + break; + } + + if (current is 1UL) { + Dispose(); + } + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs new file mode 100644 index 00000000000..ed6638178fd --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs @@ -0,0 +1,144 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using DotNext; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class Snapshot { + private static readonly string LatestSchema = $""" + CREATE TABLE metadata ( + version INTEGER NOT NULL, + index BIGINT NOT NULL, + term BIGINT NOT NULL, + ); + + CREATE TABLE database ( + id VARCHAR PRIMARY KEY, + description VARCHAR NOT NULL DEFAULT '', + epoch UBIGINT NOT NULL DEFAULT 0, + ); + + CREATE TABLE node ( + address BLOB NOT NULL, + database_id VARCHAR NOT NULL, + is_read_only_replica BOOL NOT NULL, + is_leader BOOL NOT NULL DEFAULT FALSE, + FOREIGN KEY (database_id) REFERENCES database (id) + ); + + CREATE UNIQUE INDEX node_id ON node (database_id, address); + CREATE INDEX node_database ON node (database_id); + + INSERT INTO database (id) VALUES ({Database.MainDatabaseId}); + INSERT INTO metadata VALUES ({LatestVersion}, 0, 0); + """; + + /// + /// Initializes the databases with the default schema. + /// + public void Initialize() { + using (_pool.Rent(out var connection)) { + using var transaction = connection.BeginTransaction(); + connection.ExecuteAdHocNonQuery(LatestSchema, multipleStatements: true); + connection.ExecuteNonQuery(LatestVersion); + transaction.CommitOnDispose(); + } + } + + private static void PerformMigration(DuckDBAdvancedConnection connection, + int baseVersion, + int targetVersion, + IReadOnlyDictionary> actions) { + + // Use transaction for each transition to avoid growth of DuckDB WAL + for (baseVersion += 1; baseVersion <= targetVersion; baseVersion++) { + DoUpgrade(connection, actions, baseVersion); + } + + static void DoUpgrade( + DuckDBAdvancedConnection connection, + IReadOnlyDictionary> actions, + int targetVersion) { + using var transaction = connection.BeginTransaction(); + if (actions.TryGetValue(targetVersion, out var action)) { + action.Invoke(connection); + } + + // update version + connection.ExecuteNonQuery(targetVersion); + transaction.CommitOnDispose(); + } + } + + private static Metadata LoadMetadata(DuckDBAdvancedConnection connection) + => connection + .ExecuteQuery() + .FirstOrDefault() + .ValueOrDefault; + + // This method MUST NOT be executed concurrently + private void Update(TCommand command, in CommandInfo info) + where TCommand : struct, IConsumer, allows ref struct { + using (var transaction = _writeConnection.BeginTransaction()) { + command.Invoke(_writeConnection); + + _writeConnection.ExecuteNonQuery(info); + transaction.CommitOnDispose(); + } + + _lastCommandInfo = info; + } + + // This method MUST NOT be executed concurrently + private TResult Update(TCommand command, in CommandInfo info) + where TCommand : struct, ISupplier, allows ref struct { + TResult result; + using (var transaction = _writeConnection.BeginTransaction()) { + result = command.Invoke(_writeConnection); + + _writeConnection.ExecuteNonQuery(info); + transaction.CommitOnDispose(); + } + + _lastCommandInfo = info; + return result; + } + + [StructLayout(LayoutKind.Auto)] + private readonly record struct Metadata(int Version, long LastAppliedIndex, long Term) { + public static implicit operator CommandInfo(in Metadata metadata) => new() { + Index = metadata.LastAppliedIndex, + Term = metadata.Term, + }; + } + + [StructLayout(LayoutKind.Auto)] + private readonly struct MetadataQuery : IQuery { + public static ReadOnlySpan CommandText => "SELECT * FROM metadata;"u8; + + public static Metadata Parse(ref DataChunk.Row row) + => new() { Version = row.ReadInt32(), LastAppliedIndex = row.ReadInt64(), Term = row.ReadInt64() }; + } + + [StructLayout(LayoutKind.Auto)] + private readonly struct UpdateVersionQuery : IPreparedStatement { + public static ReadOnlySpan CommandText => "UPDATE metadata SET version=?;"u8; + + public static StatementBindingResult Bind(in int version, PreparedStatement source) => new(source) { + version, + }; + } + + [StructLayout(LayoutKind.Auto)] + private readonly struct UpdateIndexAndTermQuery : IPreparedStatement { + public static ReadOnlySpan CommandText => "UPDATE metadata SET index = $1, term = $2;"u8; + + public static StatementBindingResult Bind(in CommandInfo args, PreparedStatement source) => new(source) { + args.Index, + args.Term + }; + } +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs new file mode 100644 index 00000000000..a9605b437ef --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs @@ -0,0 +1,13 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine; + +partial class Snapshot { + private const int LatestVersion = 0; + + private static SortedDictionary> MigrationActions + => new(); +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs new file mode 100644 index 00000000000..d8b593ab28d --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs @@ -0,0 +1,92 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using DotNext; +using Kurrent.Quack; +using Kurrent.Quack.ConnectionPool; + +namespace KurrentDB.KontrolPlane.StateMachine; + +/// +/// Represents Kontroller database snapshot. +/// +/// +/// In-memory snapshot uses DuckDB in-memory mode to perform updates and process queries. Thus, we're assuming +/// that the Kontroller database should not be very large (tens of megabytes). Every log entry is applied to +/// the in-memory state. When we need to create a persistent snapshot, COPY FROM DATABASE statement is used to +/// dump in-memory state to the disk as DuckDB database file. However, we don't want to block Apply loop with +/// the heavyweight dump process. So we can start COPY statement in the background while appending new log entries. +/// This concurrency model is preserved by DuckDB 1.5 and later. +/// +internal sealed partial class Snapshot : Disposable { + private static ulong InstanceIndex; + private readonly DuckDBConnectionPool _pool; + private readonly DuckDBAdvancedConnection _writeConnection; + private readonly string _databaseName; + private CommandInfo _lastCommandInfo; + + public Snapshot(int connectionPoolCapacity) { + // For every instance of this class, we have a separated instance of the in-memory database + _pool = new(MakeConnectionString(out _databaseName)) { Capacity = connectionPoolCapacity }; + _writeConnection = _pool.Open(); + } + + public ref readonly CommandInfo LastAppliedCommand => ref _lastCommandInfo; + + public DuckDBConnectionPool.Scope RentConnection(out DuckDBAdvancedConnection connection) + => _pool.Rent(out connection); + + /// + /// Reclaims memory related to deleted rows. + /// + public void ReclaimGarbage() { + ReadOnlySpan command = "FORCE CHECKPOINT;"u8; + + using (_pool.Rent(out var connection)) { + connection.ExecuteAdHocNonQuery(command); + } + } + + public void SaveToFile(string fileName) { + var command = $""" + ATTACH '{fileName}' AS snapshot; + COPY FROM DATABASE {_databaseName} TO snapshot; + DETACH snapshot; + """; + using (_pool.Rent(out var connection)) { + connection.ExecuteAdHocNonQuery(command, multipleStatements: true); + } + } + + public void LoadFromFile(string fileName, int targetVersion = LatestVersion) { + var command = $""" + ATTACH '{fileName}' AS snapshot; + COPY FROM DATABASE snapshot TO {_databaseName}; + DETACH snapshot; + """; + using (_pool.Rent(out var connection)) { + connection.ExecuteAdHocNonQuery(command, multipleStatements: true); + + var metadata = LoadMetadata(connection); + _lastCommandInfo = metadata; + PerformMigration(connection, metadata.Version, targetVersion, MigrationActions); + } + } + + private static string MakeConnectionString(out string databaseName) + => MakeConnectionString(Interlocked.Increment(ref InstanceIndex) - 1UL, out databaseName); + + private static string MakeConnectionString(ulong index, out string databaseName) { + databaseName = $"kontroller_state_{index}"; + return $"DataSource=:memory:{databaseName}"; + } + + protected override void Dispose(bool disposing) { + if (disposing) { + _pool.Dispose(); + _writeConnection.Dispose(); + } + + base.Dispose(disposing); + } +} From 4939eac1c9ec42cb2ebb2a62c2cba3b2b3e727d7 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 11:33:53 +0300 Subject: [PATCH 03/25] Code cleanup --- src/KurrentDB.KontrolPlane/Database.cs | 2 +- src/KurrentDB.KontrolPlane/DatabaseLeader.cs | 2 +- src/KurrentDB.KontrolPlane/DatabaseNode.cs | 2 +- .../{KontrollerEntity.cs => Entity.cs} | 4 +- src/KurrentDB.KontrolPlane/IKontroller.cs | 4 +- .../RaftKontroller.Impl.cs | 54 ++++++++----------- 6 files changed, 29 insertions(+), 39 deletions(-) rename src/KurrentDB.KontrolPlane/{KontrollerEntity.cs => Entity.cs} (73%) diff --git a/src/KurrentDB.KontrolPlane/Database.cs b/src/KurrentDB.KontrolPlane/Database.cs index e80ff0563c0..e299bf728a5 100644 --- a/src/KurrentDB.KontrolPlane/Database.cs +++ b/src/KurrentDB.KontrolPlane/Database.cs @@ -6,7 +6,7 @@ namespace KurrentDB.KontrolPlane; /// /// Describes Kurrent database. /// -public class Database : KontrollerEntity { +public class Database : Entity { public const string MainDatabaseId = "main"; public required string Id { get; init; } diff --git a/src/KurrentDB.KontrolPlane/DatabaseLeader.cs b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs index 720cc36d16a..5e507871f6d 100644 --- a/src/KurrentDB.KontrolPlane/DatabaseLeader.cs +++ b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs @@ -6,7 +6,7 @@ namespace KurrentDB.KontrolPlane; /// /// Represents database leader. /// -public sealed class DatabaseLeader : KontrollerEntity { +public sealed class DatabaseLeader : Entity { public required DatabaseNode Node { get; init; } public required ulong Epoch { get; init; } diff --git a/src/KurrentDB.KontrolPlane/DatabaseNode.cs b/src/KurrentDB.KontrolPlane/DatabaseNode.cs index 729e8aa84f0..15151ba2812 100644 --- a/src/KurrentDB.KontrolPlane/DatabaseNode.cs +++ b/src/KurrentDB.KontrolPlane/DatabaseNode.cs @@ -8,7 +8,7 @@ namespace KurrentDB.KontrolPlane; /// /// Describes database node. /// -public sealed class DatabaseNode : KontrollerEntity { +public sealed class DatabaseNode : Entity { public required string DatabaseId { get; init; } public required EndPoint Address { get; init; } diff --git a/src/KurrentDB.KontrolPlane/KontrollerEntity.cs b/src/KurrentDB.KontrolPlane/Entity.cs similarity index 73% rename from src/KurrentDB.KontrolPlane/KontrollerEntity.cs rename to src/KurrentDB.KontrolPlane/Entity.cs index 82dbabaee55..261085a656b 100644 --- a/src/KurrentDB.KontrolPlane/KontrollerEntity.cs +++ b/src/KurrentDB.KontrolPlane/Entity.cs @@ -3,7 +3,7 @@ namespace KurrentDB.KontrolPlane; -public abstract class KontrollerEntity { - private protected KontrollerEntity() { +public abstract class Entity { + private protected Entity() { } } diff --git a/src/KurrentDB.KontrolPlane/IKontroller.cs b/src/KurrentDB.KontrolPlane/IKontroller.cs index d1daaec5a6e..cb4843d3777 100644 --- a/src/KurrentDB.KontrolPlane/IKontroller.cs +++ b/src/KurrentDB.KontrolPlane/IKontroller.cs @@ -10,9 +10,7 @@ public interface IKontroller { ValueTask> GetDatabasesAsync(CancellationToken token = default); - ValueTask> GetDatabaseNodesAsync(string databaseId, CancellationToken token = default); - - ValueTask GetDatabaseLeaderAsync(string databaseId, CancellationToken token = default); + ValueTask GetDatabaseAsync(string databaseId, CancellationToken token = default); ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index 3f3b160d334..a91344d2d76 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -29,21 +29,9 @@ public async ValueTask> GetDatabasesAsync(CancellationTo return result; } - public async ValueTask> GetDatabaseNodesAsync(string databaseId, CancellationToken token = default) { + public async ValueTask GetDatabaseAsync(string databaseId, CancellationToken token = default) { var snapshot = await _state.CaptureCurrentStateAsync(token); - var result = new List(); - try { - using (snapshot.RentConnection(out var connection)) { - foreach (var database in connection.GetDatabaseNodes(databaseId)) { - result.Add(new DatabaseNode - { DatabaseId = databaseId, Address = database.Address, IsReadOnlyReplica = database.IsReadOnlyReplica }); - } - } - } finally { - snapshot.Release(); - } - - return result; + return GetDatabaseCluster(snapshot, databaseId); } public async ValueTask GetDatabaseLeaderAsync(string databaseId, CancellationToken token = default) { @@ -103,10 +91,14 @@ public async ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default) { ValueTask task; - try { - task = new(RenewLeaderAppointment(databaseId, leaderAddress)); - } catch (Exception e) { - task = ValueTask.FromException(e); + if (token.IsCancellationRequested) { + task = ValueTask.FromCanceled(token); + } else { + try { + task = new(RenewLeaderAppointment(databaseId, leaderAddress)); + } catch (Exception e) { + task = ValueTask.FromException(e); + } } return task; @@ -124,20 +116,20 @@ public async IAsyncEnumerable ListenDatabaseAsync(string databa snapshot.Release(); } } + } - static DatabaseCluster? GetDatabaseCluster(Snapshot snapshot, - string databaseId) { - using (snapshot.RentConnection(out var connection)) { - return connection.GetDatabase(databaseId).FirstOrDefault().TryGet(out var database) - ? new() { - Nodes = GetDatabaseNodes(connection, databaseId, out var leaderAddress), - LeaderAddress = leaderAddress, - Id = databaseId, - Epoch = database.Epoch, - Description = database.Description - } - : null; - } + private static DatabaseCluster? GetDatabaseCluster(Snapshot snapshot, + string databaseId) { + using (snapshot.RentConnection(out var connection)) { + return connection.GetDatabase(databaseId).FirstOrDefault().TryGet(out var database) + ? new() { + Nodes = GetDatabaseNodes(connection, databaseId, out var leaderAddress), + LeaderAddress = leaderAddress, + Id = databaseId, + Epoch = database.Epoch, + Description = database.Description + } + : null; } static IReadOnlyList GetDatabaseNodes(DuckDBAdvancedConnection connection, From 5acd06caf874e85a396b9efe46837fec7b522535 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 11:33:53 +0300 Subject: [PATCH 04/25] Code cleanup --- src/KurrentDB.KontrolPlane/RaftKontroller.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 2afa2c2e771..3ed9f73958e 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -3,7 +3,6 @@ using DotNext.Net.Cluster.Consensus.Raft; using DotNext.Net.Cluster.Consensus.Raft.StateMachine; -using DotNext.Threading; namespace KurrentDB.KontrolPlane; @@ -16,7 +15,6 @@ public partial class RaftKontroller : IAsyncDisposable { private readonly WriteAheadLog _wal; private readonly ClusterStateMachine _state; private readonly RaftCluster _raft; - private readonly CancellationTokenMultiplexer _multiplexer; private readonly TimeSpan _appointmentExpiration; private Task _leadershipTask; @@ -36,7 +34,6 @@ public RaftKontroller(in Options options) { }; _leadershipTask = Task.CompletedTask; - _multiplexer = new() { MaximumRetained = 128 }; _appointmentExpiration = options.AppointmentExpiration; _appointmentState = new(); } From c03607f15dc3770f8420f80ab51c8b820cee5533 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 20:22:17 +0300 Subject: [PATCH 05/25] Reduced memory allocations --- .../StateMachine/LogEntries/IProtobufSerializable.cs | 6 +++--- .../StateMachine/LogEntries/ProtobufLogEntry.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs index 0e826fb235b..02b7131ed63 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/IProtobufSerializable.cs @@ -34,9 +34,9 @@ public static ValueTask WriteToAsync(this T obj, TWriter writer, Can return task; static async ValueTask SerializeSlowAsync(T state, IAsyncBinaryWriter writer, CancellationToken token) { - using var buffer = new PoolingBufferWriter { Capacity = 4096 }; - state.WriteTo(buffer); - await writer.WriteAsync(buffer.WrittenMemory, token: token).ConfigureAwait(false); + using var buffer = MemoryAllocator.Default.AllocateExactly(state.CalculateSize()); + state.WriteTo(buffer.Span); + await writer.WriteAsync(buffer.Memory, token: token).ConfigureAwait(false); } } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs index fdec8fb0028..23527a2b7d3 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ProtobufLogEntry.cs @@ -16,7 +16,7 @@ ValueTask IDataTransferObject.WriteToAsync(TWriter writer, Cancellation bool IDataTransferObject.IsReusable => true; - long? IDataTransferObject.Length => null; + long? IDataTransferObject.Length => entry.CalculateSize(); int? IRaftLogEntry.CommandId => T.TypeId; From 8010a10b8d4307c09bb735decace7e92f8c351bb Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 20:56:33 +0300 Subject: [PATCH 06/25] Code cleanup --- .../StateMachine/state.proto | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/state.proto diff --git a/src/KurrentDB.KontrolPlane/StateMachine/state.proto b/src/KurrentDB.KontrolPlane/StateMachine/state.proto deleted file mode 100644 index ab2eab9dba0..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/state.proto +++ /dev/null @@ -1,30 +0,0 @@ -// Kontroller Database Format - -syntax = "proto3"; - -package kurrent.kontrol_plane.cluster_state; - -option csharp_namespace = "KurrentDB.KontrolPlane.StateMachine"; - -// Represents entire state of the Kontroller internal database -message ClusterState -{ - uint64 version = 1; - map Databases = 2; -} - -message Database -{ - uint64 version = 1; - string description = 2; - uint64 epoch = 3; - repeated DatabaseNode Nodes = 4; - bytes leaderAddress = 5; -} - -message DatabaseNode -{ - uint64 version = 1; - bytes address = 2; - bool isReadOnlyReplica = 3; -} From 6442069c4e642fd74b4e69f22a0786fa925e3144 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Thu, 25 Jun 2026 20:57:39 +0300 Subject: [PATCH 07/25] Removed redundant field --- src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto index 7e47d7889b4..8b44417139b 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto @@ -22,7 +22,6 @@ message AddOrUpdateDatabaseNode string databaseId = 1; bytes address = 2; bool isReadOnlyReplica = 3; - uint64 version = 4; } message RemoveDatabaseNode @@ -34,5 +33,5 @@ message RemoveDatabaseNode message AppointLeader { string databaseId = 1; - bytes address = 3; + bytes address = 2; } From e45c56f332106a67bd284466b560b94e56c57a46 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Fri, 26 Jun 2026 11:10:26 +0300 Subject: [PATCH 08/25] Initialize empty snapshot correctly --- src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs index c6521c9f2d8..27f3cb92bca 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -27,6 +27,7 @@ public ClusterStateMachine(DirectoryInfo location, int connectionPoolCapacity) { _location = location; _snapshot = new(connectionPoolCapacity); + _snapshot.Initialize(); _databases = new(); _poolCapacity = connectionPoolCapacity; } From 1cbb73a4d53f9568dfb296b2b452ed0746971d05 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Fri, 26 Jun 2026 11:21:37 +0300 Subject: [PATCH 09/25] Fixed repository pattern impl --- src/KurrentDB.KontrolPlane/IKontroller.cs | 2 +- .../RaftKontroller.Appointment.cs | 10 +++++----- src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs | 8 ++++---- .../StateMachine/ClusterStateMachine.cs | 4 ++-- .../StateMachine/Queries/AllDatabasesQuery.cs | 10 +++------- .../StateMachine/Queries/QueryHelpers.cs | 4 ++-- 6 files changed, 17 insertions(+), 21 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/IKontroller.cs b/src/KurrentDB.KontrolPlane/IKontroller.cs index cb4843d3777..cab85d8f06c 100644 --- a/src/KurrentDB.KontrolPlane/IKontroller.cs +++ b/src/KurrentDB.KontrolPlane/IKontroller.cs @@ -8,7 +8,7 @@ namespace KurrentDB.KontrolPlane; public interface IKontroller { IDatabaseReplicaSet ReplicaSet { get; init; } - ValueTask> GetDatabasesAsync(CancellationToken token = default); + ValueTask> GetDatabasesAsync(CancellationToken token = default); ValueTask GetDatabaseAsync(string databaseId, CancellationToken token = default); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 8fd9724a10a..01b38e1f2ce 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -60,15 +60,15 @@ private void StartAppointments( CancellationToken token) { // Process appointment for every database in parallel using (snapshot.RentConnection(out var connection)) { - foreach (var database in connection.GetDatabases()) { - databases.Add(database.Id); + foreach (var databaseId in connection.GetDatabases()) { + databases.Add(databaseId); IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes = connection - .GetDatabaseNodes(database.Id) + .GetDatabaseNodes(databaseId) .ToList(); - if (IsAppointmentRequired(database.Id, nodes)) - tasks.Add(AppointLeaderAsync(database.Id, nodes, token)); + if (IsAppointmentRequired(databaseId, nodes)) + tasks.Add(AppointLeaderAsync(databaseId, nodes, token)); } } } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index a91344d2d76..371b3082e24 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -13,13 +13,13 @@ namespace KurrentDB.KontrolPlane; using static StateMachine.LogEntries.ReplicationHelpers; partial class RaftKontroller : IKontroller { - public async ValueTask> GetDatabasesAsync(CancellationToken token = default) { + public async ValueTask> GetDatabasesAsync(CancellationToken token = default) { var snapshot = await _state.CaptureCurrentStateAsync(token); - var result = new List(); + var result = new HashSet(); try { using (snapshot.RentConnection(out var connection)) { - foreach (var database in connection.GetDatabases()) { - result.Add(new Database { Id = database.Id, Description = database.Description, Epoch = database.Epoch }); + foreach (var databaseId in connection.GetDatabases()) { + result.Add(databaseId); } } } finally { diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs index 27f3cb92bca..67973b183d2 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -57,8 +57,8 @@ public void Recover() { private void RefreshDatabaseTrackers(Snapshot snapshot) { var loadedTrackers = new HashSet(); using (snapshot.RentConnection(out var connection)) { - foreach (var database in connection.GetDatabases()) { - loadedTrackers.Add(database.Id); + foreach (var databaseId in connection.GetDatabases()) { + loadedTrackers.Add(databaseId); } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs index 5112fe2ab21..f749eb0559d 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesQuery.cs @@ -7,12 +7,8 @@ namespace KurrentDB.KontrolPlane.StateMachine.Queries; [StructLayout(LayoutKind.Auto)] -internal readonly struct AllDatabasesQuery : IQuery<(string Id, string Description, ulong Epoch)> { - public static ReadOnlySpan CommandText => "SELECT * FROM database;"u8; +internal readonly struct AllDatabasesQuery : IQuery { + public static ReadOnlySpan CommandText => "SELECT id FROM database;"u8; - public static (string Id, string Description, ulong Epoch) Parse(ref DataChunk.Row row) => new() { - Id = row.ReadString(), - Description = row.ReadString(), - Epoch = row.ReadUInt64(), - }; + public static string Parse(ref DataChunk.Row row) => row.ReadString(); } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs index fb3a2536192..2424c067c2d 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs @@ -7,9 +7,9 @@ namespace KurrentDB.KontrolPlane.StateMachine.Queries; internal static class QueryHelpers { - public static QueryResult<(string Id, string Description, ulong Epoch), AllDatabasesQuery> GetDatabases( + public static QueryResult GetDatabases( this DuckDBAdvancedConnection connection) - => connection.ExecuteQuery<(string, string, ulong), AllDatabasesQuery>(); + => connection.ExecuteQuery(); public static QueryResult, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader), AllNodesQuery> GetDatabaseNodes( this DuckDBAdvancedConnection connection, From 6363965849749e41904d3c7aa158fc6cdeb87113 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 15:01:45 +0300 Subject: [PATCH 10/25] Added basic tests for the Kontroller --- src/Directory.Packages.props | 6 +- .../KurrentDB.KontrolPlane.Tests.csproj | 19 ++ .../LeaderAppointmentTests.cs | 150 +++++++++++++ .../RaftKontrollerTests.cs | 201 ++++++++++++++++++ src/KurrentDB.KontrolPlane/Database.cs | 2 +- src/KurrentDB.KontrolPlane/DatabaseCluster.cs | 2 +- src/KurrentDB.KontrolPlane/DatabaseLeader.cs | 2 +- src/KurrentDB.KontrolPlane/DatabaseNode.cs | 2 +- .../{Entity.cs => IEntity.cs} | 5 +- src/KurrentDB.KontrolPlane/IKontroller.cs | 16 +- .../RaftKontroller.Appointment.cs | 49 +++-- .../RaftKontroller.Impl.cs | 33 +-- .../RaftKontroller.Leadership.cs | 9 +- .../RaftKontroller.Options.cs | 5 + src/KurrentDB.KontrolPlane/RaftKontroller.cs | 23 +- src/KurrentDB.KontrolPlane/ReplicaState.cs | 2 +- .../ClusterStateMachine.Handlers.cs | 25 +-- .../StateMachine/ClusterStateMachine.cs | 4 +- .../LogEntries/ReplicationHelpers.cs | 12 +- .../StateMachine/LogEntries/wal.proto | 1 + .../Queries/AllDatabasesWithEpochQuery.cs | 17 ++ .../StateMachine/Queries/AllNodesQuery.cs | 2 +- .../StateMachine/Queries/QueryHelpers.cs | 4 + .../StateMachine/Snapshot.Database.cs | 4 +- .../StateMachine/Snapshot.Node.cs | 34 ++- .../StateMachine/Snapshot.Schema.cs | 2 +- .../StateMachine/Snapshot.cs | 20 +- src/KurrentDB.sln | 19 ++ 28 files changed, 561 insertions(+), 109 deletions(-) create mode 100644 src/KurrentDB.KontrolPlane.Tests/KurrentDB.KontrolPlane.Tests.csproj create mode 100644 src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs create mode 100644 src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs rename src/KurrentDB.KontrolPlane/{Entity.cs => IEntity.cs} (76%) create mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesWithEpochQuery.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e23e99d6f25..0a39584d141 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -17,7 +17,7 @@ - + @@ -69,8 +69,8 @@ - - + + diff --git a/src/KurrentDB.KontrolPlane.Tests/KurrentDB.KontrolPlane.Tests.csproj b/src/KurrentDB.KontrolPlane.Tests/KurrentDB.KontrolPlane.Tests.csproj new file mode 100644 index 00000000000..8dc5c4f25cd --- /dev/null +++ b/src/KurrentDB.KontrolPlane.Tests/KurrentDB.KontrolPlane.Tests.csproj @@ -0,0 +1,19 @@ + + + true + true + true + true + enable + KurrentDB.KontrolPlane + + + + + + + + + + + diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs new file mode 100644 index 00000000000..5cbf8b03a46 --- /dev/null +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Collections.Immutable; +using System.Net; +using KurrentDB.Core.XUnit.Tests; + +namespace KurrentDB.KontrolPlane; + +public sealed class LeaderAppointmentTests : DirectoryFixture { + [Fact] + public async Task AppointLeader() { + // initialize members + var replicaSet = new TestReplicaSet(); + replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 100, Epoch: 1L)); + replicaSet.UpdateMember(TestReplicaSet.Host2, new(UncommittedOffset: 200, Epoch: 0L)); + replicaSet.UpdateMember(TestReplicaSet.Host3, new(UncommittedOffset: 150, Epoch: 1L)); // leader + + // initialize Kontroller + await using var kontroller = new RaftKontroller(new RaftKontroller.Options { + ListenAddress = new(IPAddress.Loopback, 3269), + AppointmentExpiration = TimeSpan.FromSeconds(1), + ConnectionPoolCapacity = 10, + WalOptions = new() { + Location = Directory, + }, + SingleNodeDeployment = true, + }) { + ReplicaSet = replicaSet, + }; + + await kontroller.StartAsync(TestToken); + await AddNodesAsync(kontroller); + + var leader = await WaitForLeaderAsync(kontroller); + Assert.Equal(TestReplicaSet.Host3, leader.Address); + Assert.Equal(1UL, leader.Epoch); + + await kontroller.StopAsync(TestToken); + } + + [Fact] + public async Task RenewLeaderAppointment() { + // initialize members + var replicaSet = new TestReplicaSet(); + replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 100, Epoch: 1L)); + replicaSet.UpdateMember(TestReplicaSet.Host2, new(UncommittedOffset: 200, Epoch: 0L)); + replicaSet.UpdateMember(TestReplicaSet.Host3, new(UncommittedOffset: 150, Epoch: 1L)); // leader + + // initialize Kontroller + await using var kontroller = new RaftKontroller(new RaftKontroller.Options { + ListenAddress = new(IPAddress.Loopback, 3269), + AppointmentExpiration = TimeSpan.FromSeconds(1), + ConnectionPoolCapacity = 10, + WalOptions = new() { + Location = Directory, + }, + SingleNodeDeployment = true, + }) { + ReplicaSet = replicaSet, + }; + + await kontroller.StartAsync(TestToken); + await AddNodesAsync(kontroller); + + var leader = await WaitForLeaderAsync(kontroller); + Assert.Equal(TestReplicaSet.Host3, leader.Address); + + // Start renewal process in the background + var process = new RenewalProcess(kontroller, TestReplicaSet.Host3, leader.Epoch); + var renewalTask = process.RunAsync(); + + // Change state of another member so it should be chosen as a leader, but due to renewal it cannot be appointed + replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 300, Epoch: 3L)); + + // Now stop renewal process and wait for new leader appointment + process.RequestStop(); + await renewalTask.WaitAsync(TestToken); + + do { + leader = await WaitForLeaderAsync(kontroller); + } while (!TestReplicaSet.Host1.Equals(leader.Address)); + + await kontroller.StopAsync(TestToken); + } + + private sealed class RenewalProcess(IKontroller kontroller, EndPoint address, ulong epoch) { + private volatile bool _stopped; + + public void RequestStop() => _stopped = true; + + public async Task RunAsync() { + while (!_stopped) { + // Renew every 300 ms + await Task.Delay(300); + + await kontroller.RenewLeaderAppointmentAsync(Database.MainDatabaseId, address, epoch, TestToken); + } + } + } + + private static async Task AddNodesAsync(IKontroller kontroller) { + var node = new DatabaseNode { DatabaseId = Database.MainDatabaseId, Address = TestReplicaSet.Host1 }; + await kontroller.AddOrUpdateDatabaseNodeAsync(node, TestToken); + + node = new DatabaseNode { DatabaseId = Database.MainDatabaseId, Address = TestReplicaSet.Host2 }; + await kontroller.AddOrUpdateDatabaseNodeAsync(node, TestToken); + + node = new DatabaseNode { DatabaseId = Database.MainDatabaseId, Address = TestReplicaSet.Host3 }; + await kontroller.AddOrUpdateDatabaseNodeAsync(node, TestToken); + + node = new DatabaseNode { DatabaseId = Database.MainDatabaseId, Address = TestReplicaSet.Host4 }; + await kontroller.AddOrUpdateDatabaseNodeAsync(node, TestToken); + } + + private static async Task<(EndPoint? Address, ulong Epoch)> WaitForLeaderAsync(IKontroller kontroller) { + await foreach (var database in kontroller.ListenDatabaseAsync(Database.MainDatabaseId, TestToken)) { + if (database.LeaderAddress is { } address) + return (address, database.Epoch); + } + + return (null, 0L); + } + + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + private sealed class TestReplicaSet : IDatabaseReplicaSet { + public static readonly IPEndPoint Host1 = new(IPAddress.Loopback, 3269); + public static readonly IPEndPoint Host2 = new(IPAddress.Loopback, 3270); + public static readonly IPEndPoint Host3 = new(IPAddress.Loopback, 3271); + public static readonly IPEndPoint Host4 = new(IPAddress.Loopback, 3271); + + private ImmutableDictionary _members = ImmutableDictionary.Empty; + + public void UpdateMember(IPEndPoint endPoint, ReplicaState state) { + for (ImmutableDictionary current = _members, tmp;; current = tmp) { + var newDictionary = current.SetItem(endPoint, state); + tmp = Interlocked.CompareExchange(ref _members, newDictionary, current); + + if (ReferenceEquals(current, tmp)) + break; + } + } + + ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + => Volatile.Read(in _members).TryGetValue(address, out var state) + ? new(state) + : ValueTask.FromException(new IOException()); + } +} diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs new file mode 100644 index 00000000000..48edd948bb9 --- /dev/null +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -0,0 +1,201 @@ +using System.Net; +using KurrentDB.Core.XUnit.Tests; + +namespace KurrentDB.KontrolPlane; + +public class RaftKontrollerTests : DirectoryFixture { + private static readonly IPEndPoint Address = new(IPAddress.Loopback, 3269); + private readonly RaftKontroller _kontroller; + + public RaftKontrollerTests() { + _kontroller = new(new RaftKontroller.Options { + ListenAddress = Address, + AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + ConnectionPoolCapacity = 10, + WalOptions = new() { + Location = Directory, + }, + SingleNodeDeployment = true, + }) { + ReplicaSet = new TestReplicaSet(), + }; + } + + private IKontroller Kontroller => _kontroller; + + [Fact] + public async Task LeaderElection() { + Assert.False(Kontroller.LeadershipToken.IsCancellationRequested); + Assert.Equal(Address, await Kontroller.WaitForLeaderAsync(TestToken)); + } + + [Fact] + public async Task MainDatabasePresence() { + var database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Empty(database.Nodes); + Assert.Equal(Database.MainDatabaseId, database.Id); + Assert.Null(database.LeaderAddress); + } + + [Fact] + public async Task UpdateMainDatabase() { + var database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + database = database with { Description = "Descr" }; + await Kontroller.AddOrUpdateDatabaseAsync(database, TestToken); + + database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Equal("Descr", database.Description); + } + + [Fact] + public async Task AddRemoveDatabase() { + const string databaseId = "test"; + await Kontroller.AddOrUpdateDatabaseAsync(new Database { Id = databaseId, Description = "Descr" }, TestToken); + + var database = await Kontroller.GetDatabaseAsync(databaseId, TestToken); + Assert.NotNull(database); + Assert.Equal("Descr", database.Description); + Assert.Equal(databaseId, database.Id); + + var databases = await Kontroller.GetDatabasesAsync(TestToken); + Assert.Contains(Database.MainDatabaseId, databases); + Assert.Contains(databaseId, databases); + + Assert.True(await Kontroller.RemoveDatabaseAsync(databaseId, TestToken)); + database = await Kontroller.GetDatabaseAsync(databaseId, TestToken); + Assert.Null(database); + } + + [Fact] + public async Task AddRemoveNode() { + var expectedNode = new DatabaseNode { + Address = new IPEndPoint(IPAddress.Parse("192.168.0.1"), 3262), + DatabaseId = Database.MainDatabaseId, + IsReadOnlyReplica = true, + }; + + await Kontroller.AddOrUpdateDatabaseNodeAsync(expectedNode, TestToken); + + var database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Null(database.LeaderAddress); + + var actualNode = Assert.Single(database.Nodes); + Assert.Equal(expectedNode, actualNode); + + await Kontroller.RemoveDatabaseNodeAsync(Database.MainDatabaseId, expectedNode.Address, TestToken); + + database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Empty(database.Nodes); + } + + [Fact] + public async Task UpdateNode() { + var expectedNode = new DatabaseNode { + Address = new IPEndPoint(IPAddress.Parse("192.168.0.1"), 3262), + DatabaseId = Database.MainDatabaseId, + IsReadOnlyReplica = true, + }; + + await Kontroller.AddOrUpdateDatabaseNodeAsync(expectedNode, TestToken); + + var database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Null(database.LeaderAddress); + + var actualNode = Assert.Single(database.Nodes); + Assert.Equal(expectedNode.IsReadOnlyReplica, actualNode.IsReadOnlyReplica); + + expectedNode = expectedNode with { IsReadOnlyReplica = false }; + await Kontroller.AddOrUpdateDatabaseNodeAsync(expectedNode, TestToken); + + database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); + Assert.NotNull(database); + Assert.Null(database.LeaderAddress); + + actualNode = Assert.Single(database.Nodes); + Assert.Equal(expectedNode.IsReadOnlyReplica, actualNode.IsReadOnlyReplica); + } + + [Fact] + public async Task TrackDatabaseDescription() { + await using var enumerator = Kontroller + .ListenDatabaseAsync(Database.MainDatabaseId, TestToken) + .GetAsyncEnumerator(); + + Assert.True(await enumerator.MoveNextAsync()); + var database = enumerator.Current; + Assert.Empty(database.Description); + + database = database with { Description = "Descr" }; + await Kontroller.AddOrUpdateDatabaseAsync(database, TestToken); + + Assert.True(await enumerator.MoveNextAsync()); + database = enumerator.Current; + Assert.Equal("Descr", database.Description); + } + + [Fact] + public async Task TrackDatabaseNode() { + await using var enumerator = Kontroller + .ListenDatabaseAsync(Database.MainDatabaseId, TestToken) + .GetAsyncEnumerator(); + + Assert.True(await enumerator.MoveNextAsync()); + var database = enumerator.Current; + Assert.Empty(database.Nodes); + + var expectedNode = new DatabaseNode { + Address = new IPEndPoint(IPAddress.Parse("192.168.0.1"), 3262), + DatabaseId = Database.MainDatabaseId, + IsReadOnlyReplica = true, + }; + + await Kontroller.AddOrUpdateDatabaseNodeAsync(expectedNode, TestToken); + Assert.True(await enumerator.MoveNextAsync()); + database = enumerator.Current; + + Assert.Equal(expectedNode, Assert.Single(database.Nodes)); + } + + [Fact] + public async Task TrackDatabaseRemoval() { + const string databaseId = "test"; + await Kontroller.AddOrUpdateDatabaseAsync(new Database { Id = databaseId, Description = "Descr" }, TestToken); + + await using var enumerator = Kontroller + .ListenDatabaseAsync(databaseId, TestToken) + .GetAsyncEnumerator(); + + Assert.True(await enumerator.MoveNextAsync()); + var database = enumerator.Current; + Assert.Equal("Descr", database.Description); + Assert.Empty(database.Nodes); + + Assert.True(await Kontroller.RemoveDatabaseAsync(databaseId, TestToken)); + Assert.False(await enumerator.MoveNextAsync()); + } + + public override async ValueTask DisposeAsync() { + await _kontroller.StopAsync(TestToken); + await _kontroller.DisposeAsync(); + await base.DisposeAsync(); + } + + public override async ValueTask InitializeAsync() { + await base.InitializeAsync(); + await _kontroller.StartAsync(TestToken); + await _kontroller.EnsureLeadershipAsync(TestToken); + } + + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + private sealed class TestReplicaSet : IDatabaseReplicaSet { + ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + => ValueTask.FromException(new NotSupportedException()); + } +} diff --git a/src/KurrentDB.KontrolPlane/Database.cs b/src/KurrentDB.KontrolPlane/Database.cs index e299bf728a5..4be33c8a915 100644 --- a/src/KurrentDB.KontrolPlane/Database.cs +++ b/src/KurrentDB.KontrolPlane/Database.cs @@ -6,7 +6,7 @@ namespace KurrentDB.KontrolPlane; /// /// Describes Kurrent database. /// -public class Database : Entity { +public record Database : IEntity { public const string MainDatabaseId = "main"; public required string Id { get; init; } diff --git a/src/KurrentDB.KontrolPlane/DatabaseCluster.cs b/src/KurrentDB.KontrolPlane/DatabaseCluster.cs index a74ef718db1..fd786ec97a4 100644 --- a/src/KurrentDB.KontrolPlane/DatabaseCluster.cs +++ b/src/KurrentDB.KontrolPlane/DatabaseCluster.cs @@ -8,7 +8,7 @@ namespace KurrentDB.KontrolPlane; /// /// Represents instant state of the database. /// -public sealed class DatabaseCluster : Database { +public sealed record DatabaseCluster : Database { public IReadOnlyList Nodes { get => field ?? []; init; diff --git a/src/KurrentDB.KontrolPlane/DatabaseLeader.cs b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs index 5e507871f6d..f1edc69616c 100644 --- a/src/KurrentDB.KontrolPlane/DatabaseLeader.cs +++ b/src/KurrentDB.KontrolPlane/DatabaseLeader.cs @@ -6,7 +6,7 @@ namespace KurrentDB.KontrolPlane; /// /// Represents database leader. /// -public sealed class DatabaseLeader : Entity { +public sealed record DatabaseLeader : IEntity { public required DatabaseNode Node { get; init; } public required ulong Epoch { get; init; } diff --git a/src/KurrentDB.KontrolPlane/DatabaseNode.cs b/src/KurrentDB.KontrolPlane/DatabaseNode.cs index 15151ba2812..d2f620c1b04 100644 --- a/src/KurrentDB.KontrolPlane/DatabaseNode.cs +++ b/src/KurrentDB.KontrolPlane/DatabaseNode.cs @@ -8,7 +8,7 @@ namespace KurrentDB.KontrolPlane; /// /// Describes database node. /// -public sealed class DatabaseNode : Entity { +public sealed record DatabaseNode : IEntity { public required string DatabaseId { get; init; } public required EndPoint Address { get; init; } diff --git a/src/KurrentDB.KontrolPlane/Entity.cs b/src/KurrentDB.KontrolPlane/IEntity.cs similarity index 76% rename from src/KurrentDB.KontrolPlane/Entity.cs rename to src/KurrentDB.KontrolPlane/IEntity.cs index 261085a656b..782b1a806a3 100644 --- a/src/KurrentDB.KontrolPlane/Entity.cs +++ b/src/KurrentDB.KontrolPlane/IEntity.cs @@ -3,7 +3,4 @@ namespace KurrentDB.KontrolPlane; -public abstract class Entity { - private protected Entity() { - } -} +public interface IEntity; diff --git a/src/KurrentDB.KontrolPlane/IKontroller.cs b/src/KurrentDB.KontrolPlane/IKontroller.cs index cab85d8f06c..86e20ae7635 100644 --- a/src/KurrentDB.KontrolPlane/IKontroller.cs +++ b/src/KurrentDB.KontrolPlane/IKontroller.cs @@ -6,13 +6,11 @@ namespace KurrentDB.KontrolPlane; public interface IKontroller { - IDatabaseReplicaSet ReplicaSet { get; init; } - ValueTask> GetDatabasesAsync(CancellationToken token = default); ValueTask GetDatabaseAsync(string databaseId, CancellationToken token = default); - ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default); + ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, ulong epoch, CancellationToken token = default); ValueTask AddOrUpdateDatabaseAsync(Database database, CancellationToken token = default); @@ -29,4 +27,16 @@ public interface IKontroller { /// The token that can be used to cancel the operation. /// A stream of full database snapshot. The stream finishes if becomes deleted. IAsyncEnumerable ListenDatabaseAsync(string databaseId, CancellationToken token = default); + + /// + /// Gets a token associated with leader state of the current instance of the Kontroller. + /// + CancellationToken LeadershipToken { get; } + + /// + /// Ensures that the current Kontroller instance is a part of the KPlane cluster and the cluster leader is observable. + /// + /// The token that can be used to cancel the operation. + /// The address of the KPlane leader. + ValueTask WaitForLeaderAsync(CancellationToken token = default); } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 01b38e1f2ce..6f36240f348 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -25,13 +25,17 @@ private async ValueTask ProcessAppointmentsAsync(CancellationToken token) { try { do { var snapshot = await _state.CaptureCurrentStateAsync(token); - StartAppointments(snapshot, tasks, databases, token); - RemoveDeletedDatabases(databases, deletedDatabases); - - await Task.WhenAll(tasks); - tasks.Clear(); - deletedDatabases.Clear(); - databases.Clear(); + try { + StartAppointments(snapshot, tasks, databases, token); + RemoveDeletedDatabases(databases, deletedDatabases); + + await Task.WhenAll(tasks); + } finally { + snapshot.Release(); + tasks.Clear(); + deletedDatabases.Clear(); + databases.Clear(); + } } while (await timer.WaitForNextTickAsync(token)); } finally { timer.Dispose(); @@ -60,15 +64,18 @@ private void StartAppointments( CancellationToken token) { // Process appointment for every database in parallel using (snapshot.RentConnection(out var connection)) { - foreach (var databaseId in connection.GetDatabases()) { - databases.Add(databaseId); + // Workaround: it's not possible to enumerate to query results within the same connection. + // Thus, we need to materialize (ToList) the first query + var currentDBs = connection.GetDatabasesWithEpoch().ToList(); + foreach (var database in currentDBs) { + databases.Add(database.Id); IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes = connection - .GetDatabaseNodes(databaseId) + .GetDatabaseNodes(database.Id) .ToList(); - if (IsAppointmentRequired(databaseId, nodes)) - tasks.Add(AppointLeaderAsync(databaseId, nodes, token)); + if (IsAppointmentRequired(database.Id, nodes)) + tasks.Add(AppointLeaderAsync(database.Id, database.Epoch, nodes, token)); } } } @@ -78,7 +85,7 @@ private bool IsAppointmentRequired(string databaseId, IReadOnlyList<(EndPoint Ad && (!_appointmentState.TryGetValue(databaseId, out var appointment) || appointment.IsExpired(_appointmentExpiration)); - private async Task AppointLeaderAsync(string databaseId, IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) { + private async Task AppointLeaderAsync(string databaseId, ulong epoch, IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) { var responses = new Dictionary(nodes.Count); await foreach (var task in Task.WhenEach(GetReplicaState(ReplicaSet, nodes, token))) { @@ -107,8 +114,8 @@ private async Task AppointLeaderAsync(string databaseId, IReadOnlyList<(EndPoint .Key; // Appoint the leader - await _raft.AppointLeaderAsync(databaseId, candidate, token); - _appointmentState[databaseId] = new LeaderAppointment(candidate); + if (await _raft.AppointLeaderAsync(databaseId, epoch, candidate, token)) + _appointmentState[databaseId] = new LeaderAppointment(candidate, epoch); static IEnumerable>> GetReplicaState( IDatabaseReplicaSet replicas, @@ -125,18 +132,18 @@ static async Task> GetReplicaStateAsync( => new(address, await replicas.GetReplicaStateAsync(address, token)); } - private bool RenewLeaderAppointment(string databaseId, EndPoint leaderAddress) { - if (!_appointmentState.TryGetValue(databaseId, out var expectedAppointment)) + private bool RenewLeaderAppointment(string databaseId, EndPoint leaderAddress, ulong epoch) { + if (!_appointmentState.TryGetValue(databaseId, out var expectedAppointment) || expectedAppointment.Epoch != epoch) return false; - var newAppointment = new LeaderAppointment(leaderAddress); + var newAppointment = new LeaderAppointment(leaderAddress, epoch); return _appointmentState.TryUpdate(databaseId, newAppointment, expectedAppointment); } [StructLayout(LayoutKind.Auto)] - private readonly record struct LeaderAppointment(EndPoint Address, Timestamp UpdatedAt) { - public LeaderAppointment(EndPoint address) - : this(address, new()) { + private readonly record struct LeaderAppointment(EndPoint Address, ulong Epoch, Timestamp UpdatedAt) { + public LeaderAppointment(EndPoint address, ulong epoch) + : this(address, epoch, new()) { } public bool IsExpired(TimeSpan expiration) => UpdatedAt.Elapsed >= expiration; diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index 371b3082e24..0f2d3d88c00 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -6,6 +6,7 @@ using DotNext.Net.Cluster.Consensus.Raft; using Kurrent.Quack; using KurrentDB.KontrolPlane.StateMachine; +using static System.Threading.Timeout; namespace KurrentDB.KontrolPlane; @@ -14,8 +15,8 @@ namespace KurrentDB.KontrolPlane; partial class RaftKontroller : IKontroller { public async ValueTask> GetDatabasesAsync(CancellationToken token = default) { - var snapshot = await _state.CaptureCurrentStateAsync(token); var result = new HashSet(); + var snapshot = await _state.CaptureCurrentStateAsync(token); try { using (snapshot.RentConnection(out var connection)) { foreach (var databaseId in connection.GetDatabases()) { @@ -30,28 +31,9 @@ public async ValueTask> GetDatabasesAsync(CancellationToken } public async ValueTask GetDatabaseAsync(string databaseId, CancellationToken token = default) { - var snapshot = await _state.CaptureCurrentStateAsync(token); - return GetDatabaseCluster(snapshot, databaseId); - } - - public async ValueTask GetDatabaseLeaderAsync(string databaseId, CancellationToken token = default) { var snapshot = await _state.CaptureCurrentStateAsync(token); try { - using (snapshot.RentConnection(out var connection)) { - return connection - .GetDatabaseLeader(databaseId) - .FirstOrDefault() - .TryGet(out var leader) - ? new DatabaseLeader { - Epoch = leader.Epoch, - Node = new() { - Address = leader.Address, - DatabaseId = databaseId, - IsReadOnlyReplica = leader.IsReadOnlyReplica - } - } - : null; - } + return GetDatabaseCluster(snapshot, databaseId); } finally { snapshot.Release(); } @@ -89,13 +71,13 @@ public async ValueTask RemoveDatabaseNodeAsync(string databaseId, EndPoint } } - public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, CancellationToken token = default) { + public ValueTask RenewLeaderAppointmentAsync(string databaseId, EndPoint leaderAddress, ulong epoch, CancellationToken token = default) { ValueTask task; if (token.IsCancellationRequested) { task = ValueTask.FromCanceled(token); } else { try { - task = new(RenewLeaderAppointment(databaseId, leaderAddress)); + task = new(RenewLeaderAppointment(databaseId, leaderAddress, epoch)); } catch (Exception e) { task = ValueTask.FromException(e); } @@ -152,4 +134,9 @@ static IReadOnlyList GetDatabaseNodes(DuckDBAdvancedConnection con return nodes; } } + + public CancellationToken LeadershipToken => _raft.LeadershipToken; + + public async ValueTask WaitForLeaderAsync(CancellationToken token = default) + => (await _raft.WaitForLeaderAsync(InfiniteTimeSpan, token)).EndPoint; } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs index 543b2b942c6..d671a1c82c1 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Leadership.cs @@ -1,14 +1,21 @@ // Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. // Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). +using System.Diagnostics; + namespace KurrentDB.KontrolPlane; partial class RaftKontroller { + private readonly CancellationToken _lifecycleToken; // cached to avoid ObjectDisposedException + private volatile CancellationTokenSource? _lifecycleTokenSource; + private async Task HandleLeadershipAsync() { for (;;) { CancellationToken leadershipToken; try { - leadershipToken = await _raft.WaitForLeadershipAsync(CancellationToken.None); + leadershipToken = await _raft.WaitForLeadershipAsync(_lifecycleToken); + } catch (OperationCanceledException e) when (e.CancellationToken == _lifecycleToken) { + break; } catch (ObjectDisposedException) { break; } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs index 7028b51706b..a6ba6c950a4 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs @@ -35,5 +35,10 @@ public required TimeSpan AppointmentExpiration { get; init => field = value > TimeSpan.Zero ? value : throw new ArgumentOutOfRangeException(nameof(value)); } + + public bool SingleNodeDeployment { + get; + init; + } } } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 3ed9f73958e..67ad548490b 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -19,7 +19,7 @@ public partial class RaftKontroller : IAsyncDisposable { private Task _leadershipTask; public RaftKontroller(in Options options) { - var stateLocation = new DirectoryInfo(Path.Combine(options.WalOptions.Location, "db")); + var stateLocation = new DirectoryInfo(Path.Combine(options.WalOptions.Location, "ktrl")); var configStorageLocation = Path.Combine(options.WalOptions.Location, "members.list"); _state = new(stateLocation, options.ConnectionPoolCapacity); _wal = new WriteAheadLog(options.WalOptions, _state); @@ -27,6 +27,7 @@ public RaftKontroller(in Options options) { var config = new RaftCluster.TcpConfiguration(options.ListenAddress) { PublicEndPoint = options.PublicAddress, ConfigurationStorage = new PersistentConfigurationStorage(configStorageLocation), + ColdStart = options.SingleNodeDeployment, }; _raft = new RaftCluster(config) { @@ -36,6 +37,8 @@ public RaftKontroller(in Options options) { _leadershipTask = Task.CompletedTask; _appointmentExpiration = options.AppointmentExpiration; _appointmentState = new(); + _lifecycleTokenSource = new(); + _lifecycleToken = _lifecycleTokenSource.Token; } public required IDatabaseReplicaSet ReplicaSet { @@ -43,19 +46,37 @@ public required IDatabaseReplicaSet ReplicaSet { init => field = value ?? throw new ArgumentNullException(nameof(value)); } + public Task EnsureLeadershipAsync(CancellationToken token) + => _raft.WaitForLeadershipAsync(token); + public async Task StartAsync(CancellationToken token) { + _state.Recover(); await _raft.StartAsync(token); _leadershipTask = HandleLeadershipAsync(); } public async Task StopAsync(CancellationToken token) { await _raft.StopAsync(token); + await CancelAsync(); await _leadershipTask.ConfigureAwait(false); } public async ValueTask DisposeAsync() { + await CancelAsync(); await _raft.DisposeAsync(); await _wal.DisposeAsync(); _state.Dispose(); } + + private ValueTask CancelAsync() { + return Interlocked.Exchange(ref _lifecycleTokenSource, null) is { } cts + ? CancelAndDiposeAsync(cts) + : ValueTask.CompletedTask; + + static async ValueTask CancelAndDiposeAsync(CancellationTokenSource cts) { + using (cts) { + await cts.CancelAsync(); + } + } + } } diff --git a/src/KurrentDB.KontrolPlane/ReplicaState.cs b/src/KurrentDB.KontrolPlane/ReplicaState.cs index 207a22516d0..9ccf4be606e 100644 --- a/src/KurrentDB.KontrolPlane/ReplicaState.cs +++ b/src/KurrentDB.KontrolPlane/ReplicaState.cs @@ -11,4 +11,4 @@ namespace KurrentDB.KontrolPlane; /// The latest known offset of the uncommitted log record. /// The epoch of the database node. [StructLayout(LayoutKind.Auto)] -public readonly record struct ReplicaState(long UncommittedOffset, ulong Epoch); +public readonly record struct ReplicaState(long UncommittedOffset, ulong Epoch) : IEntity; diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs index 782b07e1460..1c01767ad08 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +using DotNext.Collections.Generic; using DotNext.IO; using DotNext.Net.Cluster.Consensus.Raft.StateMachine; using DotNext.Threading; @@ -40,7 +41,8 @@ private async ValueTask ApplyAsync(LogEntry entry, CancellationToken token case LogEntries.AppointLeader.TypeId: Apply(currentState, await DeserializeAsync(entry, token), - in commandInfo); + in commandInfo, + entry.Context as StrongBox); break; default: Debug.Fail($"Unexpected entry type {entry.CommandId}"); @@ -68,9 +70,7 @@ private void Apply(Snapshot currentState, StrongBox? resultContainer) { var result = currentState.Update(command, in commandInfo); - if (_databases.TryRemove(command.DatabaseId, out var tracker)) { - tracker.TryComplete(); - } + _databases.TryRemove(command.DatabaseId).ValueOrDefault?.TryComplete(); resultContainer?.Value = result; } @@ -78,9 +78,7 @@ private void Apply(Snapshot currentState, private void Apply(Snapshot currentState, LogEntries.AddOrUpdateDatabaseNode command, in CommandInfo commandInfo) { currentState.Update(command, in commandInfo); - if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { - tracker.TryAdvance(); - } + _databases.TryGetValue(command.DatabaseId).ValueOrDefault?.TryAdvance(); } private void Apply(Snapshot currentState, @@ -89,19 +87,16 @@ private void Apply(Snapshot currentState, StrongBox? resultContainer) { var result = currentState.Update(command, in commandInfo); - if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { - tracker.TryAdvance(); - } + _databases.TryGetValue(command.DatabaseId).ValueOrDefault?.TryAdvance(); resultContainer?.Value = result; } - private void Apply(Snapshot currentState, LogEntries.AppointLeader command, in CommandInfo commandInfo) { - currentState.Update(command, in commandInfo); + private void Apply(Snapshot currentState, LogEntries.AppointLeader command, in CommandInfo commandInfo, StrongBox? resultContainer) { + var result = currentState.Update(command, in commandInfo); - if (_databases.TryGetValue(command.DatabaseId, out var tracker)) { - tracker.TryAdvance(); - } + _databases.TryGetValue(command.DatabaseId).ValueOrDefault?.TryAdvance(); + resultContainer?.Value = result; } private static ValueTask DeserializeAsync(in LogEntry entry, CancellationToken token) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs index 67973b183d2..d8de2b125db 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -27,7 +27,6 @@ public ClusterStateMachine(DirectoryInfo location, int connectionPoolCapacity) { _location = location; _snapshot = new(connectionPoolCapacity); - _snapshot.Initialize(); _databases = new(); _poolCapacity = connectionPoolCapacity; } @@ -49,6 +48,9 @@ public void Recover() { var newSnapshot = InstallSnapshot(latestSnapshotFile.FullName); _persistentSnapshot = new(latestSnapshotFile.FullName, newSnapshot.LastAppliedCommand); + } else { + _snapshot.Initialize(); + _databases[Database.MainDatabaseId] = new(); } snapshots.Clear(); // help GC diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs index 0d6015ac33b..db977bd1b7a 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs @@ -45,12 +45,16 @@ await raft.ReplicateAsync( return box.Value; } - public ValueTask AppointLeaderAsync(string databaseId, + public async ValueTask AppointLeaderAsync(string databaseId, + ulong epoch, EndPoint address, - CancellationToken token) - => raft.ReplicateAsync( + CancellationToken token) { + var box = new StrongBox(); + await raft.ReplicateAsync( new ProtobufLogEntry(new() - { Address = address.ToByteString(), DatabaseId = databaseId }) + { Address = address.ToByteString(), DatabaseId = databaseId, Epoch = epoch }) { Term = raft.Term }, token); + return box.Value; + } } } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto index 8b44417139b..0d64855b6b8 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/wal.proto @@ -34,4 +34,5 @@ message AppointLeader { string databaseId = 1; bytes address = 2; + uint64 epoch = 3; } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesWithEpochQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesWithEpochQuery.cs new file mode 100644 index 00000000000..94ab300e6bf --- /dev/null +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllDatabasesWithEpochQuery.cs @@ -0,0 +1,17 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Runtime.InteropServices; +using Kurrent.Quack; + +namespace KurrentDB.KontrolPlane.StateMachine.Queries; + +[StructLayout(LayoutKind.Auto)] +internal readonly struct AllDatabasesWithEpochQuery : IQuery<(string Id, ulong Epoch)> { + public static ReadOnlySpan CommandText => "SELECT id, epoch FROM database;"u8; + + public static (string Id, ulong Epoch) Parse(ref DataChunk.Row row) => new() { + Id = row.ReadString(), + Epoch = row.ReadUInt64() + }; +} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs index c269ea86f0a..0bcea7cfd2c 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/AllNodesQuery.cs @@ -9,7 +9,7 @@ namespace KurrentDB.KontrolPlane.StateMachine.Queries; [StructLayout(LayoutKind.Auto)] internal readonly struct AllNodesQuery : IQuery, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> { - public static ReadOnlySpan CommandText => "SELECT (address, is_read_only_replica, is_leader) FROM node WHERE database_id=?;"u8; + public static ReadOnlySpan CommandText => "SELECT address, is_read_only_replica, is_leader FROM node WHERE database_id=$1;"u8; public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { args.Item1, diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs index 2424c067c2d..e886e774084 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs @@ -11,6 +11,10 @@ public static QueryResult GetDatabases( this DuckDBAdvancedConnection connection) => connection.ExecuteQuery(); + public static QueryResult<(string Id, ulong Epoch), AllDatabasesWithEpochQuery> GetDatabasesWithEpoch( + this DuckDBAdvancedConnection connection) + => connection.ExecuteQuery<(string, ulong), AllDatabasesWithEpochQuery>(); + public static QueryResult, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader), AllNodesQuery> GetDatabaseNodes( this DuckDBAdvancedConnection connection, string databaseId) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs index 48916825b4f..6c4d2d877bf 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs @@ -14,7 +14,7 @@ public void Update(AddOrUpdateDatabase command, in CommandInfo info) => Update(new(command), info); public bool Update(RemoveDatabase command, in CommandInfo info) - => Update(new(command), info); + => command.DatabaseId is not Database.MainDatabaseId && Update(new(command), info); } [StructLayout(LayoutKind.Auto)] @@ -24,7 +24,7 @@ INSERT INTO database (id, description) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET description = $2 - WHERE database.id = $3; + WHERE database.id = $1; """u8; public static StatementBindingResult Bind(in (string DatabaseId, string Description) args, PreparedStatement source) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs index 0254ed1fc49..e36b18d3acf 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs @@ -16,8 +16,16 @@ public void Update(AddOrUpdateDatabaseNode command, in CommandInfo info) public bool Update(RemoveDatabaseNode command, in CommandInfo info) => Update(new(command), info); - public void Update(AppointLeader command, in CommandInfo info) - => Update(new(command), info); + public bool Update(AppointLeader command, in CommandInfo info) { + var result = true; + try { + Update(new(command), info); + } catch (StaleEpochException) { + result = false; + } + + return result; + } } [StructLayout(LayoutKind.Auto)] @@ -41,15 +49,15 @@ public static StatementBindingResult Bind( }; public void Invoke(DuckDBAdvancedConnection connection) - => connection.ExecuteNonQuery<(string, ReadOnlyMemory), RemoveDatabaseNodeStmt>( - new(command.DatabaseId, command.Address.Memory)); + => connection.ExecuteNonQuery<(string, ReadOnlyMemory, bool), AddOrUpdateDatabaseNodeStmt>( + new(command.DatabaseId, command.Address.Memory, command.IsReadOnlyReplica)); } [StructLayout(LayoutKind.Auto)] file readonly struct RemoveDatabaseNodeStmt(RemoveDatabaseNode command) : IPreparedStatement<(string DatabaseId, ReadOnlyMemory Address)>, ISupplier { - public static ReadOnlySpan CommandText => "DELETE FROM node WHERE databaseId=$1 AND address=$2;"u8; + public static ReadOnlySpan CommandText => "DELETE FROM node WHERE database_id=$1 AND address=$2;"u8; public static StatementBindingResult Bind(in (string DatabaseId, ReadOnlyMemory Address) args, PreparedStatement source) => new(source) { @@ -72,11 +80,12 @@ public bool Invoke(DuckDBAdvancedConnection connection) } [StructLayout(LayoutKind.Auto)] -file readonly struct IncrementEpochStmt : IPreparedStatement> { - public static ReadOnlySpan CommandText => "UPDATE database SET epoch = epoch + 1 WHERE id = ?;"u8; +file readonly struct IncrementEpochStmt : IPreparedStatement<(string Id, ulong Epoch)> { + public static ReadOnlySpan CommandText => "UPDATE database SET epoch = epoch + 1 WHERE id = $1 AND epoch = $2;"u8; - public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { - args.Item1 + public static StatementBindingResult Bind(in (string Id, ulong Epoch) args, PreparedStatement source) => new(source) { + args.Id, + args.Epoch }; } @@ -95,8 +104,13 @@ public static StatementBindingResult Bind(in (string DatabaseId, ReadOnlyMemory< public void Invoke(DuckDBAdvancedConnection connection) { connection.ExecuteNonQuery, UnsetLeaderNodeStmt>(new(command.DatabaseId)); - connection.ExecuteNonQuery, IncrementEpochStmt>(new(command.DatabaseId)); + if (connection.ExecuteNonQuery<(string, ulong), IncrementEpochStmt>( + new(command.DatabaseId, command.Epoch)) is 0L) + throw new StaleEpochException(); + connection.ExecuteNonQuery<(string, ReadOnlyMemory), AppointLeaderNodeStmt>( new(command.DatabaseId, command.Address.Memory)); } } + +file sealed class StaleEpochException() : InvalidOperationException("Epoch is old."); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs index ed6638178fd..df19b5456d2 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs @@ -32,7 +32,7 @@ FOREIGN KEY (database_id) REFERENCES database (id) CREATE UNIQUE INDEX node_id ON node (database_id, address); CREATE INDEX node_database ON node (database_id); - INSERT INTO database (id) VALUES ({Database.MainDatabaseId}); + INSERT INTO database (id) VALUES ('{Database.MainDatabaseId}'); INSERT INTO metadata VALUES ({LatestVersion}, 0, 0); """; diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs index d8b593ab28d..f494f9413f4 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs @@ -19,16 +19,16 @@ namespace KurrentDB.KontrolPlane.StateMachine; /// This concurrency model is preserved by DuckDB 1.5 and later. /// internal sealed partial class Snapshot : Disposable { - private static ulong InstanceIndex; private readonly DuckDBConnectionPool _pool; private readonly DuckDBAdvancedConnection _writeConnection; - private readonly string _databaseName; private CommandInfo _lastCommandInfo; public Snapshot(int connectionPoolCapacity) { // For every instance of this class, we have a separated instance of the in-memory database - _pool = new(MakeConnectionString(out _databaseName)) { Capacity = connectionPoolCapacity }; - _writeConnection = _pool.Open(); + _writeConnection = new DuckDBAdvancedConnection { ConnectionString = "DataSource=:memory:" }; + _writeConnection.Open(); + _pool = new(_writeConnection) { Capacity = connectionPoolCapacity }; + _referenceCounter = 1U; } public ref readonly CommandInfo LastAppliedCommand => ref _lastCommandInfo; @@ -50,7 +50,7 @@ public void ReclaimGarbage() { public void SaveToFile(string fileName) { var command = $""" ATTACH '{fileName}' AS snapshot; - COPY FROM DATABASE {_databaseName} TO snapshot; + COPY FROM DATABASE memory TO snapshot; DETACH snapshot; """; using (_pool.Rent(out var connection)) { @@ -61,7 +61,7 @@ public void SaveToFile(string fileName) { public void LoadFromFile(string fileName, int targetVersion = LatestVersion) { var command = $""" ATTACH '{fileName}' AS snapshot; - COPY FROM DATABASE snapshot TO {_databaseName}; + COPY FROM DATABASE snapshot TO memory; DETACH snapshot; """; using (_pool.Rent(out var connection)) { @@ -73,14 +73,6 @@ public void LoadFromFile(string fileName, int targetVersion = LatestVersion) { } } - private static string MakeConnectionString(out string databaseName) - => MakeConnectionString(Interlocked.Increment(ref InstanceIndex) - 1UL, out databaseName); - - private static string MakeConnectionString(ulong index, out string databaseName) { - databaseName = $"kontroller_state_{index}"; - return $"DataSource=:memory:{databaseName}"; - } - protected override void Dispose(bool disposing) { if (disposing) { _pool.Dispose(); diff --git a/src/KurrentDB.sln b/src/KurrentDB.sln index f2b40ff45d1..4a92fa43aa9 100644 --- a/src/KurrentDB.sln +++ b/src/KurrentDB.sln @@ -220,6 +220,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.Scripting", "Kurr EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.KontrolPlane", "KurrentDB.KontrolPlane\KurrentDB.KontrolPlane.csproj", "{C4FA3E05-81FA-4151-8BF8-D1B54D486627}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KurrentDB.KontrolPlane.Tests", "KurrentDB.KontrolPlane.Tests\KurrentDB.KontrolPlane.Tests.csproj", "{1CE82908-84B3-4106-9141-5A366328F3EB}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1736,6 +1738,22 @@ Global {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x64.Build.0 = Release|x64 {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x86.ActiveCfg = Release|Any CPU {C4FA3E05-81FA-4151-8BF8-D1B54D486627}.Release|x86.Build.0 = Release|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|ARM64.Build.0 = Debug|ARM64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|x64.ActiveCfg = Debug|x64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|x64.Build.0 = Debug|x64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|x86.ActiveCfg = Debug|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Debug|x86.Build.0 = Debug|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|Any CPU.Build.0 = Release|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|ARM64.ActiveCfg = Release|ARM64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|ARM64.Build.0 = Release|ARM64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|x64.ActiveCfg = Release|x64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|x64.Build.0 = Release|x64 + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|x86.ActiveCfg = Release|Any CPU + {1CE82908-84B3-4106-9141-5A366328F3EB}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1808,6 +1826,7 @@ Global {F48175E2-DB6A-BF83-F5DF-8AF95C881217} = {E47C4250-B5F4-47E7-AFF2-B5E8ECBAEFFD} {7EC3CEA5-F843-EEF2-AD47-B41941AA3978} = {E47C4250-B5F4-47E7-AFF2-B5E8ECBAEFFD} {37569A4A-ADE7-E310-41E4-8372ADE57148} = {CB56B2BD-5ABA-49C7-BD57-21B9CD5C3205} + {1CE82908-84B3-4106-9141-5A366328F3EB} = {CB56B2BD-5ABA-49C7-BD57-21B9CD5C3205} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {CD63E89D-F4FA-42A7-A10B-50CF854078FA} From 44fe0ff26ddd26aa574c61709ed2e3583bf3b76a Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 15:19:23 +0300 Subject: [PATCH 11/25] Improved test quality --- .../LeaderAppointmentTests.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index 5cbf8b03a46..dda878cc434 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -41,6 +41,8 @@ public async Task AppointLeader() { [Fact] public async Task RenewLeaderAppointment() { + var appointmentTimeout = TimeSpan.FromSeconds(1); + // initialize members var replicaSet = new TestReplicaSet(); replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 100, Epoch: 1L)); @@ -50,7 +52,7 @@ public async Task RenewLeaderAppointment() { // initialize Kontroller await using var kontroller = new RaftKontroller(new RaftKontroller.Options { ListenAddress = new(IPAddress.Loopback, 3269), - AppointmentExpiration = TimeSpan.FromSeconds(1), + AppointmentExpiration = appointmentTimeout, ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, @@ -67,12 +69,16 @@ public async Task RenewLeaderAppointment() { Assert.Equal(TestReplicaSet.Host3, leader.Address); // Start renewal process in the background - var process = new RenewalProcess(kontroller, TestReplicaSet.Host3, leader.Epoch); + var process = new RenewalProcess(kontroller, TestReplicaSet.Host3, leader.Epoch, appointmentTimeout); var renewalTask = process.RunAsync(); // Change state of another member so it should be chosen as a leader, but due to renewal it cannot be appointed replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 300, Epoch: 3L)); + await Task.Delay(appointmentTimeout * 3, TestToken); + leader = await WaitForLeaderAsync(kontroller); + Assert.Equal(TestReplicaSet.Host3, leader.Address); + // Now stop renewal process and wait for new leader appointment process.RequestStop(); await renewalTask.WaitAsync(TestToken); @@ -84,7 +90,7 @@ public async Task RenewLeaderAppointment() { await kontroller.StopAsync(TestToken); } - private sealed class RenewalProcess(IKontroller kontroller, EndPoint address, ulong epoch) { + private sealed class RenewalProcess(IKontroller kontroller, EndPoint address, ulong epoch, TimeSpan appointmentTimeout) { private volatile bool _stopped; public void RequestStop() => _stopped = true; @@ -92,7 +98,7 @@ private sealed class RenewalProcess(IKontroller kontroller, EndPoint address, ul public async Task RunAsync() { while (!_stopped) { // Renew every 300 ms - await Task.Delay(300); + await Task.Delay(appointmentTimeout / 3); await kontroller.RenewLeaderAppointmentAsync(Database.MainDatabaseId, address, epoch, TestToken); } From 1f26ef1655e93a90fd9d86289bc4ead9d6a93a5d Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 15:19:37 +0300 Subject: [PATCH 12/25] Fixed incorrect Epoch calculation --- src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs | 4 ++-- .../StateMachine/LogEntries/ReplicationHelpers.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 6f36240f348..461f6ecc70b 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -115,14 +115,14 @@ private async Task AppointLeaderAsync(string databaseId, ulong epoch, IReadOnlyL // Appoint the leader if (await _raft.AppointLeaderAsync(databaseId, epoch, candidate, token)) - _appointmentState[databaseId] = new LeaderAppointment(candidate, epoch); + _appointmentState[databaseId] = new LeaderAppointment(candidate, epoch + 1UL); // appointment increments the Epoch static IEnumerable>> GetReplicaState( IDatabaseReplicaSet replicas, IEnumerable<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) => nodes - .Where(static node => !node.IsReadOnlyReplica) // r/o replicas cannot contribute to the qorum + .Where(static node => !node.IsReadOnlyReplica) // r/o replicas cannot contribute to the quorum .Select(node => GetReplicaStateAsync(replicas, node.Address, token)); static async Task> GetReplicaStateAsync( diff --git a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs index db977bd1b7a..eb6c60d40ff 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/LogEntries/ReplicationHelpers.cs @@ -53,7 +53,7 @@ public async ValueTask AppointLeaderAsync(string databaseId, await raft.ReplicateAsync( new ProtobufLogEntry(new() { Address = address.ToByteString(), DatabaseId = databaseId, Epoch = epoch }) - { Term = raft.Term }, token); + { Term = raft.Term, Context = box }, token); return box.Value; } } From edcbe5b598e1d4a0e327409a106ebd5c0f0ead93 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 15:27:27 +0300 Subject: [PATCH 13/25] Leader appointment should work even if the previous leader becomes unavailable --- .../LeaderAppointmentTests.cs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index dda878cc434..01b51108f82 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -72,14 +72,14 @@ public async Task RenewLeaderAppointment() { var process = new RenewalProcess(kontroller, TestReplicaSet.Host3, leader.Epoch, appointmentTimeout); var renewalTask = process.RunAsync(); - // Change state of another member so it should be chosen as a leader, but due to renewal it cannot be appointed - replicaSet.UpdateMember(TestReplicaSet.Host1, new(UncommittedOffset: 300, Epoch: 3L)); + // Make the current member as unavailable, but due to renewal process a new leader cannot be appointed + replicaSet.RemoveMember(TestReplicaSet.Host3); await Task.Delay(appointmentTimeout * 3, TestToken); leader = await WaitForLeaderAsync(kontroller); Assert.Equal(TestReplicaSet.Host3, leader.Address); - // Now stop renewal process and wait for new leader appointment + // Now stop renewal process and wait for a new leader appointment process.RequestStop(); await renewalTask.WaitAsync(TestToken); @@ -148,6 +148,16 @@ public void UpdateMember(IPEndPoint endPoint, ReplicaState state) { } } + public void RemoveMember(IPEndPoint endPoint) { + for (ImmutableDictionary current = _members, tmp;; current = tmp) { + var newDictionary = current.Remove(endPoint); + tmp = Interlocked.CompareExchange(ref _members, newDictionary, current); + + if (ReferenceEquals(current, tmp)) + break; + } + } + ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) => Volatile.Read(in _members).TryGetValue(address, out var state) ? new(state) From a4d78d4bb00c6121319c3d093dfa42f5bd434615 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 15:29:18 +0300 Subject: [PATCH 14/25] Removed redundant helper method --- src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs | 8 +------- src/KurrentDB.KontrolPlane/RaftKontroller.cs | 3 --- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs index 48edd948bb9..40ca68ee52f 100644 --- a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -23,12 +23,6 @@ public RaftKontrollerTests() { private IKontroller Kontroller => _kontroller; - [Fact] - public async Task LeaderElection() { - Assert.False(Kontroller.LeadershipToken.IsCancellationRequested); - Assert.Equal(Address, await Kontroller.WaitForLeaderAsync(TestToken)); - } - [Fact] public async Task MainDatabasePresence() { var database = await Kontroller.GetDatabaseAsync(Database.MainDatabaseId, TestToken); @@ -189,7 +183,7 @@ public override async ValueTask DisposeAsync() { public override async ValueTask InitializeAsync() { await base.InitializeAsync(); await _kontroller.StartAsync(TestToken); - await _kontroller.EnsureLeadershipAsync(TestToken); + await _kontroller.WaitForLeaderAsync(TestToken); } private static CancellationToken TestToken => TestContext.Current.CancellationToken; diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 67ad548490b..5a232f5c4be 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -46,9 +46,6 @@ public required IDatabaseReplicaSet ReplicaSet { init => field = value ?? throw new ArgumentNullException(nameof(value)); } - public Task EnsureLeadershipAsync(CancellationToken token) - => _raft.WaitForLeadershipAsync(token); - public async Task StartAsync(CancellationToken token) { _state.Recover(); await _raft.StartAsync(token); From dd7df6c5be0a9a3cd4fddb3d23dbd0580151ed79 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 18:42:41 +0300 Subject: [PATCH 15/25] Fixed snapshot construction --- .../RaftKontroller.Appointment.cs | 4 +- .../RaftKontroller.Impl.cs | 4 +- .../RaftKontroller.Options.cs | 5 ++ src/KurrentDB.KontrolPlane/RaftKontroller.cs | 10 ++-- ...t.Database.cs => ClusterState.Database.cs} | 2 +- ...{Snapshot.Node.cs => ClusterState.Node.cs} | 2 +- ...fCounter.cs => ClusterState.RefCounter.cs} | 2 +- ...pshot.Schema.cs => ClusterState.Schema.cs} | 52 +++++++++---------- ...rsioning.cs => ClusterState.Versioning.cs} | 2 +- .../{Snapshot.cs => ClusterState.cs} | 4 +- .../ClusterStateMachine.Handlers.cs | 12 ++--- .../ClusterStateMachine.Snapshot.cs | 23 ++++---- .../StateMachine/ClusterStateMachine.cs | 47 ++++++++++------- 13 files changed, 94 insertions(+), 75 deletions(-) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.Database.cs => ClusterState.Database.cs} (98%) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.Node.cs => ClusterState.Node.cs} (99%) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.RefCounter.cs => ClusterState.RefCounter.cs} (96%) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.Schema.cs => ClusterState.Schema.cs} (68%) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.Versioning.cs => ClusterState.Versioning.cs} (93%) rename src/KurrentDB.KontrolPlane/StateMachine/{Snapshot.cs => ClusterState.cs} (96%) diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 461f6ecc70b..fb87487f01d 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -58,12 +58,12 @@ private void RemoveDeletedDatabases( } private void StartAppointments( - Snapshot snapshot, + ClusterState clusterState, List tasks, HashSet databases, CancellationToken token) { // Process appointment for every database in parallel - using (snapshot.RentConnection(out var connection)) { + using (clusterState.RentConnection(out var connection)) { // Workaround: it's not possible to enumerate to query results within the same connection. // Thus, we need to materialize (ToList) the first query var currentDBs = connection.GetDatabasesWithEpoch().ToList(); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index 0f2d3d88c00..a677e6e7c18 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -100,9 +100,9 @@ public async IAsyncEnumerable ListenDatabaseAsync(string databa } } - private static DatabaseCluster? GetDatabaseCluster(Snapshot snapshot, + private static DatabaseCluster? GetDatabaseCluster(ClusterState clusterState, string databaseId) { - using (snapshot.RentConnection(out var connection)) { + using (clusterState.RentConnection(out var connection)) { return connection.GetDatabase(databaseId).FirstOrDefault().TryGet(out var database) ? new() { Nodes = GetDatabaseNodes(connection, databaseId, out var leaderAddress), diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs index a6ba6c950a4..442375e3226 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs @@ -40,5 +40,10 @@ public bool SingleNodeDeployment { get; init; } + + public int SnapshotDepth { + get => field is 0 ? 100 : field; + init => field = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value)); + } } } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 5a232f5c4be..8119a14e27e 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -21,7 +21,12 @@ public partial class RaftKontroller : IAsyncDisposable { public RaftKontroller(in Options options) { var stateLocation = new DirectoryInfo(Path.Combine(options.WalOptions.Location, "ktrl")); var configStorageLocation = Path.Combine(options.WalOptions.Location, "members.list"); - _state = new(stateLocation, options.ConnectionPoolCapacity); + _state = new(stateLocation, options.ConnectionPoolCapacity) { + SnapshotDepth = options.SnapshotDepth + }; + + // Must be recovered before initialization of the WAL, which can apply log entries at construction time + _state.Recover(); _wal = new WriteAheadLog(options.WalOptions, _state); var config = new RaftCluster.TcpConfiguration(options.ListenAddress) { @@ -47,7 +52,6 @@ public required IDatabaseReplicaSet ReplicaSet { } public async Task StartAsync(CancellationToken token) { - _state.Recover(); await _raft.StartAsync(token); _leadershipTask = HandleLeadershipAsync(); } @@ -62,7 +66,7 @@ public async ValueTask DisposeAsync() { await CancelAsync(); await _raft.DisposeAsync(); await _wal.DisposeAsync(); - _state.Dispose(); + await _state.DisposeAsync(); } private ValueTask CancelAsync() { diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Database.cs similarity index 98% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Database.cs index 6c4d2d877bf..bffec22de9c 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Database.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Database.cs @@ -9,7 +9,7 @@ namespace KurrentDB.KontrolPlane.StateMachine; using LogEntries; -partial class Snapshot { +partial class ClusterState { public void Update(AddOrUpdateDatabase command, in CommandInfo info) => Update(new(command), info); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Node.cs similarity index 99% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Node.cs index e36b18d3acf..eabfeb033ab 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Node.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Node.cs @@ -9,7 +9,7 @@ namespace KurrentDB.KontrolPlane.StateMachine; using LogEntries; -partial class Snapshot { +partial class ClusterState { public void Update(AddOrUpdateDatabaseNode command, in CommandInfo info) => Update(new(command), info); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.RefCounter.cs similarity index 96% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.RefCounter.cs index 27e27080f90..122ee3f64af 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.RefCounter.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.RefCounter.cs @@ -3,7 +3,7 @@ namespace KurrentDB.KontrolPlane.StateMachine; -partial class Snapshot { +partial class ClusterState { private ulong _referenceCounter; public bool TryAcquire() { diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs similarity index 68% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs index df19b5456d2..e70477a8f51 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Schema.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs @@ -7,33 +7,33 @@ namespace KurrentDB.KontrolPlane.StateMachine; -partial class Snapshot { +partial class ClusterState { private static readonly string LatestSchema = $""" - CREATE TABLE metadata ( - version INTEGER NOT NULL, - index BIGINT NOT NULL, - term BIGINT NOT NULL, - ); - - CREATE TABLE database ( - id VARCHAR PRIMARY KEY, - description VARCHAR NOT NULL DEFAULT '', - epoch UBIGINT NOT NULL DEFAULT 0, - ); - - CREATE TABLE node ( - address BLOB NOT NULL, - database_id VARCHAR NOT NULL, - is_read_only_replica BOOL NOT NULL, - is_leader BOOL NOT NULL DEFAULT FALSE, - FOREIGN KEY (database_id) REFERENCES database (id) - ); - - CREATE UNIQUE INDEX node_id ON node (database_id, address); - CREATE INDEX node_database ON node (database_id); - - INSERT INTO database (id) VALUES ('{Database.MainDatabaseId}'); - INSERT INTO metadata VALUES ({LatestVersion}, 0, 0); + CREATE TABLE metadata ( + version INTEGER NOT NULL, + index BIGINT NOT NULL, + term BIGINT NOT NULL, + ); + + CREATE TABLE database ( + id VARCHAR PRIMARY KEY, + description VARCHAR NOT NULL DEFAULT '', + epoch UBIGINT NOT NULL DEFAULT 0, + ); + + CREATE TABLE node ( + address BLOB NOT NULL, + database_id VARCHAR NOT NULL, + is_read_only_replica BOOL NOT NULL, + is_leader BOOL NOT NULL DEFAULT FALSE, + FOREIGN KEY (database_id) REFERENCES database (id) + ); + + CREATE UNIQUE INDEX node_id ON node (database_id, address); + CREATE INDEX node_database ON node (database_id); + + INSERT INTO database (id) VALUES ('{Database.MainDatabaseId}'); + INSERT INTO metadata VALUES ({LatestVersion}, 0, 0); """; /// diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Versioning.cs similarity index 93% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Versioning.cs index a9605b437ef..b4ad6054316 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.Versioning.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Versioning.cs @@ -5,7 +5,7 @@ namespace KurrentDB.KontrolPlane.StateMachine; -partial class Snapshot { +partial class ClusterState { private const int LatestVersion = 0; private static SortedDictionary> MigrationActions diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs similarity index 96% rename from src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs rename to src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs index f494f9413f4..54c2b54160f 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Snapshot.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.cs @@ -18,12 +18,12 @@ namespace KurrentDB.KontrolPlane.StateMachine; /// the heavyweight dump process. So we can start COPY statement in the background while appending new log entries. /// This concurrency model is preserved by DuckDB 1.5 and later. /// -internal sealed partial class Snapshot : Disposable { +internal sealed partial class ClusterState : Disposable { private readonly DuckDBConnectionPool _pool; private readonly DuckDBAdvancedConnection _writeConnection; private CommandInfo _lastCommandInfo; - public Snapshot(int connectionPoolCapacity) { + public ClusterState(int connectionPoolCapacity) { // For every instance of this class, we have a separated instance of the in-memory database _writeConnection = new DuckDBAdvancedConnection { ConnectionString = "DataSource=:memory:" }; _writeConnection.Open(); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs index 1c01767ad08..3e8268d3093 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Handlers.cs @@ -13,7 +13,7 @@ namespace KurrentDB.KontrolPlane.StateMachine; partial class ClusterStateMachine { // Apply log entry from the WAL to the current state private async ValueTask ApplyAsync(LogEntry entry, CancellationToken token) { - var currentState = _snapshot; + var currentState = _state; var commandInfo = new CommandInfo(entry); switch (entry.CommandId) { case LogEntries.AddOrUpdateDatabase.TypeId: @@ -58,13 +58,13 @@ private async ValueTask ApplyAsync(LogEntry entry, CancellationToken token return entry.Index; } - private void Apply(Snapshot currentState, LogEntries.AddOrUpdateDatabase command, in CommandInfo commandInfo) { + private void Apply(ClusterState currentState, LogEntries.AddOrUpdateDatabase command, in CommandInfo commandInfo) { currentState.Update(command, in commandInfo); _databases.GetOrAdd(command.DatabaseId, static _ => new AsyncStateTracker()).TryAdvance(); } - private void Apply(Snapshot currentState, + private void Apply(ClusterState currentState, LogEntries.RemoveDatabase command, in CommandInfo commandInfo, StrongBox? resultContainer) { @@ -75,13 +75,13 @@ private void Apply(Snapshot currentState, resultContainer?.Value = result; } - private void Apply(Snapshot currentState, LogEntries.AddOrUpdateDatabaseNode command, in CommandInfo commandInfo) { + private void Apply(ClusterState currentState, LogEntries.AddOrUpdateDatabaseNode command, in CommandInfo commandInfo) { currentState.Update(command, in commandInfo); _databases.TryGetValue(command.DatabaseId).ValueOrDefault?.TryAdvance(); } - private void Apply(Snapshot currentState, + private void Apply(ClusterState currentState, LogEntries.RemoveDatabaseNode command, in CommandInfo commandInfo, StrongBox? resultContainer) { @@ -92,7 +92,7 @@ private void Apply(Snapshot currentState, resultContainer?.Value = result; } - private void Apply(Snapshot currentState, LogEntries.AppointLeader command, in CommandInfo commandInfo, StrongBox? resultContainer) { + private void Apply(ClusterState currentState, LogEntries.AppointLeader command, in CommandInfo commandInfo, StrongBox? resultContainer) { var result = currentState.Update(command, in commandInfo); _databases.TryGetValue(command.DatabaseId).ValueOrDefault?.TryAdvance(); diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs index 6bba0356488..9c3a498aa1a 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.Snapshot.cs @@ -60,27 +60,26 @@ private async ValueTask InstallSnapshotAsync(LogEntry entry, CancellationT return entry.Index; } - private Snapshot InstallSnapshot(string fileName) { - var newSnapshot = new Snapshot(_poolCapacity); + private ClusterState InstallSnapshot(string fileName) { + var newSnapshot = new ClusterState(_poolCapacity); newSnapshot.LoadFromFile(fileName); // swap current state - Interlocked.Exchange(ref _snapshot, newSnapshot).Release(); + Interlocked.Exchange(ref _state, newSnapshot).Release(); RefreshDatabaseTrackers(newSnapshot); return newSnapshot; } - private Task SaveSnapshotAsync(Snapshot snapshot, CommandInfo info, CancellationToken token) - => Task.Run(() => SaveSnapshot(snapshot, info), token); + private Task SaveSnapshotAsync(ClusterState clusterState, CommandInfo info, CancellationToken token) + => Task.Run(() => SaveSnapshot(clusterState, info), token); - private void SaveSnapshot(Snapshot snapshot, in CommandInfo info) { + private void SaveSnapshot(ClusterState clusterState, in CommandInfo info) { var snapshotFileName = Path.Combine(_location.FullName, info.Index.ToString(InvariantCulture)); - try { - snapshot.SaveToFile(snapshotFileName); - } catch when (File.Exists(snapshotFileName)) { - File.Delete(snapshotFileName); - throw; - } + var tempFileName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + clusterState.SaveToFile(tempFileName); + + // This operation is atomic on modern file systems + File.Move(tempFileName, snapshotFileName, overwrite: true); _persistentSnapshot = new(snapshotFileName, info); } diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs index d8de2b125db..8c79274f21b 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterStateMachine.cs @@ -14,19 +14,19 @@ namespace KurrentDB.KontrolPlane.StateMachine; /// /// Represents internal Kontrol Plane database. /// -internal sealed partial class ClusterStateMachine : Disposable, IStateMachine { +internal sealed partial class ClusterStateMachine : Disposable, IStateMachine, IAsyncDisposable { private readonly DirectoryInfo _location; private readonly int _poolCapacity; private readonly ConcurrentDictionary _databases; - private volatile Snapshot _snapshot; - private Task? _snapshotTask; + private volatile ClusterState _state; + private volatile Task? _snapshotTask; public ClusterStateMachine(DirectoryInfo location, int connectionPoolCapacity) { if (!location.Exists) location.Create(); _location = location; - _snapshot = new(connectionPoolCapacity); + _state = new(connectionPoolCapacity); _databases = new(); _poolCapacity = connectionPoolCapacity; } @@ -49,16 +49,16 @@ public void Recover() { var newSnapshot = InstallSnapshot(latestSnapshotFile.FullName); _persistentSnapshot = new(latestSnapshotFile.FullName, newSnapshot.LastAppliedCommand); } else { - _snapshot.Initialize(); + _state.Initialize(); _databases[Database.MainDatabaseId] = new(); } snapshots.Clear(); // help GC } - private void RefreshDatabaseTrackers(Snapshot snapshot) { + private void RefreshDatabaseTrackers(ClusterState clusterState) { var loadedTrackers = new HashSet(); - using (snapshot.RentConnection(out var connection)) { + using (clusterState.RentConnection(out var connection)) { foreach (var databaseId in connection.GetDatabases()) { loadedTrackers.Add(databaseId); } @@ -99,19 +99,19 @@ public int SnapshotDepth { /// Tracks database changes as a stream of state snapshots. /// /// - /// The caller must release the snapshot with method. + /// The caller must release the snapshot with method. /// /// The identifier of the database. /// The token that can be used to cancel the operation. /// A stream over cluster state snapshots. - public async IAsyncEnumerable TrackChangesAsync(string databaseId, + public async IAsyncEnumerable TrackChangesAsync(string databaseId, [EnumeratorCancellation] CancellationToken token) { for (AsyncStateTracker.Token currentState;; token.ThrowIfCancellationRequested()) { if (!_databases.TryGetValue(databaseId, out var tracker) || IsDisposingOrDisposed) break; currentState = tracker.CurrentState; - var snapshotCopy = _snapshot; + var snapshotCopy = _state; // The current snapshot cannot be acquired, which means that it's no longer available. Retry the operation // and do Yield() to increase a chance to get latest snapshot copy from '_snapshot' field @@ -130,15 +130,15 @@ public async IAsyncEnumerable TrackChangesAsync(string databaseId, /// Captures the current database state asynchronously. /// /// - /// The caller must release the snapshot with method. + /// The caller must release the snapshot with method. /// /// The token that can be used to cancel the operation. /// The acquired snapshot. - public async ValueTask CaptureCurrentStateAsync(CancellationToken token) { - Snapshot snapshotCopy; + public async ValueTask CaptureCurrentStateAsync(CancellationToken token) { + ClusterState clusterStateCopy; for (;; token.ThrowIfCancellationRequested()) { - snapshotCopy = _snapshot; - if (snapshotCopy.TryAcquire()) + clusterStateCopy = _state; + if (clusterStateCopy.TryAcquire()) break; // The current snapshot cannot be acquired, which means that it's no longer available. Retry the operation @@ -146,11 +146,11 @@ public async ValueTask CaptureCurrentStateAsync(CancellationToken toke await Task.Yield(); } - return snapshotCopy; + return clusterStateCopy; } ValueTask IStateMachine.ApplyAsync(LogEntry entry, CancellationToken token) { - var lastAppliedIndex = _snapshot.LastAppliedCommand.Index; + var lastAppliedIndex = _state.LastAppliedCommand.Index; if (entry.Index <= lastAppliedIndex) return ValueTask.FromResult(lastAppliedIndex); @@ -167,11 +167,22 @@ private void CompleteTrackers() { protected override void Dispose(bool disposing) { if (disposing) { - _snapshot.Release(); + _state.Release(); CompleteTrackers(); _databases.Clear(); } base.Dispose(disposing); } + + protected override async ValueTask DisposeAsyncCore() { + try { + await (_snapshotTask ?? Task.CompletedTask) + .ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); + } finally { + Dispose(disposing: true); + } + } + + public new ValueTask DisposeAsync() => base.DisposeAsync(); } From deb31ed1eba8418fb7de92f2d04306196e8a7b07 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 18:42:49 +0300 Subject: [PATCH 16/25] Added persistence tests --- .../WalPersistenceTests.cs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs diff --git a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs new file mode 100644 index 00000000000..f27fbac2b10 --- /dev/null +++ b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. +// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). + +using System.Net; +using KurrentDB.Core.XUnit.Tests; + +namespace KurrentDB.KontrolPlane; + +public sealed class WalPersistenceTests : DirectoryFixture { + private const int SnapshotDepth = 10; + + [Fact] + public async Task SnapshotPersistence() { + await using (var kontroller = new RaftKontroller(new RaftKontroller.Options { + ListenAddress = new(IPAddress.Loopback, 3269), + AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + ConnectionPoolCapacity = 10, + WalOptions = new() { + Location = Directory, + }, + SingleNodeDeployment = true, + SnapshotDepth = SnapshotDepth, + }) { + ReplicaSet = new TestReplicaSet(), + }) { + + await kontroller.StartAsync(TestToken); + + // Add databases to trigger the snapshot construction + for (var i = 0; i < SnapshotDepth + 2; i++) { + await kontroller.AddOrUpdateDatabaseAsync(new Database { Id = i.ToString() }); + } + + await kontroller.StopAsync(TestToken); + } + + // Recover persistent state + await using (var kontroller = new RaftKontroller(new RaftKontroller.Options { + ListenAddress = new(IPAddress.Loopback, 3269), + AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + ConnectionPoolCapacity = 10, + WalOptions = new() { + Location = Directory, + }, + SingleNodeDeployment = true, + SnapshotDepth = SnapshotDepth, + }) { + ReplicaSet = new TestReplicaSet(), + }) { + + await kontroller.StartAsync(TestToken); + var databases = await kontroller.GetDatabasesAsync(TestToken); + for (var i = 0; i < SnapshotDepth + 2; i++) { + Assert.Contains(i.ToString(), databases); + } + } + } + + private static CancellationToken TestToken => TestContext.Current.CancellationToken; + + private sealed class TestReplicaSet : IDatabaseReplicaSet { + ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + => ValueTask.FromException(new NotSupportedException()); + } +} From e0fc1d8c39b00d6286d66e33bc5c194a9dc82e1c Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 18:59:09 +0300 Subject: [PATCH 17/25] Avoid parallel execution of tests to avoid sharing of the same network port --- src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs | 1 + src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs | 1 + src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index 01b51108f82..04c7db0fc0d 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -7,6 +7,7 @@ namespace KurrentDB.KontrolPlane; +[Collection("KontrolPlane")] public sealed class LeaderAppointmentTests : DirectoryFixture { [Fact] public async Task AppointLeader() { diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs index 40ca68ee52f..2b482f87144 100644 --- a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -3,6 +3,7 @@ namespace KurrentDB.KontrolPlane; +[Collection("KontrolPlane")] public class RaftKontrollerTests : DirectoryFixture { private static readonly IPEndPoint Address = new(IPAddress.Loopback, 3269); private readonly RaftKontroller _kontroller; diff --git a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs index f27fbac2b10..fa182e60911 100644 --- a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs @@ -6,6 +6,7 @@ namespace KurrentDB.KontrolPlane; +[Collection("KontrolPlane")] public sealed class WalPersistenceTests : DirectoryFixture { private const int SnapshotDepth = 10; From f07d0868907f6e06dd922a87cb9c7760fd8ae659 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 19:14:11 +0300 Subject: [PATCH 18/25] Updated dependencies --- src/Directory.Packages.props | 2 +- src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 0a39584d141..29eb28d7ac8 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -17,7 +17,7 @@ - + diff --git a/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj index 5051791b853..702700a7503 100644 --- a/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj +++ b/src/KurrentDB.KontrolPlane/KurrentDB.KontrolPlane.csproj @@ -11,7 +11,6 @@ - From 74f0b1adf75e38f178db8a2bfed105babd8f424c Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 20:12:19 +0300 Subject: [PATCH 19/25] Fixed collection name --- src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs | 2 +- src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs | 2 +- src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index 04c7db0fc0d..844dd0867c5 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -7,7 +7,7 @@ namespace KurrentDB.KontrolPlane; -[Collection("KontrolPlane")] +[Collection("RaftKontroller")] public sealed class LeaderAppointmentTests : DirectoryFixture { [Fact] public async Task AppointLeader() { diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs index 2b482f87144..9107b359dc5 100644 --- a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -3,7 +3,7 @@ namespace KurrentDB.KontrolPlane; -[Collection("KontrolPlane")] +[Collection("RaftKontroller")] public class RaftKontrollerTests : DirectoryFixture { private static readonly IPEndPoint Address = new(IPAddress.Loopback, 3269); private readonly RaftKontroller _kontroller; diff --git a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs index fa182e60911..b43d53288e6 100644 --- a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs @@ -6,7 +6,7 @@ namespace KurrentDB.KontrolPlane; -[Collection("KontrolPlane")] +[Collection("RaftKontroller")] public sealed class WalPersistenceTests : DirectoryFixture { private const int SnapshotDepth = 10; From 1b06364abf70931ba75dcd16db969acf3823f339 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 20:16:33 +0300 Subject: [PATCH 20/25] Fixed naming convention --- .../LeaderAppointmentTests.cs | 40 +++++++++---------- .../RaftKontrollerTests.cs | 6 +-- .../WalPersistenceTests.cs | 8 ++-- .../{IDatabaseReplicaSet.cs => IDataPlane.cs} | 4 +- .../RaftKontroller.Appointment.cs | 4 +- src/KurrentDB.KontrolPlane/RaftKontroller.cs | 2 +- 6 files changed, 32 insertions(+), 32 deletions(-) rename src/KurrentDB.KontrolPlane/{IDatabaseReplicaSet.cs => IDataPlane.cs} (86%) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index 844dd0867c5..f82ff68a9a7 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -12,10 +12,10 @@ public sealed class LeaderAppointmentTests : DirectoryFixture TestContext.Current.CancellationToken; - private sealed class TestReplicaSet : IDatabaseReplicaSet { + private sealed class TestDataPlane : IDataPlane { public static readonly IPEndPoint Host1 = new(IPAddress.Loopback, 3269); public static readonly IPEndPoint Host2 = new(IPAddress.Loopback, 3270); public static readonly IPEndPoint Host3 = new(IPAddress.Loopback, 3271); @@ -159,7 +159,7 @@ public void RemoveMember(IPEndPoint endPoint) { } } - ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + ValueTask IDataPlane.GetReplicaStateAsync(EndPoint address, CancellationToken token) => Volatile.Read(in _members).TryGetValue(address, out var state) ? new(state) : ValueTask.FromException(new IOException()); diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs index 9107b359dc5..f04d32ae41a 100644 --- a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -18,7 +18,7 @@ public RaftKontrollerTests() { }, SingleNodeDeployment = true, }) { - ReplicaSet = new TestReplicaSet(), + ReplicaSet = new TestDataPlane(), }; } @@ -189,8 +189,8 @@ public override async ValueTask InitializeAsync() { private static CancellationToken TestToken => TestContext.Current.CancellationToken; - private sealed class TestReplicaSet : IDatabaseReplicaSet { - ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + private sealed class TestDataPlane : IDataPlane { + ValueTask IDataPlane.GetReplicaStateAsync(EndPoint address, CancellationToken token) => ValueTask.FromException(new NotSupportedException()); } } diff --git a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs index b43d53288e6..5645db44d54 100644 --- a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs @@ -22,7 +22,7 @@ public async Task SnapshotPersistence() { SingleNodeDeployment = true, SnapshotDepth = SnapshotDepth, }) { - ReplicaSet = new TestReplicaSet(), + ReplicaSet = new TestDataPlane(), }) { await kontroller.StartAsync(TestToken); @@ -46,7 +46,7 @@ public async Task SnapshotPersistence() { SingleNodeDeployment = true, SnapshotDepth = SnapshotDepth, }) { - ReplicaSet = new TestReplicaSet(), + ReplicaSet = new TestDataPlane(), }) { await kontroller.StartAsync(TestToken); @@ -59,8 +59,8 @@ public async Task SnapshotPersistence() { private static CancellationToken TestToken => TestContext.Current.CancellationToken; - private sealed class TestReplicaSet : IDatabaseReplicaSet { - ValueTask IDatabaseReplicaSet.GetReplicaStateAsync(EndPoint address, CancellationToken token) + private sealed class TestDataPlane : IDataPlane { + ValueTask IDataPlane.GetReplicaStateAsync(EndPoint address, CancellationToken token) => ValueTask.FromException(new NotSupportedException()); } } diff --git a/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs b/src/KurrentDB.KontrolPlane/IDataPlane.cs similarity index 86% rename from src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs rename to src/KurrentDB.KontrolPlane/IDataPlane.cs index 5eed74298bf..a8e323c4338 100644 --- a/src/KurrentDB.KontrolPlane/IDatabaseReplicaSet.cs +++ b/src/KurrentDB.KontrolPlane/IDataPlane.cs @@ -6,9 +6,9 @@ namespace KurrentDB.KontrolPlane; /// -/// Manages communication with the member in the replica set. +/// Manages communication with the member in the Data Plane. /// -public interface IDatabaseReplicaSet { +public interface IDataPlane { /// /// Gets the replication state for the specified member. /// diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index fb87487f01d..331cd8a2207 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -118,7 +118,7 @@ private async Task AppointLeaderAsync(string databaseId, ulong epoch, IReadOnlyL _appointmentState[databaseId] = new LeaderAppointment(candidate, epoch + 1UL); // appointment increments the Epoch static IEnumerable>> GetReplicaState( - IDatabaseReplicaSet replicas, + IDataPlane replicas, IEnumerable<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) => nodes @@ -126,7 +126,7 @@ static IEnumerable>> GetReplicaState( .Select(node => GetReplicaStateAsync(replicas, node.Address, token)); static async Task> GetReplicaStateAsync( - IDatabaseReplicaSet replicas, + IDataPlane replicas, EndPoint address, CancellationToken token) => new(address, await replicas.GetReplicaStateAsync(address, token)); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 8119a14e27e..15c7872a4c2 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -46,7 +46,7 @@ public RaftKontroller(in Options options) { _lifecycleToken = _lifecycleTokenSource.Token; } - public required IDatabaseReplicaSet ReplicaSet { + public required IDataPlane ReplicaSet { get; init => field = value ?? throw new ArgumentNullException(nameof(value)); } From 013340483a6f398f7f45baf427d1726ae69477e4 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 20:33:34 +0300 Subject: [PATCH 21/25] Fixed naming --- .../LeaderAppointmentTests.cs | 8 ++++---- src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs | 4 ++-- src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs | 8 ++++---- src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs | 2 +- src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs | 2 +- src/KurrentDB.KontrolPlane/RaftKontroller.cs | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs index f82ff68a9a7..d654b66b796 100644 --- a/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/LeaderAppointmentTests.cs @@ -20,14 +20,14 @@ public async Task AppointLeader() { // initialize Kontroller await using var kontroller = new RaftKontroller(new RaftKontroller.Options { ListenAddress = new(IPAddress.Loopback, 3269), - AppointmentExpiration = TimeSpan.FromSeconds(1), + AppointmentDuration = TimeSpan.FromSeconds(1), ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, }, SingleNodeDeployment = true, }) { - ReplicaSet = replicaSet, + DataPlane = replicaSet, }; await kontroller.StartAsync(TestToken); @@ -53,14 +53,14 @@ public async Task RenewLeaderAppointment() { // initialize Kontroller await using var kontroller = new RaftKontroller(new RaftKontroller.Options { ListenAddress = new(IPAddress.Loopback, 3269), - AppointmentExpiration = appointmentTimeout, + AppointmentDuration = appointmentTimeout, ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, }, SingleNodeDeployment = true, }) { - ReplicaSet = replicaSet, + DataPlane = replicaSet, }; await kontroller.StartAsync(TestToken); diff --git a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs index f04d32ae41a..8cc3f74770f 100644 --- a/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/RaftKontrollerTests.cs @@ -11,14 +11,14 @@ public class RaftKontrollerTests : DirectoryFixture { public RaftKontrollerTests() { _kontroller = new(new RaftKontroller.Options { ListenAddress = Address, - AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + AppointmentDuration = TimeSpan.FromDays(1), // elect leader just once ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, }, SingleNodeDeployment = true, }) { - ReplicaSet = new TestDataPlane(), + DataPlane = new TestDataPlane(), }; } diff --git a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs index 5645db44d54..d07b629e31c 100644 --- a/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs +++ b/src/KurrentDB.KontrolPlane.Tests/WalPersistenceTests.cs @@ -14,7 +14,7 @@ public sealed class WalPersistenceTests : DirectoryFixture public async Task SnapshotPersistence() { await using (var kontroller = new RaftKontroller(new RaftKontroller.Options { ListenAddress = new(IPAddress.Loopback, 3269), - AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + AppointmentDuration = TimeSpan.FromDays(1), // elect leader just once ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, @@ -22,7 +22,7 @@ public async Task SnapshotPersistence() { SingleNodeDeployment = true, SnapshotDepth = SnapshotDepth, }) { - ReplicaSet = new TestDataPlane(), + DataPlane = new TestDataPlane(), }) { await kontroller.StartAsync(TestToken); @@ -38,7 +38,7 @@ public async Task SnapshotPersistence() { // Recover persistent state await using (var kontroller = new RaftKontroller(new RaftKontroller.Options { ListenAddress = new(IPAddress.Loopback, 3269), - AppointmentExpiration = TimeSpan.FromDays(1), // elect leader just once + AppointmentDuration = TimeSpan.FromDays(1), // elect leader just once ConnectionPoolCapacity = 10, WalOptions = new() { Location = Directory, @@ -46,7 +46,7 @@ public async Task SnapshotPersistence() { SingleNodeDeployment = true, SnapshotDepth = SnapshotDepth, }) { - ReplicaSet = new TestDataPlane(), + DataPlane = new TestDataPlane(), }) { await kontroller.StartAsync(TestToken); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs index 331cd8a2207..415dd184a97 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Appointment.cs @@ -88,7 +88,7 @@ private bool IsAppointmentRequired(string databaseId, IReadOnlyList<(EndPoint Ad private async Task AppointLeaderAsync(string databaseId, ulong epoch, IReadOnlyList<(EndPoint Address, bool IsReadOnlyReplica, bool IsLeader)> nodes, CancellationToken token) { var responses = new Dictionary(nodes.Count); - await foreach (var task in Task.WhenEach(GetReplicaState(ReplicaSet, nodes, token))) { + await foreach (var task in Task.WhenEach(GetReplicaState(DataPlane, nodes, token))) { try { var pair = await task; responses.Add(pair.Key, pair.Value); diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs index 442375e3226..e282805a236 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Options.cs @@ -31,7 +31,7 @@ public int ConnectionPoolCapacity { init => field = value > 0 ? value : throw new ArgumentOutOfRangeException(nameof(value)); } - public required TimeSpan AppointmentExpiration { + public required TimeSpan AppointmentDuration { get; init => field = value > TimeSpan.Zero ? value : throw new ArgumentOutOfRangeException(nameof(value)); } diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.cs index 15c7872a4c2..0c554897377 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.cs @@ -40,13 +40,13 @@ public RaftKontroller(in Options options) { }; _leadershipTask = Task.CompletedTask; - _appointmentExpiration = options.AppointmentExpiration; + _appointmentExpiration = options.AppointmentDuration; _appointmentState = new(); _lifecycleTokenSource = new(); _lifecycleToken = _lifecycleTokenSource.Token; } - public required IDataPlane ReplicaSet { + public required IDataPlane DataPlane { get; init => field = value ?? throw new ArgumentNullException(nameof(value)); } From cdf6ad95c625b87168f564826e9b4b12cfc7ccfa Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 21:01:24 +0300 Subject: [PATCH 22/25] Migration to extension block --- .../StateMachine/Queries/QueryHelpers.cs | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs index e886e774084..f954bc3aa15 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs @@ -7,26 +7,22 @@ namespace KurrentDB.KontrolPlane.StateMachine.Queries; internal static class QueryHelpers { - public static QueryResult GetDatabases( - this DuckDBAdvancedConnection connection) - => connection.ExecuteQuery(); + extension(DuckDBAdvancedConnection connection) { + public QueryResult GetDatabases() + => connection.ExecuteQuery(); - public static QueryResult<(string Id, ulong Epoch), AllDatabasesWithEpochQuery> GetDatabasesWithEpoch( - this DuckDBAdvancedConnection connection) - => connection.ExecuteQuery<(string, ulong), AllDatabasesWithEpochQuery>(); + public QueryResult<(string Id, ulong Epoch), AllDatabasesWithEpochQuery> GetDatabasesWithEpoch() + => connection.ExecuteQuery<(string, ulong), AllDatabasesWithEpochQuery>(); - public static QueryResult, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader), AllNodesQuery> GetDatabaseNodes( - this DuckDBAdvancedConnection connection, - string databaseId) - => connection.ExecuteQuery, (EndPoint, bool, bool), AllNodesQuery>(new(databaseId)); + public QueryResult, (EndPoint Address, bool IsReadOnlyReplica, bool IsLeader), AllNodesQuery> GetDatabaseNodes( + string databaseId) + => connection.ExecuteQuery, (EndPoint, bool, bool), AllNodesQuery>(new(databaseId)); - public static QueryResult, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica), LeaderQuery> GetDatabaseLeader( - this DuckDBAdvancedConnection connection, - string databaseId) - => connection.ExecuteQuery, (EndPoint, ulong, bool), LeaderQuery>(new(databaseId)); + public QueryResult, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica), LeaderQuery> GetDatabaseLeader( + string databaseId) + => connection.ExecuteQuery, (EndPoint, ulong, bool), LeaderQuery>(new(databaseId)); - public static QueryResult, (string Description, ulong Epoch), DatabaseQuery> GetDatabase( - this DuckDBAdvancedConnection connection, - string databaseId) - => connection.ExecuteQuery, (string, ulong), DatabaseQuery>(new(databaseId)); + public QueryResult, (string Description, ulong Epoch), DatabaseQuery> GetDatabase(string databaseId) + => connection.ExecuteQuery, (string, ulong), DatabaseQuery>(new(databaseId)); + } } From 7936f8d70647b5ec6f6b4d67a32c59e0c77c48ca Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 21:02:48 +0300 Subject: [PATCH 23/25] Code cleanup --- .../StateMachine/Queries/LeaderQuery.cs | 28 ------------------- .../StateMachine/Queries/QueryHelpers.cs | 4 --- 2 files changed, 32 deletions(-) delete mode 100644 src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs deleted file mode 100644 index 4b9345eca16..00000000000 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/LeaderQuery.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) Kurrent, Inc and/or licensed to Kurrent, Inc under one or more agreements. -// Kurrent, Inc licenses this file to you under the Kurrent License v1 (see LICENSE.md). - -using System.Net; -using System.Runtime.InteropServices; -using Kurrent.Quack; - -namespace KurrentDB.KontrolPlane.StateMachine.Queries; - -[StructLayout(LayoutKind.Auto)] -internal readonly struct LeaderQuery : IQuery, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica)> { - public static ReadOnlySpan CommandText => """ - SELECT n.address, d.epoch, n.is_read_only_replica - FROM node n - JOIN database d ON n.database_id = d.id - WHERE n.is_leader=true AND d.id=? - """u8; - - public static StatementBindingResult Bind(in ValueTuple args, PreparedStatement source) => new(source) { - args.Item1 - }; - - public static (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica) Parse(ref DataChunk.Row row) => new() { - Address = row.ReadBlob().ToEndPoint(), - Epoch = row.ReadUInt64(), - IsReadOnlyReplica = row.ReadBoolean(), - }; -} diff --git a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs index f954bc3aa15..892ef395916 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/Queries/QueryHelpers.cs @@ -18,10 +18,6 @@ public QueryResult GetDatabases() string databaseId) => connection.ExecuteQuery, (EndPoint, bool, bool), AllNodesQuery>(new(databaseId)); - public QueryResult, (EndPoint Address, ulong Epoch, bool IsReadOnlyReplica), LeaderQuery> GetDatabaseLeader( - string databaseId) - => connection.ExecuteQuery, (EndPoint, ulong, bool), LeaderQuery>(new(databaseId)); - public QueryResult, (string Description, ulong Epoch), DatabaseQuery> GetDatabase(string databaseId) => connection.ExecuteQuery, (string, ulong), DatabaseQuery>(new(databaseId)); } From e385dfa3399715e5020223a76caeb68a4eb6426e Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 21:09:31 +0300 Subject: [PATCH 24/25] Removed redundant SQL statement --- src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs index e70477a8f51..664058da3ce 100644 --- a/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs +++ b/src/KurrentDB.KontrolPlane/StateMachine/ClusterState.Schema.cs @@ -43,7 +43,6 @@ public void Initialize() { using (_pool.Rent(out var connection)) { using var transaction = connection.BeginTransaction(); connection.ExecuteAdHocNonQuery(LatestSchema, multipleStatements: true); - connection.ExecuteNonQuery(LatestVersion); transaction.CommitOnDispose(); } } From ddfb3fadf75b0ababe21e53fe81884d6f099e4c6 Mon Sep 17 00:00:00 2001 From: Roman Sakno Date: Tue, 30 Jun 2026 21:11:16 +0300 Subject: [PATCH 25/25] Prevent deletion of 'main' database --- src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs index a677e6e7c18..a6d715f6619 100644 --- a/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs +++ b/src/KurrentDB.KontrolPlane/RaftKontroller.Impl.cs @@ -48,6 +48,9 @@ public async ValueTask AddOrUpdateDatabaseAsync(Database database, CancellationT } public async ValueTask RemoveDatabaseAsync(string databaseId, CancellationToken token = default) { + if (databaseId is Database.MainDatabaseId) + throw new ArgumentException($"Built-in '{Database.MainDatabaseId}' database cannot be removed.", nameof(databaseId)); + try { return await _raft.RemoveDatabaseAsync(databaseId, token); } catch (NotLeaderException e) {