Opt-in to API key forwarding. Default to using SeqCli's connection se…

src/SeqCli/Config/Forwarder/SeqCliForwarderConfig.cs

@@ -7,4 +7,5 @@ class SeqCliForwarderConfig
     public SeqCliForwarderStorageConfig Storage { get; set; } = new();
     public SeqCliForwarderDiagnosticConfig Diagnostics { get; set; } = new();
     public SeqCliForwarderApiConfig Api { get; set; } = new();
+    public bool UseApiKeyForwarding { get; set; }
 }

src/SeqCli/Config/KeyValueSettings.cs

@@ -32,7 +32,7 @@ public static void Set(SeqCliConfig config, string key, string? value)
 
         var steps = key.Split('.');
         if (steps.Length < 2)
-            throw new ArgumentException("The format of the key is incorrect; run `seqcli config list` to view all keys.");
+            throw new ArgumentException("The format of the key is incorrect; run `seqcli config` to view all keys.");
 
         object? receiver = config;
         for (var i = 0; i < steps.Length - 1; ++i)
@@ -42,7 +42,7 @@ public static void Set(SeqCliConfig config, string key, string? value)
                 .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[i]);
 
             if (nextStep == null)
-                throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys.");
+                throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys.");
 
             if (nextStep.PropertyType == typeof(Dictionary<string, SeqCliConnectionConfig>))
                 throw new NotSupportedException("Use `seqcli profile create` to configure connection profiles.");
@@ -57,10 +57,10 @@ public static void Set(SeqCliConfig config, string key, string? value)
         // would be more robust.
         var targetProperty = receiver.GetType().GetTypeInfo().DeclaredProperties
             .Where(p => p is { CanRead: true, CanWrite: true } && p.GetMethod!.IsPublic && p.SetMethod!.IsPublic && !p.GetMethod.IsStatic)
-            .SingleOrDefault(p => Camelize(p.Name) == steps[^1]);
+            .SingleOrDefault(p => Camelize(GetUserFacingName(p)) == steps[^1]);
 
         if (targetProperty == null)
-            throw new ArgumentException("The key could not be found; run `seqcli config list` to view all keys.");
+            throw new ArgumentException("The key could not be found; run `seqcli config` to view all keys.");
 
         var targetValue = ChangeType(value, targetProperty.PropertyType);
         targetProperty.SetValue(receiver, targetValue);

src/SeqCli/Forwarder/Channel/ForwardingChannelMap.cs

@@ -5,6 +5,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Seq.Api;
+using SeqCli.Config;
 using SeqCli.Forwarder.Filesystem.System;
 using SeqCli.Forwarder.Storage;
 using Serilog;
@@ -15,26 +16,33 @@ class ForwardingChannelMap
 {
     readonly string _bufferPath;
     readonly SeqConnection _connection;
-    readonly ForwardingChannel _defaultChannel;
+    readonly SeqCliConfig _config;
+    readonly string? _seqCliApiKey;
+
+    // Either seqcli is using its usual connection details and `_seqClieConnectionChannel` is the channel,
+    // or seqcli is using the incoming API key and there is one channel per API key (plus one for no API key) in the dictionary.
     readonly Lock _channelsSync = new();
-    readonly Dictionary<string, ForwardingChannel> _channels = new();
+    ForwardingChannel? _seqCliConnectionChannel = null;
+    readonly Dictionary<string, ForwardingChannel> _channelsByName = new();
+
     readonly CancellationTokenSource _shutdownTokenSource = new();
+    const string SeqCliConnectionChannelName = "SeqCliConnection";
 
-    public ForwardingChannelMap(string bufferPath, SeqConnection connection, string? defaultApiKey)
+    public ForwardingChannelMap(string bufferPath, SeqConnection connection, SeqCliConfig config, string? seqCliApiKey)
     {
         _bufferPath = bufferPath;
         _connection = connection;
-        _defaultChannel = OpenOrCreateChannel(defaultApiKey, "Default");
-
-        // TODO, load other channels at start-up
+        _config = config;
+        _seqCliApiKey = seqCliApiKey;
+
+        LoadChannels();
     }
 
     ForwardingChannel OpenOrCreateChannel(string? apiKey, string name)
     {
-        // TODO, when it's not the default, persist the API key and validate equality on reopen
-
-        var storePath = Path.Combine(_bufferPath, name);
+        var storePath = GetStorePath(name);
         var store = new SystemStoreDirectory(storePath);
+
         Log.Information("Opening local buffer in {StorePath}", storePath);
 
         return new ForwardingChannel(
@@ -45,29 +53,78 @@ ForwardingChannel OpenOrCreateChannel(string? apiKey, string name)
             apiKey,
             _shutdownTokenSource.Token);
     }
-
-    public ForwardingChannel Get(string? apiKey)
+    
+    void LoadChannels()
     {
-        if (string.IsNullOrWhiteSpace(apiKey))
+        if (_config.Forwarder.UseApiKeyForwarding)
         {
-            return _defaultChannel;
+            foreach (var directoryPath in Directory.EnumerateDirectories(_bufferPath))
+            {
+                if (directoryPath.Equals(GetStorePath(SeqCliConnectionChannelName)))
+                {
+                    // data was stored when not using API key forwarding
+                    continue;
+                }
+
+                var path = new SystemStoreDirectory(directoryPath);
+                var apiKey = path.ReadApiKey(_config);
+
+                var channelName = ApiKeyToName(apiKey);
+                var created = OpenOrCreateChannel(apiKey, channelName);
+                _channelsByName.Add(channelName, created);
+            }
         }
-
+        else
+        {
+            _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName);
+        }
+    }
+
+    string GetStorePath(string name)
+    {
+        return Path.Combine(_bufferPath, name);
+    }
+
+    public ForwardingChannel GetApiKeyForwardingChannel(string? requestApiKey)
+    {
         lock (_channelsSync)
         {
-            if (_channels.TryGetValue(apiKey, out var channel))
+            var channelName = ApiKeyToName(requestApiKey);
+
+            if (_channelsByName.TryGetValue(channelName, out var channel))
             {
                 return channel;
             }
 
-            // Seq API keys begin with four identifying characters that aren't considered part of the
-            // confidential key. TODO: we could likely do better than this.
-            var name = apiKey[..4];
-            var created = OpenOrCreateChannel(apiKey, name);
-            _channels.Add(apiKey, created);
+            var created = OpenOrCreateChannel(requestApiKey, channelName);
+            var store = new SystemStoreDirectory(GetStorePath(channelName));
+            if (requestApiKey != null)
+            {
+                store.WriteApiKey(_config, requestApiKey);
+            }
+            _channelsByName.Add(channelName, created);
             return created;
         }
     }
+
+    public ForwardingChannel GetSeqCliConnectionChannel()
+    {
+        lock (_channelsSync)
+        {
+            if (_seqCliConnectionChannel == null)
+            {
+                _seqCliConnectionChannel = OpenOrCreateChannel(_seqCliApiKey, SeqCliConnectionChannelName);
+            }
+            return _seqCliConnectionChannel;
+        }
+    }
+
+    string ApiKeyToName(string? apiKey)
+    {
+        // Seq API keys begin with four identifying characters that aren't considered part of the
+        // confidential key. TODO: we could likely do better than this.
+        return string.IsNullOrEmpty(apiKey) ? "EmptyApiKey" : apiKey[..(Math.Min(apiKey.Length, 4))];
+    }
 
     public async Task StopAsync()
     {
@@ -78,12 +135,15 @@ public async Task StopAsync()
         Task[] stopChannels;
         lock (_channelsSync)
         {
-            stopChannels = _channels.Values.Select(ch => ch.StopAsync()).ToArray();
+            stopChannels = _channelsByName.Values.Select(ch => ch.StopAsync()).ToArray();
+        }
+
+        if (_seqCliConnectionChannel != null)
+        {
+            stopChannels = stopChannels.Append(_seqCliConnectionChannel.StopAsync()).ToArray();
         }
 
-        await Task.WhenAll([
-            _defaultChannel.StopAsync(),
-            ..stopChannels]);
+        await Task.WhenAll([..stopChannels]);
 
         await _shutdownTokenSource.CancelAsync(); 
     }

src/SeqCli/Forwarder/Filesystem/System/SystemStoreDirectory.cs

@@ -16,6 +16,9 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Runtime.InteropServices;
+using System.Text;
+using SeqCli.Config;
+using Serilog;
 
 #if UNIX
 using SeqCli.Forwarder.Filesystem.System.Unix;
@@ -34,6 +37,32 @@ public SystemStoreDirectory(string path)
         if (!Directory.Exists(_directoryPath)) Directory.CreateDirectory(_directoryPath);
     }
 
+    public void WriteApiKey(SeqCliConfig config, string apiKey)
+    {
+        File.WriteAllBytes(
+            Path.Combine(_directoryPath, "api.key"), 
+            config.Encryption.DataProtector().Encrypt(Encoding.UTF8.GetBytes(apiKey)));
+    }
+
+    public string? ReadApiKey(SeqCliConfig config)
+    {
+        string? apiKey = null;
+        var path = Path.Combine(_directoryPath, "api.key");
+
+        if (!File.Exists(path)) return apiKey;
+
+        try
+        {
+            var encrypted = File.ReadAllBytes(path);
+            apiKey = Encoding.UTF8.GetString(config.Encryption.DataProtector().Decrypt(encrypted));
+        }
+        catch (Exception exception)
+        {
+            Log.Warning(exception, "Could not read or decrypt api key");
+        }
+        return apiKey;
+    }
+
     public override SystemStoreFile Create(string name)
     {
         var filePath = Path.Combine(_directoryPath, name);

src/SeqCli/Forwarder/ForwarderModule.cs

@@ -46,7 +46,7 @@ public ForwarderModule(string bufferPath, SeqCliConfig config, SeqConnection con
     protected override void Load(ContainerBuilder builder)
     {
         builder.RegisterType<ServerService>().SingleInstance();
-        builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _apiKey)).SingleInstance();
+        builder.Register(_ => new ForwardingChannelMap(_bufferPath, _connection, _config, _apiKey)).SingleInstance();
 
         builder.RegisterType<IngestionEndpoints>().As<IMapEndpoints>();
 

src/SeqCli/Forwarder/Web/Api/IngestionEndpoints.cs

@@ -24,6 +24,7 @@
 using Microsoft.AspNetCore.Http;
 using Microsoft.Net.Http.Headers;
 using SeqCli.Api;
+using SeqCli.Config;
 using SeqCli.Forwarder.Channel;
 using SeqCli.Forwarder.Diagnostics;
 using JsonException = System.Text.Json.JsonException;
@@ -37,10 +38,12 @@ class IngestionEndpoints : IMapEndpoints
     static readonly Encoding Utf8 = new UTF8Encoding(false);
 
     readonly ForwardingChannelMap _forwardingChannels;
+    readonly SeqCliConfig _config;
 
-    public IngestionEndpoints(ForwardingChannelMap forwardingChannels)
+    public IngestionEndpoints(ForwardingChannelMap forwardingChannels, SeqCliConfig config)
     {
         _forwardingChannels = forwardingChannels;
+        _config = config;
     }
 
     public void MapEndpoints(WebApplication app)
@@ -72,7 +75,10 @@ async Task<IResult> IngestCompactFormatAsync(HttpContext context)
             var cts = CancellationTokenSource.CreateLinkedTokenSource(context.RequestAborted);
             cts.CancelAfter(TimeSpan.FromSeconds(5));
 
-            var log = _forwardingChannels.Get(GetApiKey(context.Request));
+            var requestApiKey = GetApiKey(context.Request);
+            var log = _config.Forwarder.UseApiKeyForwarding 
+                ? _forwardingChannels.GetApiKeyForwardingChannel(requestApiKey) 
+                : _forwardingChannels.GetSeqCliConnectionChannel();
 
             var payload = ArrayPool<byte>.Shared.Rent(1024 * 1024 * 10);
             var writeHead = 0;