diff --git a/CHANGELOG.md b/CHANGELOG.md index 3878b7b5c..893aa8be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,14 @@ follow [Semantic Versioning](https://semver.org). Ongoing work is collected unde ### Added +- **Kiosk demo shows a business-scale deployment.** The kiosk's seeded history now runs at + per-agent business volumes — roughly 1,300–1,700 interactions a day across the four showcase + agents over the 14-day window, with production-sized token counts — so the dashboard's cost, + throughput and token cards read like a real installation instead of a toy. A new simulated + **live traffic feed** keeps fabricating agent calls after boot (paced along the same day/night + curve as the history), so the pulse band, live telemetry and recent-traces feed stay in motion + during a demo even without a real LLM endpoint configured. + - **Agent-proposed test cases.** The trace detail panel gains a **Generate tests** action: an agent reads the trace's *whole* conversation and proposes the test cases actually worth building — the turns where the agent decided something, not every turn — as GREEN promotions (lock in what it diff --git a/Proxytrace.Api.Tests/SeededRandomIsNotUsedForSecretsTests.cs b/Proxytrace.Api.Tests/SeededRandomIsNotUsedForSecretsTests.cs index 9a7b04d92..15dd37d82 100644 --- a/Proxytrace.Api.Tests/SeededRandomIsNotUsedForSecretsTests.cs +++ b/Proxytrace.Api.Tests/SeededRandomIsNotUsedForSecretsTests.cs @@ -38,6 +38,14 @@ public sealed class SeededRandomIsNotUsedForSecretsTests : BaseTest // on every boot. Produces trace statistics, never a credential. ["Proxytrace.Application.Demo.Scenarios.StatisticsBackfillScenario"] = "demo data seeding — deterministic by design, mints no credential", + // Samples the demo traffic's content/token/latency shape for the backfill scenario above + // and the kiosk live feed. Same surface, same rationale: statistics, never a credential. + ["Proxytrace.Application.Demo.Internal.DemoCallPlanner"] = + "demo traffic sampling — deterministic by design, mints no credential", + // Paces and profiles the kiosk's simulated live traffic (delays, agent/endpoint picks). + // Kiosk-only fabricated telemetry, never a credential. + ["Proxytrace.Application.Demo.Internal.KioskLiveTrafficService"] = + "kiosk live traffic pacing — fabricated demo telemetry, mints no credential", }; [TestMethod] diff --git a/Proxytrace.Application.Tests/Demo/DemoSeedingTests.cs b/Proxytrace.Application.Tests/Demo/DemoSeedingTests.cs index c743afc73..a20caaa05 100644 --- a/Proxytrace.Application.Tests/Demo/DemoSeedingTests.cs +++ b/Proxytrace.Application.Tests/Demo/DemoSeedingTests.cs @@ -118,6 +118,42 @@ public async Task Seed_Endpoints_Price_Traces_With_Displayable_Cost() } } + [TestMethod] + public async Task Seed_Backfill_Produces_Business_Scale_Call_Volume() + { + var calls = await services.GetRequiredService>() + .GetAllAsync(CancellationToken); + + // The 14-day backfill runs at per-agent business volumes (DemoTrafficCatalog: ~1,300–1,700 + // interactions/day across the four agents, tool round-trips adding a second call). The + // floor is deliberately below the expected ~30k so random volume draws never flake it, + // while still failing hard if the volumes regress to toy scale. + calls.Count.Should().BeGreaterThan(15_000, + "the kiosk dashboard must show business-scale traffic, not a handful of demo rows"); + + // Throughput on the default 24h dashboard window: the trailing day must carry a full + // business day of calls (minimum daily volume across agents is ~1,270 interactions). + var dayWindowStart = DateTimeOffset.UtcNow.AddHours(-24); + calls.Count(c => c.CreatedAt >= dayWindowStart).Should().BeGreaterThan(700, + "the default dashboard window must read as an active business day"); + } + + [TestMethod] + public async Task Seed_Backfill_Carries_A_Meaningful_Total_Spend() + { + var calls = await services.GetRequiredService>() + .GetAllAsync(CancellationToken); + + decimal total = calls + .Select(c => c.Response?.Usage is { } usage ? c.Endpoint.CalculateCost(usage) ?? 0m : 0m) + .Sum(); + + // ~€20–30/day at the catalog's volumes and token weights → several hundred euros across + // the window. The floor is far below the expectation but far above what toy data produces. + total.Should().BeGreaterThan(100m, + "the seeded window must carry a spend that reads like a real deployment's LLM bill"); + } + [TestMethod] public async Task Seed_Flags_Outlier_Calls_With_Every_Flag_Kind() { diff --git a/Proxytrace.Application.Tests/Demo/KioskLiveTrafficServiceTests.cs b/Proxytrace.Application.Tests/Demo/KioskLiveTrafficServiceTests.cs new file mode 100644 index 000000000..ea1ccf3f6 --- /dev/null +++ b/Proxytrace.Application.Tests/Demo/KioskLiveTrafficServiceTests.cs @@ -0,0 +1,65 @@ +using Autofac; +using AwesomeAssertions; +using Microsoft.Extensions.DependencyInjection; +using NSubstitute; +using Proxytrace.Application.Demo.Internal; +using Proxytrace.Application.Demo.Scenarios; +using Proxytrace.Application.Streaming; +using Proxytrace.Domain; +using Proxytrace.Domain.AgentCall; +using Proxytrace.Domain.Kiosk; +using Nordstein.Core.Testing; + +namespace Proxytrace.Application.Tests.Demo; + +[TestClass] +public class KioskLiveTrafficServiceTests : BaseTest +{ + [TestMethod] + public async Task EmitInteraction_PersistsFreshCalls_ForASeededDemoAgent() + { + var broadcaster = Substitute.For(); + IServiceProvider services = GetServices(builder => + { + builder.RegisterInstance(new KioskOptions { Enabled = true }).AsSelf(); + builder.RegisterInstance(broadcaster).As(); + }); + + await services.GetRequiredService().SeedAsync(CancellationToken); + var callRepo = services.GetRequiredService>(); + int before = (await callRepo.GetAllAsync(CancellationToken)).Count; + + var sut = services.GetRequiredService(); + await sut.EmitInteractionAsync(CancellationToken); + + var after = await callRepo.GetAllAsync(CancellationToken); + // One emission is a single call or a two-call tool round-trip. + (after.Count - before).Should().BeInRange(1, 2); + + var fresh = after.OrderByDescending(c => c.CreatedAt).Take(after.Count - before).ToList(); + fresh.Should().OnlyContain( + c => c.CreatedAt > DateTimeOffset.UtcNow.AddMinutes(-1), + "live traffic must land 'now' so the dashboard's pulse and live telemetry pick it up"); + } + + [TestMethod] + public async Task EmitInteraction_BroadcastsATraceCreatedEvent_PerPersistedCall() + { + var broadcaster = Substitute.For(); + IServiceProvider services = GetServices(builder => + { + builder.RegisterInstance(new KioskOptions { Enabled = true }).AsSelf(); + builder.RegisterInstance(broadcaster).As(); + }); + + await services.GetRequiredService().SeedAsync(CancellationToken); + var callRepo = services.GetRequiredService>(); + int before = (await callRepo.GetAllAsync(CancellationToken)).Count; + + var sut = services.GetRequiredService(); + await sut.EmitInteractionAsync(CancellationToken); + + int added = (await callRepo.GetAllAsync(CancellationToken)).Count - before; + broadcaster.Received(added).Publish(Arg.Any()); + } +} diff --git a/Proxytrace.Application/Demo/Internal/DemoCallPlanner.cs b/Proxytrace.Application/Demo/Internal/DemoCallPlanner.cs new file mode 100644 index 000000000..0a65e1ab5 --- /dev/null +++ b/Proxytrace.Application/Demo/Internal/DemoCallPlanner.cs @@ -0,0 +1,201 @@ +using System.Net; +using Nordstein.Core.AI.Completions; +using Nordstein.Core.AI.Messages; +using Nordstein.Core.AI.Tools; +using Nordstein.Core.Common.Random; +using Proxytrace.Domain.AgentCall; + +namespace Proxytrace.Application.Demo.Internal; + +/// +/// One planned agent call within a simulated interaction. holds the +/// request messages after the agent's system message (the executor prepends that, since +/// only it knows the agent); a means the +/// call errored and has no completion. +/// +internal sealed record PlannedDemoCall( + IReadOnlyList RequestTail, + AssistantMessage? ResponseMessage, + TokenUsage Usage, + int LatencyMs, + HttpStatusCode HttpStatus, + string? ErrorMessage, + string? FinishReason, + OutlierFlags OutlierFlags, + int OffsetSeconds); + +/// +/// A simulated interaction: a single call, or a two-call tool round-trip whose calls share one +/// conversation id (). +/// +internal sealed record DemoInteractionPlan( + IReadOnlyList Calls, + bool SharesConversation); + +/// +/// Samples one simulated interaction (content, token usage, latency, error/outlier dice) from a +/// profile. This is the single source of the demo +/// traffic's statistical shape: the historical backfill and the live traffic feed both draw from +/// it, so "yesterday" and "right now" describe the same business. +/// +internal sealed class DemoCallPlanner +{ + private readonly IRandom random; + + public DemoCallPlanner(IRandom random) + { + this.random = random; + } + + public DemoInteractionPlan Plan(DemoTrafficCatalog.AgentTraffic traffic) + { + bool isError = random.Double() < DemoTrafficCatalog.ErrorRate; + bool isSpike = !isError && random.Double() < DemoTrafficCatalog.TokenSpikeRate; + + if (!isError && !isSpike && traffic.ToolStories.Length > 0 && random.Double() < traffic.ToolRate) + { + return PlanToolConversation(traffic); + } + + return new DemoInteractionPlan([PlanSingleCall(traffic, isError, isSpike)], SharesConversation: false); + } + + private PlannedDemoCall PlanSingleCall(DemoTrafficCatalog.AgentTraffic traffic, bool isError, bool isSpike) + { + var flags = OutlierFlags.None; + string userText; + string assistantText; + ulong inTok; + ulong outTok; + ulong cachedIn; + + if (isSpike) + { + // A conversation that ballooned: far above the profile's baseline mean, with a pasted + // wall of text in the request to match the numbers. The one-off paste also misses the + // prompt cache. + var spike = traffic.Spike; + userText = spike.User; + assistantText = spike.Assistant; + inTok = (ulong)random.Int(spike.MinIn, spike.MaxIn + 1); + outTok = (ulong)random.Int(spike.MinOut, spike.MaxOut + 1); + cachedIn = 0; + flags |= OutlierFlags.HighTokens; + } + else + { + (userText, assistantText) = random.Any(traffic.Pool); + var shape = traffic.Text; + inTok = (ulong)random.Int(shape.MinIn, shape.MaxIn + 1); + outTok = (ulong)random.Int(shape.MinOut, shape.MaxOut + 1); + + // Most calls hit the prompt cache for part of the input; ~30% miss entirely, giving + // the cache-hit KPI and distribution a realistic spread. + cachedIn = CachedShare(inTok); + } + + int latencyMs; + if (random.Double() < DemoTrafficCatalog.LatencySpikeRate) + { + latencyMs = random.Int(4200, 9001); + flags |= OutlierFlags.HighLatency; + } + else if (flags.HasFlag(OutlierFlags.HighTokens)) + { + // A big context takes longer, but stays inside the unflagged latency tail. + latencyMs = random.Int(1600, 3401); + } + else + { + latencyMs = random.Double() < DemoTrafficCatalog.LatencyTailRate + ? random.Int(1500, 3501) + : random.Int(400, 901); + } + + var userMsg = new UserMessage([Content.FromText(userText)]); + + if (isError) + { + var variant = random.Any(DemoTrafficCatalog.ErrorVariants); + return new PlannedDemoCall( + RequestTail: [userMsg], + ResponseMessage: null, + Usage: new TokenUsage(0, 0), + LatencyMs: latencyMs, + HttpStatus: variant.Status, + ErrorMessage: variant.Message, + FinishReason: null, + OutlierFlags: OutlierFlags.None, + OffsetSeconds: 0); + } + + return new PlannedDemoCall( + RequestTail: [userMsg], + ResponseMessage: new AssistantMessage([Content.FromText(assistantText)], []), + Usage: new TokenUsage(inTok, outTok, cachedIn), + LatencyMs: latencyMs, + HttpStatus: HttpStatusCode.OK, + ErrorMessage: null, + FinishReason: "stop", + OutlierFlags: flags, + OffsetSeconds: 0); + } + + private DemoInteractionPlan PlanToolConversation(DemoTrafficCatalog.AgentTraffic traffic) + { + var story = random.Any(traffic.ToolStories); + string orderId = random.Int(10000, 100000).ToString(); + var user = new UserMessage([Content.FromText(story.User(orderId))]); + + var toolRequest = new ToolRequest( + id: $"call_{story.ToolName}_{orderId}", + name: story.ToolName, + arguments: story.Arguments(orderId)); + var assistantToolMsg = new AssistantMessage([], [toolRequest]); + + var shape = traffic.ToolTurn; + ulong toolTurnIn = (ulong)random.Int(shape.MinIn, shape.MaxIn + 1); + var toolCall = new PlannedDemoCall( + RequestTail: [user], + ResponseMessage: assistantToolMsg, + Usage: new TokenUsage( + toolTurnIn, + (ulong)random.Int(shape.MinOut, shape.MaxOut + 1), + CachedShare(toolTurnIn)), + LatencyMs: random.Int(380, 721), + HttpStatus: HttpStatusCode.OK, + ErrorMessage: null, + FinishReason: "tool_calls", + OutlierFlags: OutlierFlags.None, + OffsetSeconds: 0); + + var toolMsg = new ToolMessage(new ToolResponse( + toolRequest, [Content.FromText(story.ToolResult(orderId))])); + var finalAssistant = new AssistantMessage([Content.FromText(story.Final(orderId))], []); + + // The answer turn re-sends the grown conversation (tool result included) and writes the + // user-facing reply, so it carries more input and roughly twice the output of the request + // turn. + ulong finalTurnIn = toolTurnIn + (ulong)random.Int(120, 421); + var answerCall = new PlannedDemoCall( + RequestTail: [user, assistantToolMsg, toolMsg], + ResponseMessage: finalAssistant, + Usage: new TokenUsage( + finalTurnIn, + (ulong)random.Int(shape.MinOut * 2, shape.MaxOut * 2 + 1), + CachedShare(finalTurnIn)), + LatencyMs: random.Int(430, 821), + HttpStatus: HttpStatusCode.OK, + ErrorMessage: null, + FinishReason: "stop", + OutlierFlags: OutlierFlags.None, + OffsetSeconds: random.Int(2, 9)); + + return new DemoInteractionPlan([toolCall, answerCall], SharesConversation: true); + } + + private ulong CachedShare(ulong inTok) + => random.Double() < DemoTrafficCatalog.UncachedShareRate + ? 0UL + : (ulong)(inTok * random.Double(0.3, 0.8)); +} diff --git a/Proxytrace.Application/Demo/Internal/DemoTrafficCatalog.cs b/Proxytrace.Application/Demo/Internal/DemoTrafficCatalog.cs new file mode 100644 index 000000000..116ddc413 --- /dev/null +++ b/Proxytrace.Application/Demo/Internal/DemoTrafficCatalog.cs @@ -0,0 +1,311 @@ +using System.Net; + +namespace Proxytrace.Application.Demo.Internal; + +/// +/// The shared content and shape of the kiosk's simulated agent traffic: per-agent conversation +/// pools, tool round-trip stories, token/volume shapes, and the rates that govern errors, latency +/// tails and outlier spikes. Consumed by StatisticsBackfillScenario (the 14-day historical +/// backfill) and KioskLiveTrafficService (the continuous live feed), so both paint the same +/// business out of the same material and the live traffic is statistically indistinguishable from +/// the history it extends. +/// +internal static class DemoTrafficCatalog +{ + internal const double ErrorRate = 0.03; + internal const double LatencyTailRate = 0.05; + + // Rare, genuinely extreme calls flagged as outliers (matching what the ingestion-time + // detector would flag at mean ± 3σ against the backfill's baseline), so the "outliers only" + // trace filter, the distribution histograms and Tracey's anomaly tools have real data. + internal const double LatencySpikeRate = 0.008; + internal const double TokenSpikeRate = 0.01; + internal const double UncachedShareRate = 0.30; + + internal static readonly int[] DiurnalWeights = + [2, 1, 1, 1, 1, 2, 4, 7, 9, 10, 10, 9, 8, 9, 10, 10, 9, 7, 5, 4, 3, 3, 2, 2]; + + internal static readonly (HttpStatusCode Status, string Message)[] ErrorVariants = + [ + (HttpStatusCode.TooManyRequests, "rate_limit_exceeded"), + (HttpStatusCode.InternalServerError, "internal_error"), + (HttpStatusCode.BadGateway, "bad_gateway"), + ]; + + /// + /// The seeded demo endpoints a profile's traffic is spread across. Consumers resolve these to + /// the actual IModelEndpoint entities via DemoSeedContext. + /// + internal enum DemoEndpointKey + { + Gpt54, + Gpt54Mini, + ClaudeSonnet, + } + + /// + /// One endpoint's share of a profile's traffic; shares within a mix sum to 1. + /// + internal sealed record EndpointShare(DemoEndpointKey Endpoint, double Weight); + + /// + /// Sampled token ranges for one call. Input counts are sized like a production agent's context + /// (system prompt + conversation history + retrieved data), not a bare two-line chat — the + /// token volumes and the resulting cost cards are part of what the kiosk showcases. + /// + internal sealed record TokenShape(int MinIn, int MaxIn, int MinOut, int MaxOut); + + /// + /// Content for a HighTokens outlier: a request with a pasted wall of text and token ranges + /// that match it, so the flag is visibly justified when the trace is opened. + /// + internal sealed record SpikeSample( + string User, + string Assistant, + int MinIn, + int MaxIn, + int MinOut, + int MaxOut); + + /// + /// A two-call tool round-trip (request tool → answer from tool result), templated on a random + /// order id. + /// + internal sealed record ToolStory( + Func User, + string ToolName, + Func Arguments, + Func ToolResult, + Func Final); + + /// + /// Everything that characterizes one demo agent's traffic: what its conversations say, how + /// often they go through a tool round-trip, how many tokens they carry, and how many + /// interactions it handles per day. Daily volumes are per-agent because a real fleet is + /// lopsided — triage and support churn through hundreds of cheap/medium calls while code + /// review sees a few dozen expensive ones. + /// + internal sealed record AgentTraffic( + (string User, string Assistant)[] Pool, + SpikeSample Spike, + ToolStory[] ToolStories, + double ToolRate, + TokenShape Text, + TokenShape ToolTurn, + int MinCallsPerDay, + int MaxCallsPerDay, + EndpointShare[] EndpointMix); + + // Plain-text answers only where no tool is needed; anything that requires an order lookup or a + // return goes through the support ToolStories instead, matching the agent's system prompt. + private static readonly (string User, string Assistant)[] SupportPool = + [ + ("My refund hasn't arrived yet.", "Refunds usually settle in 3-5 business days of us receiving the return. Could you share the order number so I can check where yours is?"), + ("Can you help me change my shipping address?", "Sure — what's the order number?"), + ("Do you offer international shipping?", "Yes — international shipping is available to 38 countries."), + ("How do I reset my password?", "Use the 'Forgot password' link on the login page; a reset email arrives within a minute."), + ("My package shows delivered but it isn't here.", "I'm sorry to hear that — could you send me the order number? I'll check the carrier scan and sort out a replacement."), + ("What payment methods do you accept?", "We accept all major credit cards, PayPal and Klarna. Bank transfer is available for orders over €500."), + ]; + + // Tool-using support interactions: two agent calls sharing a conversation — the model requests + // lookup_order/start_return, then answers from the tool result. + private static readonly ToolStory[] SupportToolStories = + [ + new( + id => $"Hi, where is my order #{id}?", + "lookup_order", + id => $$"""{"order_id":"{{id}}"}""", + id => $$"""{"order_id":"{{id}}","status":"in_transit","carrier":"DHL","eta_days":2}""", + id => $"Order #{id} is in transit with DHL and should arrive within 2 business days. Anything else I can help with?"), + new( + id => $"Has order #{id} shipped yet? I ordered last weekend.", + "lookup_order", + id => $$"""{"order_id":"{{id}}"}""", + id => $$"""{"order_id":"{{id}}","status":"processing","warehouse":"Leipzig"}""", + id => $"Not yet — order #{id} is still being packed at our Leipzig warehouse. Dispatch is expected within 24 hours and the tracking link will arrive by email."), + new( + id => $"Order #{id} arrived with a cracked screen. I'd like to return it.", + "start_return", + id => $$"""{"order_id":"{{id}}","reason":"damaged"}""", + id => $$"""{"return_id":"RMA-{{id}}","label_url":"https://shop.example.com/labels/RMA-{{id}}","refund_estimate_days":3}""", + id => $"Sorry about that! Return RMA-{id} is open for order #{id} — the prepaid label is on its way to your inbox, and the refund lands within 3 business days of us receiving the device."), + ]; + + private static readonly (string User, string Assistant)[] CodeReviewPool = + [ + ("Review this pull request for null safety.", "Two potential NREs in PaymentProcessor.cs; suggest adding guard clauses."), + ("Audit this method for SQL injection risk.", "Parameter `userId` is concatenated; switch to a parameterised query."), + ("Is this implementation thread-safe?", "Field `_cache` is read without synchronisation; consider ConcurrentDictionary."), + ("Comment on naming consistency.", "Mix of camelCase and snake_case in DTOs; align with project style guide."), + ("Flag any obvious performance smells.", "Inner loop allocates a new List per iteration; hoist outside the loop."), + ("Check error handling.", "Bare catch suppresses cancellation; rethrow OperationCanceledException."), + ]; + + // At ToolRate 1.0 every successful analytics interaction goes through the tool stories; this + // pool only supplies the user question for error calls (no response is rendered there). + private static readonly (string User, string Assistant)[] AnalyticsPool = + [ + ("How many active users did we have last week?", "Let me run that query."), + ("Top acquisition channels in May?", "Let me run that query."), + ("DAU/MAU ratio for last month?", "Let me run that query."), + ("Revenue split by region?", "Let me run that query."), + ("Churn rate by plan tier?", "Let me run that query."), + ("Median order value last quarter?", "Let me run that query."), + ]; + + private static readonly ToolStory[] AnalyticsToolStories = + [ + new( + _ => "How many active users did we have last week?", + "run_sql", + _ => """{"query":"SELECT COUNT(DISTINCT user_id) FROM events WHERE event_at >= now() - INTERVAL '7 days';"}""", + n => $$"""{"rows":[{"count":{{n}}}],"row_count":1,"duration_ms":388}""", + n => $"Active users in the last 7 full days: {n}.\n```sql\nSELECT COUNT(DISTINCT user_id)\nFROM events\nWHERE event_at >= now() - INTERVAL '7 days';\n```"), + new( + _ => "How many orders did we take yesterday?", + "run_sql", + _ => """{"query":"SELECT COUNT(*) FROM orders WHERE placed_at::date = (now() - INTERVAL '1 day')::date;"}""", + n => $$"""{"rows":[{"count":{{n}}}],"row_count":1,"duration_ms":214}""", + n => $"Orders taken yesterday: {n}.\n```sql\nSELECT COUNT(*) FROM orders\nWHERE placed_at::date = (now() - INTERVAL '1 day')::date;\n```"), + new( + _ => "What was our total revenue last month?", + "run_sql", + _ => """{"query":"SELECT SUM(total) AS revenue FROM orders WHERE placed_at >= date_trunc('month', now()) - INTERVAL '1 month' AND placed_at < date_trunc('month', now());"}""", + n => $$"""{"rows":[{"revenue":{{n}}.00}],"row_count":1,"duration_ms":492}""", + n => $"Total revenue last month: €{n}.\n```sql\nSELECT SUM(total) AS revenue FROM orders\nWHERE placed_at >= date_trunc('month', now()) - INTERVAL '1 month'\n AND placed_at < date_trunc('month', now());\n```"), + new( + _ => "Which columns can I segment users by?", + "get_schema", + _ => """{"table":"users"}""", + _ => """{"table":"users","columns":[{"name":"id","type":"bigint"},{"name":"plan","type":"text"},{"name":"channel","type":"text"},{"name":"region","type":"text"},{"name":"created_at","type":"timestamptz"},{"name":"last_active_at","type":"timestamptz"}]}""", + _ => "The users table segments on: plan, channel, region, plus the created_at/last_active_at timestamps for cohorting. id is the join key to events."), + ]; + + // Billing/plan questions stay in the plain pool — the triage agent has no tool to look those + // up (that gap is the seeded lookup_customer_plan theory); how-to and known-bug emails go + // through search_kb via the triage ToolStories. + private static readonly (string User, string Assistant)[] TriagePool = + [ + ("Subject: API returns 429 for our nightly import since Tuesday.", "Category: Bug. Priority: P2."), + ("Subject: Please upgrade us to the annual plan.", "Category: Billing. Priority: P3."), + ("Subject: Invoice PDF shows the wrong company address.", "Category: Billing. Priority: P3."), + ("Subject: Can we get SSO with Okta?", "Category: Feature Request. Priority: P4."), + ("Subject: Everything is down again!!", "Category: UI Feedback. Priority: P3."), + ]; + + private static readonly ToolStory[] TriageToolStories = + [ + new( + _ => "Subject: Password reset email never arrives.", + "search_kb", + _ => """{"query":"password reset email not arriving"}""", + _ => """{"articles":[{"id":"KB-217","title":"Password reset emails and domain allowlists","url":"https://help.example.com/kb/217"}]}""", + _ => "Category: Account Access. Priority: P3. Suggested reply: check spam and the domain allowlist (KB-217); an admin can also trigger the reset from the members page."), + new( + _ => "Subject: Dashboard loads blank in Safari.", + "search_kb", + _ => """{"query":"dashboard blank page Safari"}""", + _ => """{"articles":[{"id":"KB-334","title":"Blank dashboard in Safari 17","url":"https://help.example.com/kb/334"}]}""", + _ => "Category: Bug. Priority: P3. Suggested reply: known Safari 17 issue — clearing site data restores the dashboard; permanent fix is rolling out (KB-334)."), + new( + _ => "Subject: How do I bulk-invite my whole team?", + "search_kb", + _ => """{"query":"bulk invite team members CSV"}""", + _ => """{"articles":[{"id":"KB-089","title":"Importing members from CSV","url":"https://help.example.com/kb/089"}]}""", + _ => "Category: How-To. Priority: P4. Suggested reply: Settings → Members → Import CSV handles up to 500 invites at once (KB-089)."), + ]; + + // HighTokens outliers carry content that visibly justifies the token count — a pasted wall of + // text in the request — so opening a flagged trace never shows a two-line chat labelled + // "high token count". Spike ranges sit far above each profile's baseline mean + 3σ so the + // seeded flags match what the ingestion-time detector would compute. + private static readonly SpikeSample SupportSpike = new( + "I've been going back and forth with your team for three weeks about order #58121 and I'm done repeating myself. " + + "Pasting the ENTIRE email thread below so you finally have the full context:\n\n" + + string.Join("\n", Enumerable.Range(1, 90).Select(i => + $"> [message {i}] Re: order #58121 — delivery rescheduled again, promised callback never happened, partial refund of €12.40 discussed but not issued.")), + "Thanks for the full history — here is where order #58121 actually stands: the delivery was rescheduled twice by the carrier, " + + "the €12.40 partial refund agreed in the middle of the thread was never issued, and the replacement lamp shade was never dispatched. " + + "I've escalated this to our fulfillment lead with the full thread attached — you'll receive the refund confirmation and the " + + "replacement tracking link by email today, plus a €10 goodwill voucher for the runaround.", + MinIn: 7000, MaxIn: 11000, MinOut: 300, MaxOut: 520); + + private static readonly SpikeSample CodeReviewSpike = new( + "Please review this whole feature branch in one go (full diff below):\n\n" + + string.Join("\n", Enumerable.Range(1, 150).Select(i => + $"+ public async Task GetOrderVariant_{i:D3}(Guid id) => await orderQuery.LoadAsync(id).ConfigureAwait(false);")), + "This diff adds 150 near-identical query methods — collapse them into one method taking the variant as a parameter. " + + "Concrete issues: (1) every method awaits with ConfigureAwait(false) but the callers rely on the request culture, so localized " + + "formatting silently breaks; (2) none of the methods pass a CancellationToken, so slow queries survive request aborts; " + + "(3) `LoadAsync` throws on missing rows while callers expect null — each variant is a latent 500. " + + "A diff this size also exceeds what one review pass can meaningfully cover; split it next time.", + MinIn: 9000, MaxIn: 14000, MinOut: 450, MaxOut: 800); + + private static readonly SpikeSample AnalyticsSpike = new( + "Here is our full warehouse schema, how many orders did we take yesterday?\n\n" + + string.Join("\n", Enumerable.Range(1, 120).Select(i => + $"CREATE TABLE shard_{i:D3}_orders (id BIGSERIAL PRIMARY KEY, user_id BIGINT NOT NULL, total NUMERIC(10,2) NOT NULL, placed_at TIMESTAMPTZ NOT NULL DEFAULT now());")), + "Your question only needs the orders tables, not the full 120-table schema — pasting it on every request is what makes " + + "these calls expensive. Here's the query I'd run:\n```sql\nSELECT COUNT(*) FROM all_orders\nWHERE placed_at::date = (now() - INTERVAL '1 day')::date;\n```\nWant me to execute it?", + MinIn: 7000, MaxIn: 12000, MinOut: 160, MaxOut: 320); + + private static readonly SpikeSample TriageSpike = new( + "Subject: FWD: FWD: RE: unresolved ticket — forwarding our complete internal thread so you can see everything:\n\n" + + string.Join("\n", Enumerable.Range(1, 80).Select(i => + $"> [reply {i}] RE: intermittent 502s on the reporting API since the last maintenance window; retried nightly import, same result.")), + "Category: Bug. Priority: P2.", + MinIn: 5200, MaxIn: 8000, MinOut: 18, MaxOut: 40); + + // Daily volumes and token shapes sized like a mid-size business running these agents in + // production: support and triage churn through hundreds of interactions a day, analytics is a + // steady internal tool, code review sees fewer but much heavier calls. Together (~1,300–1,700 + // interactions/day at these token weights) the seeded window carries a daily LLM spend in the + // tens of euros — dashboard numbers that read like a real deployment instead of a toy. + // Tool round-trips are the showcase's bread and butter (ToolRate); Analytics runs at 1.0 — its + // prompt forbids invented numbers, so every successful answer is grounded in run_sql/get_schema. + + internal static readonly AgentTraffic Support = new( + SupportPool, + SupportSpike, + SupportToolStories, + ToolRate: 0.35, + Text: new TokenShape(MinIn: 1400, MaxIn: 3600, MinOut: 180, MaxOut: 600), + ToolTurn: new TokenShape(MinIn: 1000, MaxIn: 1500, MinOut: 26, MaxOut: 60), + MinCallsPerDay: 420, + MaxCallsPerDay: 560, + EndpointMix: [new(DemoEndpointKey.Gpt54, 0.70), new(DemoEndpointKey.ClaudeSonnet, 0.30)]); + + internal static readonly AgentTraffic CodeReview = new( + CodeReviewPool, + CodeReviewSpike, + ToolStories: [], + ToolRate: 0, + Text: new TokenShape(MinIn: 2200, MaxIn: 4800, MinOut: 350, MaxOut: 900), + ToolTurn: new TokenShape(MinIn: 2200, MaxIn: 4800, MinOut: 350, MaxOut: 900), + MinCallsPerDay: 100, + MaxCallsPerDay: 150, + EndpointMix: [new(DemoEndpointKey.ClaudeSonnet, 0.80), new(DemoEndpointKey.Gpt54Mini, 0.20)]); + + internal static readonly AgentTraffic Analytics = new( + AnalyticsPool, + AnalyticsSpike, + AnalyticsToolStories, + ToolRate: 1.0, + Text: new TokenShape(MinIn: 900, MaxIn: 1600, MinOut: 60, MaxOut: 160), + ToolTurn: new TokenShape(MinIn: 900, MaxIn: 1600, MinOut: 40, MaxOut: 90), + MinCallsPerDay: 190, + MaxCallsPerDay: 260, + EndpointMix: [new(DemoEndpointKey.Gpt54, 0.60), new(DemoEndpointKey.Gpt54Mini, 0.40)]); + + internal static readonly AgentTraffic Triage = new( + TriagePool, + TriageSpike, + TriageToolStories, + ToolRate: 0.35, + Text: new TokenShape(MinIn: 420, MaxIn: 980, MinOut: 30, MaxOut: 120), + ToolTurn: new TokenShape(MinIn: 380, MaxIn: 700, MinOut: 22, MaxOut: 48), + MinCallsPerDay: 560, + MaxCallsPerDay: 760, + EndpointMix: [new(DemoEndpointKey.Gpt54Mini, 1.00)]); +} diff --git a/Proxytrace.Application/Demo/Internal/KioskLiveTrafficService.cs b/Proxytrace.Application/Demo/Internal/KioskLiveTrafficService.cs new file mode 100644 index 000000000..d6699fb07 --- /dev/null +++ b/Proxytrace.Application/Demo/Internal/KioskLiveTrafficService.cs @@ -0,0 +1,240 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Nordstein.Core.AI.Completions; +using Nordstein.Core.AI.Messages; +using Nordstein.Core.Common.Random; +using Nordstein.Core.Common.Time; +using Proxytrace.Application.Streaming; +using Proxytrace.Domain; +using Proxytrace.Domain.Agent; +using Proxytrace.Domain.AgentCall; +using Proxytrace.Domain.ModelEndpoint; + +namespace Proxytrace.Application.Demo.Internal; + +/// +/// Keeps the kiosk feeling alive after boot: continuously fabricates agent calls from the same +/// / material as the historical +/// backfill and publishes each one to the trace stream, so the dashboard's pulse band, live +/// telemetry, recent-traces feed and the Traces list keep moving without a real LLM endpoint. +/// The pace follows the catalog's diurnal curve (boosted — a kiosk audience should see a new +/// trace every few seconds at peak, not one a minute) with jittered gaps so arrivals read as +/// organic traffic rather than a metronome. Kiosk-only: the composition root swaps in a +/// NullHostedService outside kiosk mode. +/// +internal sealed class KioskLiveTrafficService : BackgroundService +{ + /// + /// Live traffic runs this much faster than the backfill's historical rate. On stage the point + /// is visible motion; a strict continuation of the historical rate would sit idle for half a + /// minute at a time. + /// + private const double LiveRateBoost = 2.5; + + private const double MinDelaySeconds = 4; + private const double MaxDelaySeconds = 120; + + private static readonly TimeSpan SeedPollInterval = TimeSpan.FromMilliseconds(500); + private static readonly TimeSpan SeedWaitTimeout = TimeSpan.FromMinutes(5); + + // The root provider is needed to open a fresh DI scope (and therefore a fresh storage + // context) per emitted interaction — the same reason DemoSeederHostedService injects it. + private readonly IServiceProvider rootServices; + private readonly DemoSeedContext ctx; + private readonly DemoCallPlanner planner; + private readonly ITraceBroadcaster traceBroadcaster; + private readonly IRandom random; + private readonly IClock clock; + private readonly ILogger logger; + + public KioskLiveTrafficService( + IServiceProvider rootServices, + DemoSeedContext ctx, + DemoCallPlanner planner, + ITraceBroadcaster traceBroadcaster, + IRandom random, + IClock clock, + ILogger logger) + { + this.rootServices = rootServices; + this.ctx = ctx; + this.planner = planner; + this.traceBroadcaster = traceBroadcaster; + this.random = random; + this.clock = clock; + this.logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + if (!await WaitForSeedAsync(stoppingToken)) + { + return; + } + + logger.LogInformation("Kiosk live traffic feed started"); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await EmitInteractionAsync(stoppingToken); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // One failed fabrication must not kill the feed for the rest of the demo. + logger.LogWarning(ex, "Kiosk live traffic emission failed; continuing"); + } + + await Task.Delay(NextDelay(), stoppingToken); + } + } + + /// + /// The demo seeder (an IHostedService registered before this service) populates the + /// context before this loop starts; the wait is a safety net for compositions that start + /// hosted services concurrently, not an expected code path. + /// + private async Task WaitForSeedAsync(CancellationToken stoppingToken) + { + DateTimeOffset deadline = clock.UtcNow + SeedWaitTimeout; + while (ctx.CustomerSupportAgent is null) + { + if (clock.UtcNow > deadline) + { + logger.LogError( + "Kiosk live traffic feed disabled: demo seeding did not complete within {Timeout}", + SeedWaitTimeout); + return false; + } + + await Task.Delay(SeedPollInterval, stoppingToken); + } + + return true; + } + + /// + /// Fabricates one interaction (a single call or a two-call tool round-trip) for a + /// volume-weighted random agent, persists it, and publishes each call to the trace stream. + /// Internal so tests can drive one emission directly without running the timed loop. + /// + internal async Task EmitInteractionAsync(CancellationToken cancellationToken) + { + var (traffic, agentId) = PickProfile(); + Guid endpointId = PickEndpointId(traffic); + + using var scope = rootServices.CreateScope(); + var services = scope.ServiceProvider; + var agent = await services.GetRequiredService>() + .GetAsync(agentId, cancellationToken); + var endpoint = await services.GetRequiredService>() + .GetAsync(endpointId, cancellationToken); + var callFactory = services.GetRequiredService(); + var completionFactory = services.GetRequiredService(); + var paramsFactory = services.GetRequiredService(); + var callRepo = services.GetRequiredService>(); + + var plan = planner.Plan(traffic); + Guid? conversationId = plan.SharesConversation ? Guid.NewGuid() : null; + + foreach (var planned in plan.Calls) + { + ICompletion? response = planned.ResponseMessage is null + ? null + : completionFactory( + planned.ResponseMessage, + planned.Usage, + TimeSpan.FromMilliseconds(planned.LatencyMs)); + + var call = callFactory( + agent: agent, + version: agent.CurrentVersion, + endpoint: endpoint, + request: new Conversation([agent.CreateSystemMessage(), .. planned.RequestTail]), + response: response, + httpStatus: planned.HttpStatus, + finishReason: planned.FinishReason, + errorMessage: planned.ErrorMessage, + modelParameters: paramsFactory(temperature: 0.3), + conversationId: conversationId, + outlierFlags: planned.OutlierFlags); + + call = await callRepo.AddAsync(call, cancellationToken); + traceBroadcaster.Publish(TraceCreatedEvent.Create(call)); + } + } + + private (DemoTrafficCatalog.AgentTraffic Traffic, Guid AgentId) PickProfile() + { + var profiles = Profiles(); + double total = profiles.Sum(p => AverageDailyVolume(p.Traffic)); + double pick = random.Double() * total; + double acc = 0; + foreach (var profile in profiles) + { + acc += AverageDailyVolume(profile.Traffic); + if (pick <= acc) + { + return profile; + } + } + return profiles[^1]; + } + + private Guid PickEndpointId(DemoTrafficCatalog.AgentTraffic traffic) + { + double pick = random.Double(); + double acc = 0; + foreach (var share in traffic.EndpointMix) + { + acc += share.Weight; + if (pick <= acc) + { + return EndpointId(share.Endpoint); + } + } + return EndpointId(traffic.EndpointMix[^1].Endpoint); + } + + private Guid EndpointId(DemoTrafficCatalog.DemoEndpointKey key) + => key switch + { + DemoTrafficCatalog.DemoEndpointKey.Gpt54 => ctx.RequireGpt54Endpoint().Id, + DemoTrafficCatalog.DemoEndpointKey.Gpt54Mini => ctx.RequireGpt54MiniEndpoint().Id, + DemoTrafficCatalog.DemoEndpointKey.ClaudeSonnet => ctx.RequireClaudeEndpoint().Id, + _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown demo endpoint key"), + }; + + private (DemoTrafficCatalog.AgentTraffic Traffic, Guid AgentId)[] Profiles() => + [ + (DemoTrafficCatalog.Support, ctx.RequireCustomerSupportAgent().Id), + (DemoTrafficCatalog.CodeReview, ctx.RequireCodeReviewAgent().Id), + (DemoTrafficCatalog.Analytics, ctx.RequireDataAnalyticsAgent().Id), + (DemoTrafficCatalog.Triage, ctx.RequireEmailTriageAgent().Id), + ]; + + private static double AverageDailyVolume(DemoTrafficCatalog.AgentTraffic traffic) + => (traffic.MinCallsPerDay + traffic.MaxCallsPerDay) / 2.0; + + /// + /// The mean gap between interactions follows the boosted diurnal rate for the current hour; + /// each individual gap is jittered ±50% so the feed doesn't tick like a clock. + /// + private TimeSpan NextDelay() + { + int[] weights = DemoTrafficCatalog.DiurnalWeights; + double hourShare = (double)weights[clock.UtcNow.Hour] / weights.Sum(); + double totalDaily = Profiles().Sum(p => AverageDailyVolume(p.Traffic)); + double interactionsPerHour = totalDaily * hourShare * LiveRateBoost; + double meanSeconds = 3600.0 / Math.Max(interactionsPerHour, 1.0); + double seconds = Math.Clamp( + meanSeconds * random.Double(0.5, 1.5), MinDelaySeconds, MaxDelaySeconds); + return TimeSpan.FromSeconds(seconds); + } +} diff --git a/Proxytrace.Application/Demo/Scenarios/StatisticsBackfillScenario.cs b/Proxytrace.Application/Demo/Scenarios/StatisticsBackfillScenario.cs index 3e0e69a95..2724b6341 100644 --- a/Proxytrace.Application/Demo/Scenarios/StatisticsBackfillScenario.cs +++ b/Proxytrace.Application/Demo/Scenarios/StatisticsBackfillScenario.cs @@ -1,6 +1,7 @@ using System.Net; using JetBrains.Annotations; using Nordstein.Core.Common.Random; +using Proxytrace.Application.Demo.Internal; using Proxytrace.Domain; using Proxytrace.Domain.Agent; using Proxytrace.Domain.AgentCall; @@ -10,200 +11,19 @@ using Proxytrace.Domain.TestResult; using Proxytrace.Domain.TestRun; using Proxytrace.Domain.TestRunGroup; -using Nordstein.Core.AI.Tools; namespace Proxytrace.Application.Demo.Scenarios; +/// +/// Backfills the trailing with business-scale agent traffic — content, +/// rates, token weights and per-agent daily volumes all come from +/// via , shared with the live traffic feed. Also staggers the seeded +/// test-run groups across the window so the suite history reads as an ongoing practice. +/// [UsedImplicitly] internal sealed class StatisticsBackfillScenario : IDemoScenario { private const int WindowDays = 14; - private const int MinCallsPerDay = 50; - private const int MaxCallsPerDay = 81; - private const double ErrorRate = 0.03; - private const double LatencyTailRate = 0.05; - - // Rare, genuinely extreme calls flagged as outliers (matching what the ingestion-time - // detector would flag at mean ± 3σ against this backfill's baseline), so the "outliers only" - // trace filter, the distribution histograms and Tracey's anomaly tools have real data. - private const double LatencySpikeRate = 0.008; - private const double TokenSpikeRate = 0.01; - private const double UncachedShareRate = 0.30; - - // Tool round-trips are the showcase's bread and butter: each profile carries a per-agent share - // of interactions that go through a two-call tool conversation (see BackfillProfile.ToolRate). - // Analytics runs at 1.0 — its prompt forbids invented numbers, so every successful answer is - // grounded in run_sql/get_schema. - - private static readonly int[] DiurnalWeights = - [2, 1, 1, 1, 1, 2, 4, 7, 9, 10, 10, 9, 8, 9, 10, 10, 9, 7, 5, 4, 3, 3, 2, 2]; - - private static readonly (HttpStatusCode Status, string Message)[] ErrorVariants = - [ - (HttpStatusCode.TooManyRequests, "rate_limit_exceeded"), - (HttpStatusCode.InternalServerError, "internal_error"), - (HttpStatusCode.BadGateway, "bad_gateway"), - ]; - - // Plain-text answers only where no tool is needed; anything that requires an order lookup or a - // return goes through SupportToolStories instead, matching the agent's system prompt. - private static readonly (string User, string Assistant)[] SupportPool = - [ - ("My refund hasn't arrived yet.", "Refunds usually settle in 3-5 business days of us receiving the return. Could you share the order number so I can check where yours is?"), - ("Can you help me change my shipping address?", "Sure — what's the order number?"), - ("Do you offer international shipping?", "Yes — international shipping is available to 38 countries."), - ("How do I reset my password?", "Use the 'Forgot password' link on the login page; a reset email arrives within a minute."), - ("My package shows delivered but it isn't here.", "I'm sorry to hear that — could you send me the order number? I'll check the carrier scan and sort out a replacement."), - ("What payment methods do you accept?", "We accept all major credit cards, PayPal and Klarna. Bank transfer is available for orders over €500."), - ]; - - // Tool-using support interactions: two agent calls sharing a conversation — the model requests - // lookup_order/start_return, then answers from the tool result. - private static readonly ToolStory[] SupportToolStories = - [ - new( - id => $"Hi, where is my order #{id}?", - "lookup_order", - id => $$"""{"order_id":"{{id}}"}""", - id => $$"""{"order_id":"{{id}}","status":"in_transit","carrier":"DHL","eta_days":2}""", - id => $"Order #{id} is in transit with DHL and should arrive within 2 business days. Anything else I can help with?"), - new( - id => $"Has order #{id} shipped yet? I ordered last weekend.", - "lookup_order", - id => $$"""{"order_id":"{{id}}"}""", - id => $$"""{"order_id":"{{id}}","status":"processing","warehouse":"Leipzig"}""", - id => $"Not yet — order #{id} is still being packed at our Leipzig warehouse. Dispatch is expected within 24 hours and the tracking link will arrive by email."), - new( - id => $"Order #{id} arrived with a cracked screen. I'd like to return it.", - "start_return", - id => $$"""{"order_id":"{{id}}","reason":"damaged"}""", - id => $$"""{"return_id":"RMA-{{id}}","label_url":"https://shop.example.com/labels/RMA-{{id}}","refund_estimate_days":3}""", - id => $"Sorry about that! Return RMA-{id} is open for order #{id} — the prepaid label is on its way to your inbox, and the refund lands within 3 business days of us receiving the device."), - ]; - - private static readonly (string User, string Assistant)[] CodeReviewPool = - [ - ("Review this pull request for null safety.", "Two potential NREs in PaymentProcessor.cs; suggest adding guard clauses."), - ("Audit this method for SQL injection risk.", "Parameter `userId` is concatenated; switch to a parameterised query."), - ("Is this implementation thread-safe?", "Field `_cache` is read without synchronisation; consider ConcurrentDictionary."), - ("Comment on naming consistency.", "Mix of camelCase and snake_case in DTOs; align with project style guide."), - ("Flag any obvious performance smells.", "Inner loop allocates a new List per iteration; hoist outside the loop."), - ("Check error handling.", "Bare catch suppresses cancellation; rethrow OperationCanceledException."), - ]; - - // At ToolRate 1.0 every successful analytics interaction goes through AnalyticsToolStories; - // this pool only supplies the user question for error calls (no response is rendered there). - private static readonly (string User, string Assistant)[] AnalyticsPool = - [ - ("How many active users did we have last week?", "Let me run that query."), - ("Top acquisition channels in May?", "Let me run that query."), - ("DAU/MAU ratio for last month?", "Let me run that query."), - ("Revenue split by region?", "Let me run that query."), - ("Churn rate by plan tier?", "Let me run that query."), - ("Median order value last quarter?", "Let me run that query."), - ]; - - private static readonly ToolStory[] AnalyticsToolStories = - [ - new( - _ => "How many active users did we have last week?", - "run_sql", - _ => """{"query":"SELECT COUNT(DISTINCT user_id) FROM events WHERE event_at >= now() - INTERVAL '7 days';"}""", - n => $$"""{"rows":[{"count":{{n}}}],"row_count":1,"duration_ms":388}""", - n => $"Active users in the last 7 full days: {n}.\n```sql\nSELECT COUNT(DISTINCT user_id)\nFROM events\nWHERE event_at >= now() - INTERVAL '7 days';\n```"), - new( - _ => "How many orders did we take yesterday?", - "run_sql", - _ => """{"query":"SELECT COUNT(*) FROM orders WHERE placed_at::date = (now() - INTERVAL '1 day')::date;"}""", - n => $$"""{"rows":[{"count":{{n}}}],"row_count":1,"duration_ms":214}""", - n => $"Orders taken yesterday: {n}.\n```sql\nSELECT COUNT(*) FROM orders\nWHERE placed_at::date = (now() - INTERVAL '1 day')::date;\n```"), - new( - _ => "What was our total revenue last month?", - "run_sql", - _ => """{"query":"SELECT SUM(total) AS revenue FROM orders WHERE placed_at >= date_trunc('month', now()) - INTERVAL '1 month' AND placed_at < date_trunc('month', now());"}""", - n => $$"""{"rows":[{"revenue":{{n}}.00}],"row_count":1,"duration_ms":492}""", - n => $"Total revenue last month: €{n}.\n```sql\nSELECT SUM(total) AS revenue FROM orders\nWHERE placed_at >= date_trunc('month', now()) - INTERVAL '1 month'\n AND placed_at < date_trunc('month', now());\n```"), - new( - _ => "Which columns can I segment users by?", - "get_schema", - _ => """{"table":"users"}""", - _ => """{"table":"users","columns":[{"name":"id","type":"bigint"},{"name":"plan","type":"text"},{"name":"channel","type":"text"},{"name":"region","type":"text"},{"name":"created_at","type":"timestamptz"},{"name":"last_active_at","type":"timestamptz"}]}""", - _ => "The users table segments on: plan, channel, region, plus the created_at/last_active_at timestamps for cohorting. id is the join key to events."), - ]; - - // Billing/plan questions stay in the plain pool — the triage agent has no tool to look those - // up (that gap is the seeded lookup_customer_plan theory); how-to and known-bug emails go - // through search_kb via TriageToolStories. - private static readonly (string User, string Assistant)[] TriagePool = - [ - ("Subject: API returns 429 for our nightly import since Tuesday.", "Category: Bug. Priority: P2."), - ("Subject: Please upgrade us to the annual plan.", "Category: Billing. Priority: P3."), - ("Subject: Invoice PDF shows the wrong company address.", "Category: Billing. Priority: P3."), - ("Subject: Can we get SSO with Okta?", "Category: Feature Request. Priority: P4."), - ("Subject: Everything is down again!!", "Category: UI Feedback. Priority: P3."), - ]; - - private static readonly ToolStory[] TriageToolStories = - [ - new( - _ => "Subject: Password reset email never arrives.", - "search_kb", - _ => """{"query":"password reset email not arriving"}""", - _ => """{"articles":[{"id":"KB-217","title":"Password reset emails and domain allowlists","url":"https://help.example.com/kb/217"}]}""", - _ => "Category: Account Access. Priority: P3. Suggested reply: check spam and the domain allowlist (KB-217); an admin can also trigger the reset from the members page."), - new( - _ => "Subject: Dashboard loads blank in Safari.", - "search_kb", - _ => """{"query":"dashboard blank page Safari"}""", - _ => """{"articles":[{"id":"KB-334","title":"Blank dashboard in Safari 17","url":"https://help.example.com/kb/334"}]}""", - _ => "Category: Bug. Priority: P3. Suggested reply: known Safari 17 issue — clearing site data restores the dashboard; permanent fix is rolling out (KB-334)."), - new( - _ => "Subject: How do I bulk-invite my whole team?", - "search_kb", - _ => """{"query":"bulk invite team members CSV"}""", - _ => """{"articles":[{"id":"KB-089","title":"Importing members from CSV","url":"https://help.example.com/kb/089"}]}""", - _ => "Category: How-To. Priority: P4. Suggested reply: Settings → Members → Import CSV handles up to 500 invites at once (KB-089)."), - ]; - - // HighTokens outliers carry content that visibly justifies the token count — a pasted wall of - // text in the request — so opening a flagged trace never shows a two-line chat labelled - // "high token count". - private static readonly SpikeSample SupportSpike = new( - "I've been going back and forth with your team for three weeks about order #58121 and I'm done repeating myself. " - + "Pasting the ENTIRE email thread below so you finally have the full context:\n\n" - + string.Join("\n", Enumerable.Range(1, 90).Select(i => - $"> [message {i}] Re: order #58121 — delivery rescheduled again, promised callback never happened, partial refund of €12.40 discussed but not issued.")), - "Thanks for the full history — here is where order #58121 actually stands: the delivery was rescheduled twice by the carrier, " - + "the €12.40 partial refund agreed in the middle of the thread was never issued, and the replacement lamp shade was never dispatched. " - + "I've escalated this to our fulfillment lead with the full thread attached — you'll receive the refund confirmation and the " - + "replacement tracking link by email today, plus a €10 goodwill voucher for the runaround.", - MinIn: 2600, MaxIn: 4200, MinOut: 170, MaxOut: 300); - - private static readonly SpikeSample CodeReviewSpike = new( - "Please review this whole feature branch in one go (full diff below):\n\n" - + string.Join("\n", Enumerable.Range(1, 150).Select(i => - $"+ public async Task GetOrderVariant_{i:D3}(Guid id) => await orderQuery.LoadAsync(id).ConfigureAwait(false);")), - "This diff adds 150 near-identical query methods — collapse them into one method taking the variant as a parameter. " - + "Concrete issues: (1) every method awaits with ConfigureAwait(false) but the callers rely on the request culture, so localized " - + "formatting silently breaks; (2) none of the methods pass a CancellationToken, so slow queries survive request aborts; " - + "(3) `LoadAsync` throws on missing rows while callers expect null — each variant is a latent 500. " - + "A diff this size also exceeds what one review pass can meaningfully cover; split it next time.", - MinIn: 3800, MaxIn: 5200, MinOut: 320, MaxOut: 520); - - private static readonly SpikeSample AnalyticsSpike = new( - "Here is our full warehouse schema, how many orders did we take yesterday?\n\n" - + string.Join("\n", Enumerable.Range(1, 120).Select(i => - $"CREATE TABLE shard_{i:D3}_orders (id BIGSERIAL PRIMARY KEY, user_id BIGINT NOT NULL, total NUMERIC(10,2) NOT NULL, placed_at TIMESTAMPTZ NOT NULL DEFAULT now());")), - "Your question only needs the orders tables, not the full 120-table schema — pasting it on every request is what makes " - + "these calls expensive. Here's the query I'd run:\n```sql\nSELECT COUNT(*) FROM all_orders\nWHERE placed_at::date = (now() - INTERVAL '1 day')::date;\n```\nWant me to execute it?", - MinIn: 3000, MaxIn: 5200, MinOut: 140, MaxOut: 260); - - private static readonly SpikeSample TriageSpike = new( - "Subject: FWD: FWD: RE: unresolved ticket — forwarding our complete internal thread so you can see everything:\n\n" - + string.Join("\n", Enumerable.Range(1, 80).Select(i => - $"> [reply {i}] RE: intermittent 502s on the reporting API since the last maintenance window; retried nightly import, same result.")), - "Category: Bug. Priority: P2.", - MinIn: 2400, MaxIn: 3600, MinOut: 18, MaxOut: 34); private static readonly IReadOnlyDictionary SuiteSchedule = new Dictionary { @@ -218,6 +38,7 @@ private static readonly (string User, string Assistant)[] TriagePool = }; private readonly DemoSeedContext ctx; + private readonly DemoCallPlanner planner; private readonly IAgentCall.CreateExisting agentCallExisting; private readonly ICompletion.Create completionFactory; private readonly IModelParameters.Create paramsFactory; @@ -230,6 +51,7 @@ private static readonly (string User, string Assistant)[] TriagePool = public StatisticsBackfillScenario( DemoSeedContext ctx, + DemoCallPlanner planner, IAgentCall.CreateExisting agentCallExisting, ICompletion.Create completionFactory, IModelParameters.Create paramsFactory, @@ -241,6 +63,7 @@ public StatisticsBackfillScenario( IRandom random) { this.ctx = ctx; + this.planner = planner; this.agentCallExisting = agentCallExisting; this.completionFactory = completionFactory; this.paramsFactory = paramsFactory; @@ -261,34 +84,10 @@ public async Task SeedAsync(CancellationToken cancellationToken) var profiles = new[] { - new BackfillProfile( - ctx.RequireCustomerSupportAgent(), - [new(ctx.RequireGpt54Endpoint(), 0.70), new(ctx.RequireClaudeEndpoint(), 0.30)], - SupportPool, - SupportSpike, - SupportToolStories, - ToolRate: 0.35), - new BackfillProfile( - ctx.RequireCodeReviewAgent(), - [new(ctx.RequireClaudeEndpoint(), 0.80), new(ctx.RequireGpt54MiniEndpoint(), 0.20)], - CodeReviewPool, - CodeReviewSpike, - [], - ToolRate: 0), - new BackfillProfile( - ctx.RequireDataAnalyticsAgent(), - [new(ctx.RequireGpt54Endpoint(), 0.60), new(ctx.RequireGpt54MiniEndpoint(), 0.40)], - AnalyticsPool, - AnalyticsSpike, - AnalyticsToolStories, - ToolRate: 1.0), - new BackfillProfile( - ctx.RequireEmailTriageAgent(), - [new(ctx.RequireGpt54MiniEndpoint(), 1.00)], - TriagePool, - TriageSpike, - TriageToolStories, - ToolRate: 0.35), + new BackfillProfile(ctx.RequireCustomerSupportAgent(), DemoTrafficCatalog.Support), + new BackfillProfile(ctx.RequireCodeReviewAgent(), DemoTrafficCatalog.CodeReview), + new BackfillProfile(ctx.RequireDataAnalyticsAgent(), DemoTrafficCatalog.Analytics), + new BackfillProfile(ctx.RequireEmailTriageAgent(), DemoTrafficCatalog.Triage), }; var calls = new List(); @@ -308,10 +107,11 @@ private void CollectAgentCalls( DateTimeOffset now, List calls) { + var traffic = profile.Traffic; for (int day = 0; day < WindowDays; day++) { var dayStart = windowStart.AddDays(day); - int count = random.Int(MinCallsPerDay, MaxCallsPerDay); + int count = random.Int(traffic.MinCallsPerDay, traffic.MaxCallsPerDay); for (int i = 0; i < count; i++) { var createdAt = SampleTimestamp(dayStart); @@ -320,178 +120,48 @@ private void CollectAgentCalls( continue; } - var endpoint = PickWeighted(profile.EndpointMix); - bool isError = random.Double() < ErrorRate; - bool isSpike = !isError && random.Double() < TokenSpikeRate; + var endpoint = PickWeighted(traffic.EndpointMix); + var plan = planner.Plan(traffic); + Guid? conversationId = plan.SharesConversation ? Guid.NewGuid() : null; - if (!isError && !isSpike && profile.ToolStories.Length > 0 && random.Double() < profile.ToolRate) + foreach (var planned in plan.Calls) { - CollectToolConversation(profile, endpoint, createdAt, calls); - continue; - } - - var flags = OutlierFlags.None; - string userText; - string assistantText; - ulong inTok; - ulong outTok; - ulong cachedIn; - - if (isSpike) - { - // A conversation that ballooned: far above the ~325-token mean of the window, - // with a pasted wall of text in the request to match the numbers. The one-off - // paste also misses the prompt cache. - var spike = profile.Spike; - userText = spike.User; - assistantText = spike.Assistant; - inTok = (ulong)random.Int(spike.MinIn, spike.MaxIn + 1); - outTok = (ulong)random.Int(spike.MinOut, spike.MaxOut + 1); - cachedIn = 0; - flags |= OutlierFlags.HighTokens; - } - else - { - (userText, assistantText) = random.Any(profile.Pool); - inTok = (ulong)random.Int(200, 451); - outTok = (ulong)random.Int(40, 221); - - // Most calls hit the prompt cache for part of the input; ~30% miss entirely, - // giving the cache-hit KPI and distribution a realistic spread. - cachedIn = random.Double() < UncachedShareRate - ? 0UL - : (ulong)(inTok * random.Double(0.3, 0.8)); - } - - int latencyMs; - if (random.Double() < LatencySpikeRate) - { - latencyMs = random.Int(4200, 9001); - flags |= OutlierFlags.HighLatency; + calls.Add(BuildBackdatedCall( + profile.Agent, endpoint, planned, + createdAt.AddSeconds(planned.OffsetSeconds), conversationId)); } - else if (flags.HasFlag(OutlierFlags.HighTokens)) - { - // A big context takes longer, but stays inside the unflagged latency tail. - latencyMs = random.Int(1600, 3401); - } - else - { - latencyMs = random.Double() < LatencyTailRate - ? random.Int(1500, 3501) - : random.Int(400, 901); - } - - var systemMsg = profile.Agent.CreateSystemMessage(); - var userMsg = new UserMessage([Content.FromText(userText)]); - var request = new Conversation([systemMsg, userMsg]); - - ICompletion? response = isError - ? null - : completionFactory( - new AssistantMessage([Content.FromText(assistantText)], []), - new TokenUsage(inTok, outTok, cachedIn), - TimeSpan.FromMilliseconds(latencyMs)); - - (HttpStatusCode status, string? errorMessage, string? finishReason) = isError - ? BuildError() - : (HttpStatusCode.OK, (string?)null, "stop"); - - calls.Add(BuildBackdatedCall( - profile.Agent, endpoint, request, response, - status, errorMessage, finishReason, - createdAt, conversationId: null, - outlierFlags: isError ? OutlierFlags.None : flags)); } } } - private void CollectToolConversation( - BackfillProfile profile, - IModelEndpoint endpoint, - DateTimeOffset createdAt, - List calls) - { - var story = random.Any(profile.ToolStories); - string orderId = random.Int(10000, 100000).ToString(); - var conversationId = Guid.NewGuid(); - var system = profile.Agent.CreateSystemMessage(); - var user = new UserMessage([Content.FromText(story.User(orderId))]); - - var toolRequest = new ToolRequest( - id: $"call_{story.ToolName}_{orderId}", - name: story.ToolName, - arguments: story.Arguments(orderId)); - var assistantToolMsg = new AssistantMessage([], [toolRequest]); - - ulong toolTurnIn = (ulong)random.Int(230, 331); - calls.Add(BuildBackdatedCall( - profile.Agent, endpoint, - request: new Conversation([system, user]), - response: completionFactory( - assistantToolMsg, - new TokenUsage(toolTurnIn, (ulong)random.Int(24, 46), CachedShare(toolTurnIn)), - TimeSpan.FromMilliseconds(random.Int(380, 721))), - httpStatus: HttpStatusCode.OK, - errorMessage: null, - finishReason: "tool_calls", - createdAt: createdAt, - conversationId: conversationId, - outlierFlags: OutlierFlags.None)); - - var toolMsg = new ToolMessage(new ToolResponse( - toolRequest, [Content.FromText(story.ToolResult(orderId))])); - var finalAssistant = new AssistantMessage([Content.FromText(story.Final(orderId))], []); - - ulong finalTurnIn = toolTurnIn + (ulong)random.Int(70, 141); - calls.Add(BuildBackdatedCall( - profile.Agent, endpoint, - request: new Conversation([system, user, assistantToolMsg, toolMsg]), - response: completionFactory( - finalAssistant, - new TokenUsage(finalTurnIn, (ulong)random.Int(45, 91), CachedShare(finalTurnIn)), - TimeSpan.FromMilliseconds(random.Int(430, 821))), - httpStatus: HttpStatusCode.OK, - errorMessage: null, - finishReason: "stop", - createdAt: createdAt.AddSeconds(random.Int(2, 8)), - conversationId: conversationId, - outlierFlags: OutlierFlags.None)); - } - - private ulong CachedShare(ulong inTok) - => random.Double() < UncachedShareRate - ? 0UL - : (ulong)(inTok * random.Double(0.3, 0.8)); - private IAgentCall BuildBackdatedCall( IAgent agent, IModelEndpoint endpoint, - Conversation request, - ICompletion? response, - HttpStatusCode httpStatus, - string? errorMessage, - string? finishReason, + PlannedDemoCall planned, DateTimeOffset createdAt, - Guid? conversationId, - OutlierFlags outlierFlags) - => agentCallExisting( + Guid? conversationId) + { + var request = new Conversation([agent.CreateSystemMessage(), .. planned.RequestTail]); + ICompletion? response = planned.ResponseMessage is null + ? null + : completionFactory( + planned.ResponseMessage, + planned.Usage, + TimeSpan.FromMilliseconds(planned.LatencyMs)); + + return agentCallExisting( agent: agent, version: agent.CurrentVersion, endpoint: endpoint, request: request, response: response, - httpStatus: httpStatus, - finishReason: finishReason, - errorMessage: errorMessage, + httpStatus: planned.HttpStatus, + finishReason: planned.FinishReason, + errorMessage: planned.ErrorMessage, modelParameters: paramsFactory(temperature: 0.3), existing: new BackdatedData(Guid.NewGuid(), createdAt, createdAt), conversationId: conversationId, - outlierFlags: outlierFlags); - - private (HttpStatusCode Status, string? ErrorMessage, string? FinishReason) BuildError() - { - var variant = random.Any(ErrorVariants); - return (variant.Status, variant.Message, null); + outlierFlags: planned.OutlierFlags); } private DateTimeOffset SampleTimestamp(DateTimeOffset dayStart) @@ -504,35 +174,45 @@ private DateTimeOffset SampleTimestamp(DateTimeOffset dayStart) private int SampleDiurnalHour() { - int total = DiurnalWeights.Sum(); + int[] weights = DemoTrafficCatalog.DiurnalWeights; + int total = weights.Sum(); int pick = random.Int(0, total); int acc = 0; - for (int h = 0; h < DiurnalWeights.Length; h++) + for (int h = 0; h < weights.Length; h++) { - acc += DiurnalWeights[h]; + acc += weights[h]; if (pick < acc) { return h; } } - return DiurnalWeights.Length - 1; + return weights.Length - 1; } - private IModelEndpoint PickWeighted(IReadOnlyList mix) + private IModelEndpoint PickWeighted(IReadOnlyList mix) { double pick = random.Double(); double acc = 0; - foreach (var w in mix) + foreach (var share in mix) { - acc += w.Weight; + acc += share.Weight; if (pick <= acc) { - return w.Endpoint; + return Resolve(share.Endpoint); } } - return mix[^1].Endpoint; + return Resolve(mix[^1].Endpoint); } + private IModelEndpoint Resolve(DemoTrafficCatalog.DemoEndpointKey key) + => key switch + { + DemoTrafficCatalog.DemoEndpointKey.Gpt54 => ctx.RequireGpt54Endpoint(), + DemoTrafficCatalog.DemoEndpointKey.Gpt54Mini => ctx.RequireGpt54MiniEndpoint(), + DemoTrafficCatalog.DemoEndpointKey.ClaudeSonnet => ctx.RequireClaudeEndpoint(), + _ => throw new ArgumentOutOfRangeException(nameof(key), key, "Unknown demo endpoint key"), + }; + private async Task StaggerTestRunsAsync(DateTimeOffset now, CancellationToken cancellationToken) { var groupOrder = new List(); @@ -613,36 +293,7 @@ private async Task BackdateGroupAsync( private sealed record BackdatedData(Guid Id, DateTimeOffset CreatedAt, DateTimeOffset UpdatedAt) : IDomainEntityData; - private sealed record EndpointWeight(IModelEndpoint Endpoint, double Weight); - private sealed record BackfillProfile( IAgent Agent, - IReadOnlyList EndpointMix, - (string User, string Assistant)[] Pool, - SpikeSample Spike, - ToolStory[] ToolStories, - double ToolRate); - - /// - /// Content for a HighTokens outlier: a request with a pasted wall of text and token ranges - /// that match it, so the flag is visibly justified when the trace is opened. - /// - private sealed record SpikeSample( - string User, - string Assistant, - int MinIn, - int MaxIn, - int MinOut, - int MaxOut); - - /// - /// A two-call tool round-trip (request tool → answer from tool result), templated on a random - /// order id. - /// - private sealed record ToolStory( - Func User, - string ToolName, - Func Arguments, - Func ToolResult, - Func Final); + DemoTrafficCatalog.AgentTraffic Traffic); } diff --git a/Proxytrace.Application/Module.cs b/Proxytrace.Application/Module.cs index 42149b1f9..38c3ab2da 100644 --- a/Proxytrace.Application/Module.cs +++ b/Proxytrace.Application/Module.cs @@ -595,6 +595,10 @@ protected override void Load(ContainerBuilder builder) .AsSelf() .SingleInstance(); + builder.RegisterType() + .AsSelf() + .SingleInstance(); + var scenarioTypes = typeof(Module).Assembly.GetTypes() .Where(t => t is { IsClass: true, IsAbstract: false } && typeof(IDemoScenario).IsAssignableFrom(t)); @@ -688,5 +692,33 @@ protected override void Load(ContainerBuilder builder) }); }); } + + builder.RegisterType() + .AsSelf() + .SingleInstance() + .IfNotRegistered(typeof(KioskLiveTrafficService)); + + // Registered after the demo seeder so its StartAsync (and thus the traffic loop) begins + // only once seeding has completed and the DemoSeedContext is populated. + const string kioskLiveTrafficHostedServiceKey = "Proxytrace.Application.KioskLiveTrafficService.Registered"; + if (!builder.Properties.ContainsKey(kioskLiveTrafficHostedServiceKey)) + { + builder.Properties[kioskLiveTrafficHostedServiceKey] = true; + builder.RegisterServiceCollection(services => + { + services.AddSingleton(sp => + { + // The live feed runs in every kiosk (read-only or with a live endpoint) — it + // is what keeps the dashboard moving; outside kiosk mode it must never run. + var kiosk = sp.GetRequiredService(); + if (!kiosk.Enabled) + { + return new NullHostedService(); + } + + return sp.GetRequiredService(); + }); + }); + } } } diff --git a/docs/commands.md b/docs/commands.md index 4939bef63..96c5cb701 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -120,6 +120,15 @@ Ports: The frontend is fully browsable; the OpenAI proxy route is not mounted (`/openai/v1/*` returns 404) and the sample client idles. +**Demo data.** Seeding paints a business-scale deployment: a 14-day traffic backfill at per-agent +daily volumes (~1,300–1,700 interactions/day across the four demo agents, with production-sized +token counts, so cost/throughput cards read like a real installation), plus a continuous +**simulated live traffic feed** (`KioskLiveTrafficService`) that keeps fabricating agent calls +after boot — the dashboard's pulse band, live telemetry and recent-traces feed stay in motion +without a real LLM endpoint. Content, rates and volumes live in +`Proxytrace.Application/Demo/Internal/DemoTrafficCatalog.cs`, shared by the backfill and the live +feed so both describe the same business. + **Live demo mode:** copy `kiosk.env.example` to `.env` and fill in your LLM credentials: ```bash diff --git a/manual/admin/deployment.md b/manual/admin/deployment.md index 93b1c3180..7b6c6ff04 100644 --- a/manual/admin/deployment.md +++ b/manual/admin/deployment.md @@ -56,6 +56,11 @@ docker compose -f docker-compose.kiosk.yml up --build # API :5200, UI :5201, s Locally, `./dev.sh` runs the kiosk shape by default; `SPLIT=1 ./dev.sh` runs the split shape with a throwaway Redis. +Kiosk mode self-seeds a full demo dataset on boot: four showcase agents with two weeks of +business-scale traffic history (hundreds to thousands of calls per agent per day, realistic +token volumes and costs), plus a continuously simulated live traffic feed — new traces keep +arriving on the dashboard and in the traces list even without a real LLM endpoint configured. + ### Live showcase stack The compose file ships a bundled **sample chat client** (`:5202`) and supports a **live LLM