diff --git a/Cargo.lock b/Cargo.lock index c1a7e12f67..db4925022f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7174,6 +7174,7 @@ dependencies = [ "moltis-common", "moltis-config", "moltis-metrics", + "moltis-observability", "moltis-sessions", "moltis-skills", "reqwest 0.13.2", @@ -7299,16 +7300,19 @@ dependencies = [ name = "moltis-channels" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.4.2", "moltis-common", "moltis-config", "moltis-metrics", + "moltis-observability", "rand 0.10.1", "serde", "serde_json", "thiserror 2.0.18", + "time", "tokio", "tracing", "url", @@ -7332,6 +7336,7 @@ dependencies = [ "moltis-config", "moltis-memory", "moltis-metrics", + "moltis-observability", "moltis-plugins", "moltis-projects", "moltis-providers", @@ -7596,6 +7601,7 @@ dependencies = [ "moltis-network-filter", "moltis-nostr", "moltis-oauth", + "moltis-observability", "moltis-onboarding", "moltis-openclaw-import", "moltis-plugins", @@ -8032,6 +8038,30 @@ dependencies = [ "uuid", ] +[[package]] +name = "moltis-observability" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "futures", + "moltis-common", + "moltis-config", + "moltis-metrics", + "rand 0.10.1", + "reqwest 0.13.2", + "secrecy 0.8.0", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "tokio", + "tracing", + "uuid", + "wiremock", +] + [[package]] name = "moltis-onboarding" version = "0.1.0" @@ -11723,6 +11753,12 @@ dependencies = [ "sha1 0.10.6", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -13834,6 +13870,7 @@ dependencies = [ "getrandom 0.4.2", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 9516b49c79..16929ddeb4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ default-members = [ "crates/node-host", "crates/nostr", "crates/oauth", + "crates/observability", "crates/onboarding", "crates/openclaw-import", "crates/plugins", @@ -106,6 +107,7 @@ members = [ "crates/node-host", "crates/nostr", "crates/oauth", + "crates/observability", "crates/onboarding", "crates/openclaw-import", "crates/plugins", @@ -212,7 +214,7 @@ sqlx = { features = ["migrate", "runtime-tokio", "sqlite"], version = "0.8" } # HTTP client reqwest = { features = ["form", "json", "multipart", "query", "socks", "stream"], version = "0.13" } # UUID -uuid = { features = ["v4"], version = "1" } +uuid = { features = ["v4", "v5"], version = "1" } # Async async-stream = "0.3" async-trait = "0.1" @@ -402,6 +404,7 @@ moltis-network-filter = { default-features = false, path = "crates/network-fil moltis-node-host = { path = "crates/node-host" } moltis-nostr = { path = "crates/nostr" } moltis-oauth = { path = "crates/oauth" } +moltis-observability = { default-features = false, path = "crates/observability" } moltis-onboarding = { path = "crates/onboarding" } moltis-openclaw-import = { path = "crates/openclaw-import" } moltis-plugins = { path = "crates/plugins" } diff --git a/README.md b/README.md index 605105c8a7..8902b31baf 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Verify releases with `gh attestation verify -R moltis-org/moltis` or - **Safer Agent Editing** — Automatic checkpoints before built-in skill and memory mutations, restore tooling, session branching - **Extensibility** — MCP servers (stdio + HTTP/SSE), skill system, 15 lifecycle hook events with circuit breaker, destructive command guard - **Security** — Encryption-at-rest vault (XChaCha20-Poly1305 + Argon2id), password + passkey + API key auth, sandbox isolation, SSRF/CSWSH protection -- **Operations** — Cron scheduling, OpenTelemetry tracing, Prometheus metrics, cloud deploy (Fly.io, DigitalOcean), Tailscale integration, managed SSH deploy keys, host-pinned remote targets, live tool inventory in Settings, and CLI/web remote-exec doctor flows +- **Operations** — Cron scheduling, agent instrumentation (Langfuse for LLM traces, OTLP for Grafana/Datadog), Prometheus metrics, cloud deploy (Fly.io, DigitalOcean), Tailscale integration, managed SSH deploy keys, host-pinned remote targets, live tool inventory in Settings, and CLI/web remote-exec doctor flows ## How It Works diff --git a/crates/agents/Cargo.toml b/crates/agents/Cargo.toml index 3108c93c3f..6a2bb0e706 100644 --- a/crates/agents/Cargo.toml +++ b/crates/agents/Cargo.toml @@ -13,7 +13,8 @@ async-stream = { workspace = true } async-trait = { workspace = true } futures = { workspace = true } include_dir = { workspace = true } -moltis-common = { workspace = true } +moltis-common = { workspace = true } +moltis-observability = { workspace = true } moltis-config = { workspace = true } moltis-sessions = { workspace = true } moltis-skills = { workspace = true } diff --git a/crates/agents/src/model/mod.rs b/crates/agents/src/model/mod.rs index 049c0920db..81fb528a95 100644 --- a/crates/agents/src/model/mod.rs +++ b/crates/agents/src/model/mod.rs @@ -5,8 +5,8 @@ pub use moltis_config::schema::{AgentToolControls, ReasoningEffort, ToolChoice}; mod types; pub use types::{ - CompletionResponse, MAX_CAPTURED_PROVIDER_RAW_EVENTS, ModelMetadata, TOOL_CALL_METADATA_KEYS, - ToolCall, ToolCallArgumentDiagnostic, ToolCallArgumentSource, Usage, + CompletionResponse, InputTokenAccounting, MAX_CAPTURED_PROVIDER_RAW_EVENTS, ModelMetadata, + TOOL_CALL_METADATA_KEYS, ToolCall, ToolCallArgumentDiagnostic, ToolCallArgumentSource, Usage, push_capped_provider_raw_event, }; @@ -20,7 +20,9 @@ pub use convert::{ }; mod stream; -pub use stream::{LlmProvider, StreamEvent}; +pub use stream::{ + LlmProvider, ProviderAttemptEvent, ProviderIdentity, StreamEvent, TrackedStreamEvent, +}; #[cfg(test)] fn document_absolute_path_from_media_ref(media_ref: &str) -> String { @@ -242,6 +244,24 @@ mod tests { assert_eq!(total.cache_write_tokens, 44); } + #[test] + fn usage_normalizes_inclusive_input_into_exclusive_buckets() { + let usage = Usage::from_input_tokens(InputTokenAccounting::Inclusive, 1_000, 50, 800, 20); + + assert_eq!(usage.input_tokens, 180); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); + } + + #[test] + fn usage_preserves_anthropic_exclusive_input() { + let usage = Usage::from_input_tokens(InputTokenAccounting::Exclusive, 180, 50, 800, 20); + + assert_eq!(usage.input_tokens, 180); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); + } + // ── to_openai_value ────────────────────────────────────────────── #[test] diff --git a/crates/agents/src/model/stream.rs b/crates/agents/src/model/stream.rs index 5f9846d0f4..9d237cde0f 100644 --- a/crates/agents/src/model/stream.rs +++ b/crates/agents/src/model/stream.rs @@ -8,6 +8,40 @@ use super::{ types::{CompletionResponse, ModelMetadata, Usage}, }; +/// Concrete provider and model selected for one completion attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderIdentity { + pub provider: String, + pub model: String, +} + +impl ProviderIdentity { + #[must_use] + pub fn new(provider: impl Into, model: impl Into) -> Self { + Self { + provider: provider.into(), + model: model.into(), + } + } +} + +/// Lifecycle events for attempts made by a provider wrapper. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProviderAttemptEvent { + Started(ProviderIdentity), + Failed { + identity: ProviderIdentity, + error: String, + }, +} + +/// Event returned by the runner-facing tracked streaming API. +#[derive(Debug, Clone)] +pub enum TrackedStreamEvent { + Attempt(ProviderAttemptEvent), + Event(StreamEvent), +} + // ── Stream events ─────────────────────────────────────────────────────────── /// Events emitted during streaming LLM completion. @@ -72,6 +106,26 @@ pub trait LlmProvider: Send + Sync { self.complete(messages, tools).await } + /// Complete while reporting each concrete provider attempt. + async fn complete_with_options_tracked( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + on_attempt: &mut (dyn FnMut(ProviderAttemptEvent) + Send), + ) -> anyhow::Result { + let identity = ProviderIdentity::new(self.name(), self.id()); + on_attempt(ProviderAttemptEvent::Started(identity.clone())); + let result = self.complete_with_options(messages, tools, options).await; + if let Err(error) = &result { + on_attempt(ProviderAttemptEvent::Failed { + identity, + error: error.to_string(), + }); + } + result + } + /// Whether this provider supports tool/function calling. /// Defaults to false; providers that handle the `tools` parameter /// in `complete()` should override this to return true. @@ -134,6 +188,24 @@ pub trait LlmProvider: Send + Sync { self.stream_with_tools(messages, tools) } + /// Stream while reporting the concrete provider/model selected for the attempt. + fn stream_with_tools_and_options_tracked( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + let attempt = TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started( + ProviderIdentity::new(self.name(), self.id()), + )); + Box::pin( + tokio_stream::once(attempt).chain( + self.stream_with_tools_and_options(messages, tools, options) + .map(TrackedStreamEvent::Event), + ), + ) + } + /// Configured reasoning effort for this provider instance, if any. /// /// Providers that support extended thinking (Anthropic, OpenAI o-series) diff --git a/crates/agents/src/model/types.rs b/crates/agents/src/model/types.rs index 2b4a5eb4cb..95267c3977 100644 --- a/crates/agents/src/model/types.rs +++ b/crates/agents/src/model/types.rs @@ -71,13 +71,48 @@ pub const TOOL_CALL_METADATA_KEYS: &[&str] = &["thought_signature"]; #[derive(Debug, Clone, Default)] pub struct Usage { + /// Fresh input tokens, excluding cache reads and cache writes. pub input_tokens: u32, pub output_tokens: u32, pub cache_read_tokens: u32, pub cache_write_tokens: u32, } +/// Provider-reported input-token accounting semantics. +/// +/// OpenAI-style APIs report cached tokens as part of their input total, while +/// Anthropic reports fresh input, cache reads, and cache writes separately. +/// Provider adapters must identify which contract they received so `Usage` +/// always exposes mutually exclusive buckets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InputTokenAccounting { + Inclusive, + Exclusive, +} + impl Usage { + #[must_use] + pub const fn from_input_tokens( + accounting: InputTokenAccounting, + reported_input_tokens: u32, + output_tokens: u32, + cache_read_tokens: u32, + cache_write_tokens: u32, + ) -> Self { + let input_tokens = match accounting { + InputTokenAccounting::Inclusive => reported_input_tokens + .saturating_sub(cache_read_tokens) + .saturating_sub(cache_write_tokens), + InputTokenAccounting::Exclusive => reported_input_tokens, + }; + Self { + input_tokens, + output_tokens, + cache_read_tokens, + cache_write_tokens, + } + } + #[must_use] pub fn saturating_add(&self, other: &Self) -> Self { Self { diff --git a/crates/agents/src/provider_chain.rs b/crates/agents/src/provider_chain.rs index 04285a54ca..c35b59d738 100644 --- a/crates/agents/src/provider_chain.rs +++ b/crates/agents/src/provider_chain.rs @@ -14,12 +14,19 @@ use std::{ time::{Duration, Instant}, }; -use {async_trait::async_trait, tokio_stream::Stream, tracing::warn}; +use { + async_trait::async_trait, + tokio_stream::{Stream, StreamExt}, + tracing::warn, +}; #[cfg(feature = "metrics")] use moltis_metrics::{counter, histogram, labels, llm as llm_metrics}; -use crate::model::{ChatMessage, CompletionResponse, LlmProvider, StreamEvent}; +use crate::model::{ + AgentToolControls, ChatMessage, CompletionResponse, LlmProvider, ProviderAttemptEvent, + ProviderIdentity, StreamEvent, TrackedStreamEvent, +}; /// How a provider error should be handled. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -212,6 +219,99 @@ impl ProviderChain { fn primary(&self) -> &ChainEntry { &self.chain[0] } + + fn failover_stream( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + Box::pin(async_stream::stream! { + let mut errors = Vec::new(); + let force_primary = self.chain.iter().all(|entry| entry.state.is_tripped()); + + for (index, entry) in self.chain.iter().enumerate() { + if entry.state.is_tripped() && !(force_primary && index == 0) { + continue; + } + + let identity = ProviderIdentity::new(entry.provider.name(), entry.provider.id()); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started( + identity.clone(), + )); + + let mut stream = entry.provider.stream_with_tools_and_options( + messages.clone(), + tools.clone(), + options.clone(), + ); + let mut emitted_output = false; + let mut retry_next = false; + + while let Some(event) = stream.next().await { + match event { + StreamEvent::Error(error) => { + let anyhow_error = anyhow::anyhow!(error.clone()); + let kind = classify_error(&anyhow_error); + entry.state.record_failure(); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { + identity: identity.clone(), + error: error.clone(), + }); + + if kind.should_failover() && !emitted_output { + warn!( + provider = entry.provider.id(), + error = %error, + kind = ?kind, + "provider stream failed, trying next in chain" + ); + errors.push(format!("{}: {error}", entry.provider.id())); + retry_next = true; + break; + } + + yield TrackedStreamEvent::Event(StreamEvent::Error(error)); + return; + }, + StreamEvent::Done(usage) => { + entry.state.record_success(); + yield TrackedStreamEvent::Event(StreamEvent::Done(usage)); + return; + }, + StreamEvent::ProviderRaw(raw) => { + yield TrackedStreamEvent::Event(StreamEvent::ProviderRaw(raw)); + }, + event => { + emitted_output = true; + yield TrackedStreamEvent::Event(event); + }, + } + } + + if retry_next { + continue; + } + + let error = "provider stream ended without a terminal event".to_string(); + entry.state.record_failure(); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { + identity: identity.clone(), + error: error.clone(), + }); + if emitted_output { + yield TrackedStreamEvent::Event(StreamEvent::Error(error)); + return; + } + errors.push(format!("{}: {error}", entry.provider.id())); + } + + yield TrackedStreamEvent::Event(StreamEvent::Error(format!( + "all providers in failover chain failed: {}", + errors.join("; ") + ))); + }) + } } #[async_trait] @@ -236,20 +336,47 @@ impl LlmProvider for ProviderChain { &self, messages: &[ChatMessage], tools: &[serde_json::Value], + ) -> anyhow::Result { + self.complete_with_options(messages, tools, &AgentToolControls::default()) + .await + } + + async fn complete_with_options( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + ) -> anyhow::Result { + let mut ignore_attempt = |_: ProviderAttemptEvent| {}; + self.complete_with_options_tracked(messages, tools, options, &mut ignore_attempt) + .await + } + + async fn complete_with_options_tracked( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + on_attempt: &mut (dyn FnMut(ProviderAttemptEvent) + Send), ) -> anyhow::Result { let mut errors = Vec::new(); + let force_primary = self.chain.iter().all(|entry| entry.state.is_tripped()); #[cfg(feature = "metrics")] let start = Instant::now(); - for entry in &self.chain { - if entry.state.is_tripped() { + for (index, entry) in self.chain.iter().enumerate() { + if entry.state.is_tripped() && !(force_primary && index == 0) { continue; } let provider_name = entry.provider.name().to_string(); let model_id = entry.provider.id().to_string(); - match entry.provider.complete(messages, tools).await { + match entry + .provider + .complete_with_options_tracked(messages, tools, options, on_attempt) + .await + { Ok(resp) => { entry.state.record_success(); @@ -353,16 +480,37 @@ impl LlmProvider for ProviderChain { messages: Vec, tools: Vec, ) -> Pin + Send + '_>> { - // For streaming, we try the first non-tripped provider. - // If the stream yields an Error event, we can't transparently retry mid-stream, - // so we pick the best available provider upfront. - for entry in &self.chain { - if !entry.state.is_tripped() { - return entry.provider.stream_with_tools(messages, tools); - } - } - // All tripped — try primary anyway (it may have cooled down by now). - self.primary().provider.stream_with_tools(messages, tools) + Box::pin( + self.failover_stream(messages, tools, AgentToolControls::default()) + .filter_map(|event| match event { + TrackedStreamEvent::Attempt(_) => None, + TrackedStreamEvent::Event(event) => Some(event), + }), + ) + } + + fn stream_with_tools_and_options( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + Box::pin( + self.failover_stream(messages, tools, options) + .filter_map(|event| match event { + TrackedStreamEvent::Attempt(_) => None, + TrackedStreamEvent::Event(event) => Some(event), + }), + ) + } + + fn stream_with_tools_and_options_tracked( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + self.failover_stream(messages, tools, options) } } @@ -453,6 +601,34 @@ mod tests { } } + struct EmptyStreamProvider; + + #[async_trait] + impl LlmProvider for EmptyStreamProvider { + fn name(&self) -> &str { + "empty" + } + + fn id(&self) -> &str { + "empty" + } + + async fn complete( + &self, + _messages: &[ChatMessage], + _tools: &[serde_json::Value], + ) -> anyhow::Result { + anyhow::bail!("not used") + } + + fn stream( + &self, + _messages: Vec, + ) -> Pin + Send + '_>> { + Box::pin(tokio_stream::empty()) + } + } + #[tokio::test] async fn primary_succeeds_no_failover() { let chain = ProviderChain::new(vec![ @@ -479,6 +655,38 @@ mod tests { assert_eq!(resp.text.as_deref(), Some("ok")); } + #[tokio::test] + async fn non_streaming_attempts_report_fallback_identity() { + let chain = ProviderChain::new(vec![ + Arc::new(FailingProvider { + id: "primary", + error_msg: "429 rate limit exceeded", + }), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + let mut attempts = Vec::new(); + + let response = chain + .complete_with_options_tracked( + &[], + &[], + &AgentToolControls::default(), + &mut |attempt| attempts.push(attempt), + ) + .await + .unwrap(); + + assert_eq!(response.text.as_deref(), Some("ok")); + assert_eq!(attempts, vec![ + ProviderAttemptEvent::Started(ProviderIdentity::new("failing", "primary")), + ProviderAttemptEvent::Failed { + identity: ProviderIdentity::new("failing", "primary"), + error: "429 rate limit exceeded".into(), + }, + ProviderAttemptEvent::Started(ProviderIdentity::new("success", "fallback")), + ]); + } + #[tokio::test] async fn failover_on_server_error() { let chain = ProviderChain::new(vec![ @@ -584,15 +792,140 @@ mod tests { Arc::new(SuccessProvider { id: "backup" }), ]); - // Trip the first provider. + // Trip the first provider through stream failures. for _ in 0..3 { - let _ = chain.complete(&[], &[]).await; + chain.stream(vec![]).collect::>().await; } + assert!(chain.chain[0].state.is_tripped()); // Stream should use backup. - let mut stream = chain.stream(vec![]); + let mut stream = chain.stream_with_tools_and_options_tracked( + vec![], + vec![], + AgentToolControls::default(), + ); let event = stream.next().await.unwrap(); - assert!(matches!(event, StreamEvent::Done(_))); + assert!(matches!( + event, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == ProviderIdentity::new("success", "backup") + )); + assert!(matches!( + stream.next().await, + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn stream_error_fails_over_and_records_attempts() { + let chain = ProviderChain::new(vec![ + Arc::new(FailingProvider { + id: "primary", + error_msg: "503 service unavailable", + }), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(matches!( + &events[0], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == &ProviderIdentity::new("failing", "primary") + )); + assert!(matches!( + &events[1], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { identity, error }) + if identity == &ProviderIdentity::new("failing", "primary") + && error == "503 service unavailable" + )); + assert!(matches!( + &events[2], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == &ProviderIdentity::new("success", "fallback") + )); + assert!(matches!( + events[3], + TrackedStreamEvent::Event(StreamEvent::Done(_)) + )); + assert_eq!( + chain.chain[0] + .state + .consecutive_failures + .load(Ordering::SeqCst), + 1 + ); + assert_eq!( + chain.chain[1] + .state + .consecutive_failures + .load(Ordering::SeqCst), + 0 + ); + } + + #[tokio::test] + async fn empty_stream_fails_over_instead_of_reporting_success() { + let chain = ProviderChain::new(vec![ + Arc::new(EmptyStreamProvider), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(events.iter().any(|event| matches!( + event, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { identity, error }) + if identity == &ProviderIdentity::new("empty", "empty") + && error.contains("without a terminal event") + ))); + assert!(matches!( + events.last(), + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn all_tripped_streaming_chain_still_probes_primary() { + let chain = ProviderChain::single(Arc::new(SuccessProvider { id: "primary" })); + for _ in 0..3 { + chain.chain[0].state.record_failure(); + } + assert!(chain.chain[0].state.is_tripped()); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(matches!( + events.first(), + Some(TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity))) + if identity == &ProviderIdentity::new("success", "primary") + )); + assert!(matches!( + events.last(), + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn all_tripped_non_streaming_chain_still_probes_primary() { + let chain = ProviderChain::single(Arc::new(SuccessProvider { id: "primary" })); + for _ in 0..3 { + chain.chain[0].state.record_failure(); + } + assert!(chain.chain[0].state.is_tripped()); + + let response = chain.complete(&[], &[]).await.unwrap(); + + assert_eq!(response.text.as_deref(), Some("ok")); } #[test] diff --git a/crates/agents/src/runner/instrumentation.rs b/crates/agents/src/runner/instrumentation.rs new file mode 100644 index 0000000000..aaf1ab0086 --- /dev/null +++ b/crates/agents/src/runner/instrumentation.rs @@ -0,0 +1,641 @@ +//! Bridge between the agent loop and `moltis-observability`. +//! +//! Keeps trace-scope derivation out of the runner bodies: the loops call +//! [`begin_turn`] once and then open steps. Every function here degrades to a +//! no-op when instrumentation is disabled, so the runner needs no `cfg` gates. + +use { + moltis_common::hooks::ChannelBinding, + moltis_observability::{ + ObservationKind, RecorderSettings, TokenUsage, TraceScope, TurnRecorder, + }, +}; + +use crate::{ + model::{ + AgentToolControls, ChatMessage, ContentPart, LlmProvider, ProviderIdentity, Usage, + UserContent, + }, + runner::{AgentRunError, AgentRunResult, tool_result::persisted_tool_result_failure}, +}; + +/// Derive the trace scope from the session and channel context. +/// +/// The session key becomes the backend's session id, so a Langfuse session +/// view lines up one-to-one with a Moltis conversation. Channel provenance +/// becomes tags, and the sender becomes the user id — namespaced by channel so +/// `telegram:42` and `slack:42` are never conflated into one person. +#[must_use] +pub fn trace_scope( + session_key: &str, + channel: Option<&ChannelBinding>, + environment: String, + release: String, +) -> TraceScope { + let mut tags = Vec::new(); + let mut user_id = None; + + if let Some(binding) = channel { + if let Some(channel_type) = &binding.channel_type { + tags.push(format!("channel:{channel_type}")); + } + if let Some(surface) = &binding.surface { + tags.push(format!("surface:{surface}")); + } + if let Some(chat_type) = &binding.chat_type { + tags.push(format!("chat:{chat_type}")); + } + if let Some(sender) = &binding.sender_id { + user_id = Some(binding.channel_type.as_ref().map_or_else( + || sender.clone(), + |channel_type| format!("{channel_type}:{sender}"), + )); + } + } + + // The agent id is the leading segment of `agent::...`. + if let Some(agent_id) = session_key + .strip_prefix("agent:") + .and_then(|rest| rest.split(':').next()) + && !agent_id.is_empty() + { + tags.push(format!("agent:{agent_id}")); + } + + TraceScope { + session_id: (!session_key.is_empty()).then(|| session_key.to_string()), + user_id, + tags, + environment: Some(environment), + release: Some(release), + version: None, + } +} + +/// Recorder settings derived from `[instrumentation]`. +/// +/// Capture switches come from the Langfuse sub-table because it is the only +/// backend that receives payload bodies at all; the APM profiles gate content +/// independently through their own `content` mode. +#[must_use] +pub fn recorder_settings(config: &moltis_config::InstrumentationConfig) -> RecorderSettings { + RecorderSettings { + redaction: moltis_observability::RedactionPolicy::from_needles(&config.redact), + capture_input: config.langfuse.capture_input, + capture_output: config.langfuse.capture_output, + capture_tool_io: config.langfuse.capture_tool_io, + sample_rate: config.sample_rate.clamp(0.0, 1.0), + } +} + +/// Release identifier reported to backends, defaulting to the running version. +#[must_use] +pub fn release(config: &moltis_config::InstrumentationConfig) -> String { + config + .release + .clone() + .unwrap_or_else(|| moltis_config::VERSION.to_string()) +} + +/// Begin recording an agent run, if instrumentation is enabled. +/// +/// Returns `None` when no sink is installed or the turn was not sampled, in +/// which case every downstream call is skipped by the caller's `Option` checks. +#[must_use] +pub fn begin_turn( + session_key: &str, + trace_correlation_key: Option<&str>, + channel: Option<&ChannelBinding>, + provider: &str, + model: &str, + user_content: &UserContent, + settings: RecorderSettings, + environment: String, + release: String, +) -> Option { + let scope = trace_scope(session_key, channel, environment, release); + let recorder = TurnRecorder::begin("agent-run", scope, settings)?; + + // Only top-level chat runs provide a correlation key. Sub-agents deliberately + // omit it, so they cannot overwrite the trace used to attribute the parent + // run's delivered reply. + if let Some(key) = trace_correlation_key { + moltis_observability::remember_trace(key, recorder.trace_id()); + } + + recorder.set_metadata("provider", serde_json::Value::String(provider.to_string())); + recorder.set_metadata("model", serde_json::Value::String(model.to_string())); + recorder.set_input(user_content_to_json(user_content)); + Some(recorder) +} + +/// Render turn input for the trace. +/// +/// Multimodal turns report only their shape: the image bytes are already +/// carried elsewhere and inlining base64 here would dwarf every other +/// attribute in the payload. +#[must_use] +pub fn user_content_to_json(content: &UserContent) -> serde_json::Value { + match content { + UserContent::Text(text) => serde_json::Value::String(text.clone()), + UserContent::Multimodal(parts) => serde_json::json!({ + "type": "multimodal", + "parts": parts.len(), + }), + } +} + +/// Convert provider usage counters into the observability model. +/// +/// Provider `input_tokens` is already exclusive of cached tokens, so the +/// buckets are carried across as-is rather than being summed: Langfuse prices +/// cache reads and writes differently from fresh input, and folding them +/// together silently inflates reported cost. +#[must_use] +pub const fn to_token_usage(usage: &Usage) -> TokenUsage { + TokenUsage::from_provider_totals( + usage.input_tokens, + usage.output_tokens, + usage.cache_read_tokens, + usage.cache_write_tokens, + ) +} + +/// Name for the generation step of a given iteration. +#[must_use] +pub fn generation_name(provider: &str, model: &str) -> String { + format!("{provider}/{model}") +} + +/// Attach request parameters already exposed by the provider and agent model. +pub fn set_generation_parameters( + step: &mut moltis_observability::StepGuard, + provider: &dyn LlmProvider, + controls: &AgentToolControls, +) { + if let Some(effort) = provider.reasoning_effort() { + step.set_model_parameter("reasoning_effort", serde_json::json!(effort.as_str())); + } + if let Some(tool_choice) = &controls.tool_choice { + step.set_model_parameter("tool_choice", serde_json::json!(tool_choice)); + } +} + +/// Open a generation observation for the concrete provider selected for an attempt. +pub fn begin_generation_step( + recorder: Option<&TurnRecorder>, + identity: &ProviderIdentity, + provider: &dyn LlmProvider, + controls: &AgentToolControls, + iteration: usize, + tool_count: usize, + messages: &[ChatMessage], +) -> Option { + recorder.map(|recorder| { + let mut step = recorder.step( + ObservationKind::Generation, + generation_name(&identity.provider, &identity.model), + ); + step.set_model(identity.model.clone()); + step.set_metadata("provider", serde_json::json!(identity.provider)); + set_generation_parameters(&mut step, provider, controls); + step.set_metadata("iteration", serde_json::json!(iteration)); + step.set_metadata("tool_count", serde_json::json!(tool_count)); + step.set_input(serde_json::Value::Array( + messages.iter().map(message_to_telemetry_value).collect(), + )); + step + }) +} + +/// Serialize a provider message without copying inline image payloads into telemetry. +fn message_to_telemetry_value(message: &ChatMessage) -> serde_json::Value { + let ChatMessage::User { + content: UserContent::Multimodal(parts), + name, + } = message + else { + return message.to_openai_value(); + }; + + let content = parts + .iter() + .map(|part| match part { + ContentPart::Text(text) => serde_json::json!({ "type": "text", "text": text }), + ContentPart::Image { media_type, .. } => serde_json::json!({ + "type": "image", + "media_type": media_type, + "data": "[omitted]", + }), + }) + .collect::>(); + let mut value = serde_json::json!({ "role": "user", "content": content }); + if let Some(sanitized) = name + .as_ref() + .and_then(|name| ChatMessage::sanitize_message_name(name)) + { + value["name"] = serde_json::Value::String(sanitized); + } + value +} + +/// Extract a provider's terminal reason from a raw streaming event. +#[must_use] +pub fn finish_reason(raw: &serde_json::Value) -> Option { + raw.pointer("/choices/0/finish_reason") + .or_else(|| raw.get("stop_reason")) + .or_else(|| raw.pointer("/response/status")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +/// Extract separately reported reasoning-token usage from a raw event. +#[must_use] +pub fn reasoning_tokens(raw: &serde_json::Value) -> Option { + [ + "/usage/completion_tokens_details/reasoning_tokens", + "/usage/output_tokens_details/reasoning_tokens", + "/response/usage/output_tokens_details/reasoning_tokens", + "/choices/0/usage/completion_tokens_details/reasoning_tokens", + ] + .iter() + .find_map(|pointer| raw.pointer(pointer).and_then(serde_json::Value::as_u64)) + .map(|tokens| u32::try_from(tokens).unwrap_or(u32::MAX)) +} + +/// Close the trace root for every return path from an agent loop. +pub fn finish_turn( + recorder: Option<&TurnRecorder>, + result: &Result, +) { + let Some(recorder) = recorder else { + return; + }; + match result { + Ok(run) => { + recorder.set_output(serde_json::Value::String(run.text.clone())); + recorder.set_metadata("iterations", serde_json::json!(run.iterations)); + recorder.set_metadata("tool_calls", serde_json::json!(run.tool_calls_made)); + recorder.finish(); + }, + Err(error) => recorder.finish_with_error(error.to_string()), + } +} + +/// Close a tool observation with the exact content persisted for the model. +pub fn finish_tool_step( + step: Option, + persisted_output: &str, + success: bool, + error: Option<&str>, + persisted_result: &serde_json::Value, +) { + let Some(mut step) = step else { + return; + }; + step.set_output(serde_json::Value::String(persisted_output.to_string())); + let failure = error + .map(str::to_string) + .or_else(|| persisted_tool_result_failure(persisted_result)); + if !success || failure.is_some() { + step.fail(failure.unwrap_or_else(|| "tool execution failed".to_string())); + } + step.finish(); +} + +/// The observation kind a tool call should be recorded as. +/// +/// Retrieval tools are reported as `RETRIEVER` so backends that special-case +/// RAG steps light up, rather than showing every tool as an opaque `TOOL`. +#[must_use] +pub fn tool_observation_kind(tool_name: &str) -> ObservationKind { + const RETRIEVAL_TOOLS: &[&str] = &[ + "memory_search", + "memory_query", + "code_search", + "code_index_search", + "web_search", + "search", + ]; + if RETRIEVAL_TOOLS.contains(&tool_name) { + ObservationKind::Retriever + } else { + ObservationKind::Tool + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use std::{ + sync::{Arc, Mutex}, + time::Duration, + }; + + use moltis_observability::{Event, ObservationSink, RecorderSettings, RedactionPolicy}; + + use super::*; + + #[derive(Default)] + struct CollectingSink { + events: Mutex>, + } + + #[async_trait::async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + "agent-test" + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + fn test_recorder(sink: Arc) -> TurnRecorder { + TurnRecorder::begin_with_sink(sink, "agent-run", TraceScope::default(), RecorderSettings { + redaction: RedactionPolicy::default(), + capture_input: true, + capture_output: true, + capture_tool_io: true, + sample_rate: 1.0, + }) + .expect("sampled recorder") + } + + fn binding() -> ChannelBinding { + ChannelBinding { + surface: Some("chat".into()), + session_kind: None, + channel_type: Some("telegram".into()), + account_id: Some("acct-1".into()), + chat_id: Some("chat-1".into()), + chat_type: Some("dm".into()), + sender_id: Some("42".into()), + } + } + + #[test] + fn session_key_becomes_the_backend_session_id() { + let scope = trace_scope( + "agent:main:main", + None, + "production".into(), + "20260726.01".into(), + ); + assert_eq!(scope.session_id.as_deref(), Some("agent:main:main")); + } + + #[test] + fn empty_session_keys_are_omitted_rather_than_grouped() { + // An empty string would collapse every unattributed turn into one + // giant session in the backend's session view. + let scope = trace_scope("", None, "production".into(), "1.0".into()); + assert!(scope.session_id.is_none()); + } + + #[test] + fn user_id_is_namespaced_by_channel() { + // Otherwise Telegram user 42 and Slack user 42 merge into one person. + let scope = trace_scope( + "agent:main:main", + Some(&binding()), + "production".into(), + "1.0".into(), + ); + assert_eq!(scope.user_id.as_deref(), Some("telegram:42")); + } + + #[test] + fn sender_without_a_channel_type_is_used_verbatim() { + let unnamed = ChannelBinding { + channel_type: None, + ..binding() + }; + let scope = trace_scope( + "agent:main:main", + Some(&unnamed), + "prod".into(), + "1.0".into(), + ); + assert_eq!(scope.user_id.as_deref(), Some("42")); + } + + #[test] + fn channel_provenance_and_agent_become_tags() { + let scope = trace_scope( + "agent:support:channel:telegram:account:a:peer:user:42", + Some(&binding()), + "production".into(), + "1.0".into(), + ); + + assert!(scope.tags.contains(&"channel:telegram".to_string())); + assert!(scope.tags.contains(&"surface:chat".to_string())); + assert!(scope.tags.contains(&"chat:dm".to_string())); + assert!(scope.tags.contains(&"agent:support".to_string())); + } + + #[test] + fn a_turn_with_no_channel_still_produces_a_usable_scope() { + let scope = trace_scope("agent:main:main", None, "staging".into(), "1.0".into()); + + assert!(scope.user_id.is_none()); + assert_eq!(scope.tags, vec!["agent:main".to_string()]); + assert_eq!(scope.environment.as_deref(), Some("staging")); + } + + #[test] + fn malformed_session_keys_do_not_produce_an_empty_agent_tag() { + let scope = trace_scope("agent:", None, "prod".into(), "1.0".into()); + assert!( + !scope.tags.iter().any(|t| t == "agent:"), + "empty agent tag is noise in the backend's tag filter" + ); + } + + #[test] + fn usage_conversion_preserves_cache_buckets_separately() { + let usage = Usage { + input_tokens: 100, + output_tokens: 50, + cache_read_tokens: 900, + cache_write_tokens: 20, + }; + let converted = to_token_usage(&usage); + + assert_eq!(converted.input, 100); + assert_eq!(converted.cache_read, 900); + assert_eq!(converted.cache_write, 20); + // Summing cache into input would inflate the priced fresh-input count. + assert_ne!(converted.input, 1020); + } + + #[test] + fn multimodal_input_reports_shape_not_image_bytes() { + let json = user_content_to_json(&UserContent::Multimodal(Vec::new())); + assert_eq!(json["type"], "multimodal"); + assert_eq!(json["parts"], 0); + } + + #[test] + fn generation_messages_omit_inline_image_data() { + let sentinel = "sensitive-base64-payload"; + let message = ChatMessage::user_multimodal_named( + vec![ + ContentPart::Text("describe this".into()), + ContentPart::Image { + media_type: "image/png".into(), + data: sentinel.into(), + }, + ], + "Channel User", + ); + + let value = message_to_telemetry_value(&message); + let encoded = value.to_string(); + assert!(!encoded.contains(sentinel)); + assert!(!encoded.contains("data:image/png;base64")); + assert_eq!(value["content"][1]["media_type"], "image/png"); + assert_eq!(value["content"][1]["data"], "[omitted]"); + assert_eq!(value["name"], "Channel_User"); + } + + #[test] + fn text_input_is_carried_verbatim() { + let json = user_content_to_json(&UserContent::Text("hello".into())); + assert_eq!(json, serde_json::Value::String("hello".into())); + } + + #[test] + fn retrieval_tools_are_recorded_as_retriever_observations() { + assert_eq!( + tool_observation_kind("memory_search"), + ObservationKind::Retriever + ); + assert_eq!( + tool_observation_kind("code_search"), + ObservationKind::Retriever + ); + assert_eq!(tool_observation_kind("exec"), ObservationKind::Tool); + } + + #[test] + fn generation_name_identifies_provider_and_model() { + assert_eq!( + generation_name("anthropic", "claude-opus-4"), + "anthropic/claude-opus-4" + ); + } + + #[test] + fn finish_reason_supports_chat_anthropic_and_responses_shapes() { + assert_eq!( + finish_reason(&serde_json::json!({"choices": [{"finish_reason": "tool_calls"}]})) + .as_deref(), + Some("tool_calls") + ); + assert_eq!( + finish_reason(&serde_json::json!({"stop_reason": "end_turn"})).as_deref(), + Some("end_turn") + ); + assert_eq!( + finish_reason(&serde_json::json!({"response": {"status": "completed"}})).as_deref(), + Some("completed") + ); + } + + #[test] + fn reasoning_tokens_supports_chat_and_responses_usage() { + assert_eq!( + reasoning_tokens(&serde_json::json!({ + "usage": {"completion_tokens_details": {"reasoning_tokens": 42}} + })), + Some(42) + ); + assert_eq!( + reasoning_tokens(&serde_json::json!({ + "response": {"usage": {"output_tokens_details": {"reasoning_tokens": 84}}} + })), + Some(84) + ); + } + + #[test] + fn collecting_sink_receives_successful_and_failed_turn_completions() { + let sink = Arc::new(CollectingSink::default()); + let successful = test_recorder(Arc::clone(&sink)); + finish_turn( + Some(&successful), + &Ok(AgentRunResult { + text: "finished".into(), + iterations: 2, + tool_calls_made: 1, + usage: Usage::default(), + request_usage: Usage::default(), + raw_llm_responses: Vec::new(), + }), + ); + + let failed = test_recorder(Arc::clone(&sink)); + finish_turn( + Some(&failed), + &Err(AgentRunError::Other(anyhow::anyhow!("provider failed"))), + ); + + let traces = sink + .events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .filter_map(|event| match event { + Event::Trace(trace) => Some((**trace).clone()), + _ => None, + }) + .collect::>(); + assert!(traces.iter().any(|trace| { + trace.output == Some(serde_json::Value::String("finished".into())) + && trace.metadata.get("iterations") == Some(&serde_json::json!(2)) + })); + assert!(traces.iter().any(|trace| { + trace.metadata.get("error") == Some(&serde_json::json!("provider failed")) + })); + } + + #[test] + fn collecting_sink_gets_persisted_tool_output_and_logical_failure() { + let sink = Arc::new(CollectingSink::default()); + let recorder = test_recorder(Arc::clone(&sink)); + let mut step = recorder.step(ObservationKind::Tool, "browser"); + step.set_input(serde_json::json!({"url": "https://example.com"})); + let persisted = serde_json::json!({"result": {"success": false, "secret": "[removed]"}}); + let output = persisted.to_string(); + + finish_tool_step(Some(step), &output, true, None, &persisted); + + let events = sink + .events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let tool = events.iter().find_map(|event| match event { + Event::ObservationEnd(observation) if observation.kind == ObservationKind::Tool => { + Some(observation) + }, + _ => None, + }); + let tool = tool.expect("completed tool observation"); + assert_eq!(tool.output, Some(serde_json::Value::String(output))); + assert_eq!(tool.level, moltis_observability::Level::Error); + assert_eq!( + tool.status_message.as_deref(), + Some("tool returned success: false") + ); + } +} diff --git a/crates/agents/src/runner/mod.rs b/crates/agents/src/runner/mod.rs index 31f47fb61e..1f9ef0e675 100644 --- a/crates/agents/src/runner/mod.rs +++ b/crates/agents/src/runner/mod.rs @@ -1,6 +1,7 @@ //! Agent runner: LLM call loop with tool execution, retry, and streaming support. mod helpers; +pub mod instrumentation; mod non_streaming; pub mod retry; mod streaming; diff --git a/crates/agents/src/runner/non_streaming.rs b/crates/agents/src/runner/non_streaming.rs index bbfc375091..66972138e5 100644 --- a/crates/agents/src/runner/non_streaming.rs +++ b/crates/agents/src/runner/non_streaming.rs @@ -10,7 +10,10 @@ use { use moltis_common::hooks::{HookAction, HookPayload, HookRegistry}; use crate::{ - model::{AgentToolControls, ChatMessage, LlmProvider, ToolCall, ToolChoice, UserContent}, + model::{ + AgentToolControls, ChatMessage, LlmProvider, ProviderAttemptEvent, ToolCall, ToolChoice, + UserContent, + }, response_sanitizer::recover_tool_calls_from_content, tool_arg_validator::{coerce_scalar_args, validate_tool_args}, tool_loop_detector::ToolCallFingerprint, @@ -27,14 +30,14 @@ use super::{ channel_binding_from_tool_context, dispatch_after_llm_call_hook, dispatch_before_agent_start_hook, empty_tool_name_retry_prompt, explicit_shell_command_from_user_content, find_empty_tool_name_call, finish_agent_run, - has_named_tool_call, is_substantive_answer_text, log_tool_argument_diagnostic, + has_named_tool_call, instrumentation, is_substantive_answer_text, log_tool_argument_diagnostic, record_answer_text, resolve_tool_lookup, retry::{ RATE_LIMIT_MAX_RETRIES, is_context_window_error, next_retry_delay_ms, resolve_agent_max_iterations, }, sanitize_tool_name, - tool_result::sanitize_tool_result, + tool_result::{sanitize_tool_result, tool_result_failure}, }; use crate::tool_loop_detector::ToolLoopDetector; @@ -154,9 +157,29 @@ pub async fn run_agent_loop_with_context_and_limits( .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); + let trace_correlation_key = tool_context + .as_ref() + .and_then(|ctx| ctx.get("_trace_correlation_key")) + .and_then(|value| value.as_str()); let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); + // Instrumentation. This path serves sub-agents (`spawn_agent`) and silent + // turns, so leaving it uninstrumented would drop exactly the nested-agent + // traces the observation taxonomy exists to represent. + let turn_recorder = Arc::new(instrumentation::begin_turn( + &session_key_for_hooks, + trace_correlation_key, + channel_for_hooks.as_ref(), + provider.name(), + provider.id(), + user_content, + instrumentation::recorder_settings(&config.instrumentation), + config.instrumentation.environment.clone(), + instrumentation::release(&config.instrumentation), + )); + + let result: std::result::Result = async { dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -288,13 +311,57 @@ pub async fn run_agent_loop_with_context_and_limits( cb(RunnerEvent::Thinking); } - let mut response = match provider - .complete_with_options(&messages, &schemas_for_api, &tool_controls) - .await - { + let mut generation_step = None; + let completion_result = { + let mut on_attempt = |event| match event { + ProviderAttemptEvent::Started(identity) => { + if let Some(recorder) = turn_recorder.as_ref().as_ref() { + recorder.set_metadata( + "provider", + serde_json::Value::String(identity.provider.clone()), + ); + recorder.set_metadata( + "model", + serde_json::Value::String(identity.model.clone()), + ); + } + generation_step = instrumentation::begin_generation_step( + turn_recorder.as_ref().as_ref(), + &identity, + provider.as_ref(), + &tool_controls, + iterations, + schemas_for_api.len(), + &messages, + ); + }, + ProviderAttemptEvent::Failed { error, .. } => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + step.finish(); + } + }, + }; + + provider + .complete_with_options_tracked( + &messages, + &schemas_for_api, + &tool_controls, + &mut on_attempt, + ) + .await + }; + let mut response = match completion_result { Ok(r) => r, Err(e) => { let msg = e.to_string(); + // Close as failed before any early return, so a failed call is + // not rendered as a successful span by the drop guard. + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } if is_context_window_error(&msg) { return Err(AgentRunError::ContextWindowExceeded(msg)); } @@ -329,6 +396,21 @@ pub async fn run_agent_loop_with_context_and_limits( cb(RunnerEvent::ThinkingDone); } + if let Some(mut step) = generation_step.take() { + step.set_usage(instrumentation::to_token_usage(&response.usage)); + step.set_output(serde_json::json!({ + "text": response.text, + "tool_calls": response.tool_calls.iter().map(|call| serde_json::json!({ + "id": call.id, + "name": call.name, + })).collect::>(), + })); + if !response.tool_calls.is_empty() { + step.set_metadata("tool_calls", serde_json::json!(response.tool_calls.len())); + } + step.finish(); + } + usage_accumulator.record_request(response.usage.clone()); info!( @@ -543,6 +625,7 @@ pub async fn run_agent_loop_with_context_and_limits( .tool_calls .iter() .map(|tc| { + let tool_turn_recorder = Arc::clone(&turn_recorder); let sanitized = sanitize_tool_name(&tc.name); if *sanitized != tc.name { debug!(original = %tc.name, sanitized = %sanitized, "sanitized mangled tool name"); @@ -624,14 +707,27 @@ pub async fn run_agent_loop_with_context_and_limits( } async move { + let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { + recorder.step( + instrumentation::tool_observation_kind(&tc_name), + tc_name.clone(), + ) + }); if let Some(err_msg) = validation_error { + if let Some(step) = tool_step.as_mut() { + step.set_input(args); + } return ( false, serde_json::json!({ "error": err_msg.clone() }), Some(err_msg), true, + tool_step, ); } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } // Run BeforeToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::BeforeToolCall { @@ -649,6 +745,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ); }, Ok(HookAction::ModifyPayload(v)) => { @@ -668,6 +765,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str.clone() }), Some(err_str), false, + tool_step, ); } } @@ -679,27 +777,21 @@ pub async fn run_agent_loop_with_context_and_limits( } } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } + if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { - // Check if the result indicates a logical failure - // (e.g., BrowserResponse with success: false) - let has_error = val.get("error").is_some() - || val.get("success") == Some(&serde_json::json!(false)); - let error_msg = if has_error { - val.get("error") - .and_then(|e| e.as_str()) - .map(String::from) - } else { - None - }; - + let error = tool_result_failure(&val); + let success = error.is_none(); // Dispatch AfterToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: !has_error, + success, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -708,12 +800,13 @@ pub async fn run_agent_loop_with_context_and_limits( } } - if has_error { - // Tool executed but returned an error in the result - (false, serde_json::json!({ "result": val }), error_msg, false) - } else { - (true, serde_json::json!({ "result": val }), None, false) - } + ( + success, + serde_json::json!({ "result": val }), + error, + false, + tool_step, + ) }, Err(e) => { let err_str = e.to_string(); @@ -735,6 +828,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) }, } @@ -745,6 +839,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } @@ -764,7 +859,8 @@ pub async fn run_agent_loop_with_context_and_limits( // stale intervention — the reset() abandons it cleanly. // 2. A batch that races through both escalation stages must still // deliver the stage-1 nudge first, not skip straight to strip. - for (tc, (success, mut result, error, rejected)) in response.tool_calls.iter().zip(results) + for (tc, (success, mut result, error, rejected, mut tool_step)) in + response.tool_calls.iter().zip(results) { if success { info!(tool = %tc.name, id = %tc.id, "tool execution succeeded"); @@ -804,7 +900,7 @@ pub async fn run_agent_loop_with_context_and_limits( id: tc.id.clone(), name: tc.name.clone(), success, - error, + error: error.clone(), result: if success { result.get("result").cloned() } else { @@ -849,6 +945,13 @@ pub async fn run_agent_loop_with_context_and_limits( // multimodal content in tool results. Images are stripped but the UI // still receives them via ToolCallEnd event. let tool_result_str = sanitize_tool_result(&result.to_string(), max_tool_result_bytes); + instrumentation::finish_tool_step( + tool_step.take(), + &tool_result_str, + success, + error.as_deref(), + &result, + ); debug!( tool = %tc.name, id = %tc.id, @@ -868,6 +971,11 @@ pub async fn run_agent_loop_with_context_and_limits( on_event, ); } + } + .await; + + instrumentation::finish_turn(turn_recorder.as_ref().as_ref(), &result); + result } /// Convenience wrapper matching the old stub signature. diff --git a/crates/agents/src/runner/streaming.rs b/crates/agents/src/runner/streaming.rs index 941be50f59..2a539dea68 100644 --- a/crates/agents/src/runner/streaming.rs +++ b/crates/agents/src/runner/streaming.rs @@ -16,8 +16,9 @@ use moltis_common::hooks::{HookAction, HookPayload, HookRegistry}; use crate::{ model::{ - AgentToolControls, ChatMessage, LlmProvider, StreamEvent, ToolCall, ToolChoice, Usage, - UserContent, decode_tool_call_arguments_from_str, push_capped_provider_raw_event, + AgentToolControls, ChatMessage, LlmProvider, ProviderAttemptEvent, ProviderIdentity, + StreamEvent, ToolCall, ToolChoice, TrackedStreamEvent, Usage, UserContent, + decode_tool_call_arguments_from_str, push_capped_provider_raw_event, }, response_sanitizer::{clean_response, recover_tool_calls_from_content}, tool_arg_validator::{coerce_scalar_args, validate_tool_args}, @@ -35,14 +36,14 @@ use super::{ channel_binding_from_tool_context, dispatch_after_llm_call_hook, dispatch_before_agent_start_hook, empty_tool_name_retry_prompt, explicit_shell_command_from_user_content, find_empty_tool_name_call, finish_agent_run, - has_named_tool_call, is_substantive_answer_text, log_tool_argument_diagnostic, + has_named_tool_call, instrumentation, is_substantive_answer_text, log_tool_argument_diagnostic, resolve_tool_lookup, retry::{ RATE_LIMIT_MAX_RETRIES, is_context_window_error, next_retry_delay_ms, resolve_agent_max_iterations, }, sanitize_tool_name, streaming_tool_call_message_content, - tool_result::sanitize_tool_result, + tool_result::{sanitize_tool_result, tool_result_failure}, }; use crate::tool_loop_detector::ToolLoopDetector; @@ -144,9 +145,28 @@ pub async fn run_agent_loop_streaming_with_limits( .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); + let trace_correlation_key = tool_context + .as_ref() + .and_then(|ctx| ctx.get("_trace_correlation_key")) + .and_then(|value| value.as_str()); let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); + // Instrumentation. `None` when no backend is configured or the turn was + // not sampled, in which case every call below is skipped. + let turn_recorder = Arc::new(instrumentation::begin_turn( + &session_key_for_hooks, + trace_correlation_key, + channel_for_hooks.as_ref(), + provider.name(), + provider.id(), + user_content, + instrumentation::recorder_settings(&config.instrumentation), + config.instrumentation.environment.clone(), + instrumentation::release(&config.instrumentation), + )); + + let result: std::result::Result = async { dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -289,7 +309,10 @@ pub async fn run_agent_loop_streaming_with_limits( // Use streaming API. #[cfg(feature = "metrics")] let iter_start = std::time::Instant::now(); - let mut stream = provider.stream_with_tools_and_options( + let mut generation_step = None; + let mut selected_identity: Option = None; + + let mut stream = provider.stream_with_tools_and_options_tracked( messages.clone(), schemas_for_api.clone(), tool_controls.clone(), @@ -310,31 +333,82 @@ pub async fn run_agent_loop_streaming_with_limits( std::collections::HashMap::new(); let mut request_usage = Usage::default(); let mut stream_error: Option = None; + let mut finish_reason: Option = None; + let mut reasoning_tokens: u32 = 0; while let Some(event) = stream.next().await { match event { - StreamEvent::Delta(text) => { + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) => { + if let Some(recorder) = turn_recorder.as_ref().as_ref() { + recorder.set_metadata( + "provider", + serde_json::Value::String(identity.provider.clone()), + ); + recorder.set_metadata( + "model", + serde_json::Value::String(identity.model.clone()), + ); + } + generation_step = instrumentation::begin_generation_step( + turn_recorder.as_ref().as_ref(), + &identity, + provider.as_ref(), + &tool_controls, + iterations, + schemas_for_api.len(), + &messages, + ); + selected_identity = Some(identity); + }, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { error, .. }) => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); + step.finish(); + } + request_usage = Usage::default(); + finish_reason = None; + reasoning_tokens = 0; + }, + TrackedStreamEvent::Event(StreamEvent::Delta(text)) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } accumulated_text.push_str(&text); if let Some(cb) = on_event { cb(RunnerEvent::TextDelta(text.clone())); cb(RunnerEvent::FinalText(text)); } }, - StreamEvent::ProviderRaw(raw) => { + TrackedStreamEvent::Event(StreamEvent::ProviderRaw(raw)) => { + if let Some(reason) = instrumentation::finish_reason(&raw) { + finish_reason = Some(reason); + } + if let Some(tokens) = instrumentation::reasoning_tokens(&raw) { + reasoning_tokens = tokens; + } push_capped_provider_raw_event(&mut raw_llm_responses, raw); }, - StreamEvent::ReasoningDelta(text) => { + TrackedStreamEvent::Event(StreamEvent::ReasoningDelta(text)) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } accumulated_reasoning.push_str(&text); if let Some(cb) = on_event { cb(RunnerEvent::ThinkingText(accumulated_reasoning.clone())); } }, - StreamEvent::ToolCallStart { + TrackedStreamEvent::Event(StreamEvent::ToolCallStart { id, name, index, metadata, - } => { + }) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } let vec_pos = tool_calls.len(); debug!(tool = %name, id = %id, stream_index = index, vec_pos, "tool call started in stream"); tool_calls.push(ToolCall { @@ -347,17 +421,23 @@ pub async fn run_agent_loop_streaming_with_limits( stream_idx_to_vec_pos.insert(index, vec_pos); tool_call_args.insert(index, String::new()); }, - StreamEvent::ToolCallArgumentsDelta { index, delta } => { + TrackedStreamEvent::Event(StreamEvent::ToolCallArgumentsDelta { + index, + delta, + }) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } if let Some(args) = tool_call_args.get_mut(&index) { args.push_str(&delta); } }, - StreamEvent::ToolCallComplete { index } => { + TrackedStreamEvent::Event(StreamEvent::ToolCallComplete { index }) => { // Arguments are finalized after stream completes. // Just log for now - we'll parse accumulated args later. debug!(index, "tool call arguments complete"); }, - StreamEvent::Done(usage) => { + TrackedStreamEvent::Event(StreamEvent::Done(usage)) => { request_usage = usage.clone(); debug!( input_tokens = request_usage.input_tokens, @@ -369,8 +449,12 @@ pub async fn run_agent_loop_streaming_with_limits( #[cfg(feature = "metrics")] { - let provider_name = provider.name().to_string(); - let model_id = provider.id().to_string(); + let provider_name = selected_identity + .as_ref() + .map_or_else(|| provider.name().to_string(), |id| id.provider.clone()); + let model_id = selected_identity + .as_ref() + .map_or_else(|| provider.id().to_string(), |id| id.model.clone()); let duration = iter_start.elapsed().as_secs_f64(); counter!( llm_metrics::COMPLETIONS_TOTAL, @@ -410,7 +494,7 @@ pub async fn run_agent_loop_streaming_with_limits( .record(duration); } }, - StreamEvent::Error(msg) => { + TrackedStreamEvent::Event(StreamEvent::Error(msg)) => { stream_error = Some(msg); break; }, @@ -423,6 +507,16 @@ pub async fn run_agent_loop_streaming_with_limits( // Handle stream errors — retry on transient failures/rate limits. if let Some(err) = stream_error { + // Close the generation as failed before any early return, so a + // failed call is visible in the backend rather than silently + // ending as a span with no error on it. + if let Some(mut step) = generation_step.take() { + step.fail(err.clone()); + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); + step.finish(); + } if is_context_window_error(&err) { return Err(AgentRunError::ContextWindowExceeded(err)); } @@ -453,6 +547,27 @@ pub async fn run_agent_loop_streaming_with_limits( return Err(AgentRunError::Other(anyhow::anyhow!(err))); } + if let Some(mut step) = generation_step.take() { + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); + step.set_output(serde_json::json!({ + "text": accumulated_text, + "reasoning": accumulated_reasoning, + "tool_calls": tool_calls.iter().map(|call| serde_json::json!({ + "id": call.id, + "name": call.name, + })).collect::>(), + })); + if !tool_calls.is_empty() { + step.set_metadata("tool_calls", serde_json::json!(tool_calls.len())); + } + if let Some(reason) = finish_reason { + step.set_metadata("finish_reason", serde_json::Value::String(reason)); + } + step.finish(); + } + usage_accumulator.record_request(request_usage.clone()); // Finalize tool call arguments from accumulated strings. @@ -689,6 +804,7 @@ pub async fn run_agent_loop_streaming_with_limits( let tool_futures: Vec<_> = tool_calls .iter() .map(|tc| { + let tool_turn_recorder = Arc::clone(&turn_recorder); let sanitized = sanitize_tool_name(&tc.name); if *sanitized != tc.name { debug!(original = %tc.name, sanitized = %sanitized, "sanitized mangled tool name"); @@ -765,14 +881,27 @@ pub async fn run_agent_loop_streaming_with_limits( } async move { + let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { + recorder.step( + instrumentation::tool_observation_kind(&tc_name), + tc_name.clone(), + ) + }); if let Some(err_msg) = validation_error { + if let Some(step) = tool_step.as_mut() { + step.set_input(args); + } return ( false, serde_json::json!({ "error": err_msg.clone() }), Some(err_msg), true, + tool_step, ); } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } // Run BeforeToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::BeforeToolCall { @@ -790,6 +919,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ); } Ok(HookAction::ModifyPayload(v)) => { @@ -809,6 +939,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str.clone() }), Some(err_str), false, + tool_step, ); } } @@ -820,26 +951,20 @@ pub async fn run_agent_loop_streaming_with_limits( } } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } + if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { - // Check if the result indicates a logical failure - // (e.g., BrowserResponse with success: false) - let has_error = val.get("error").is_some() - || val.get("success") == Some(&serde_json::json!(false)); - let error_msg = if has_error { - val.get("error") - .and_then(|e| e.as_str()) - .map(String::from) - } else { - None - }; - + let error = tool_result_failure(&val); + let success = error.is_none(); if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: !has_error, + success, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -848,11 +973,13 @@ pub async fn run_agent_loop_streaming_with_limits( } } - if has_error { - (false, serde_json::json!({ "result": val }), error_msg, false) - } else { - (true, serde_json::json!({ "result": val }), None, false) - } + ( + success, + serde_json::json!({ "result": val }), + error, + false, + tool_step, + ) } Err(e) => { let err_str = e.to_string(); @@ -873,6 +1000,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } @@ -883,6 +1011,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } @@ -896,7 +1025,9 @@ pub async fn run_agent_loop_streaming_with_limits( // Intervention is derived from the detector's post-batch state // via `consume_pending_action()` below — see the non-streaming // path for the rationale (issue #658). - for (tc, (success, mut result, error, rejected)) in tool_calls.iter().zip(results) { + for (tc, (success, mut result, error, rejected, mut tool_step)) in + tool_calls.iter().zip(results) + { if success { info!(tool = %tc.name, id = %tc.id, "tool execution succeeded"); trace!(tool = %tc.name, result = %result, "tool result"); @@ -933,7 +1064,7 @@ pub async fn run_agent_loop_streaming_with_limits( id: tc.id.clone(), name: tc.name.clone(), success, - error, + error: error.clone(), result: if success { result.get("result").cloned() } else { @@ -978,6 +1109,13 @@ pub async fn run_agent_loop_streaming_with_limits( // multimodal content in tool results. Images are stripped but the UI // still receives them via ToolCallEnd event. let tool_result_str = sanitize_tool_result(&result.to_string(), max_tool_result_bytes); + instrumentation::finish_tool_step( + tool_step.take(), + &tool_result_str, + success, + error.as_deref(), + &result, + ); debug!( tool = %tc.name, id = %tc.id, @@ -1011,4 +1149,9 @@ pub async fn run_agent_loop_streaming_with_limits( } } } + } + .await; + + instrumentation::finish_turn(turn_recorder.as_ref().as_ref(), &result); + result } diff --git a/crates/agents/src/runner/tests/basic.rs b/crates/agents/src/runner/tests/basic.rs index 32f13a792a..0e59d23904 100644 --- a/crates/agents/src/runner/tests/basic.rs +++ b/crates/agents/src/runner/tests/basic.rs @@ -135,6 +135,63 @@ async fn test_simple_text_response() { assert_eq!(result.tool_calls_made, 0); } +struct NestedErrorDataTool; + +#[async_trait] +impl crate::tool_registry::AgentTool for NestedErrorDataTool { + fn name(&self) -> &str { + "echo_tool" + } + + fn description(&self) -> &str { + "Returns successful nested data" + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object"}) + } + + async fn execute(&self, _params: serde_json::Value) -> Result { + Ok(serde_json::json!({ + "success": true, + "result": {"error": "page value"} + })) + } +} + +#[tokio::test] +async fn test_nested_error_data_does_not_fail_a_successful_tool_call() { + let provider = Arc::new(ToolCallingProvider { + call_count: std::sync::atomic::AtomicUsize::new(0), + }); + let mut tools = ToolRegistry::new(); + tools.register(Box::new(NestedErrorDataTool)); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = Arc::clone(&events); + let on_event: OnEvent = Box::new(move |event| { + captured.lock().unwrap().push(event); + }); + + run_agent_loop( + provider, + &tools, + "You are a test bot.", + &UserContent::text("Hi"), + Some(&on_event), + None, + ) + .await + .unwrap(); + + assert!( + events + .lock() + .unwrap() + .iter() + .any(|event| { matches!(event, RunnerEvent::ToolCallEnd { success: true, .. }) }) + ); +} + #[tokio::test] async fn test_non_streaming_runner_uses_max_iteration_override() { let provider = Arc::new(ToolCallingProvider { diff --git a/crates/agents/src/runner/tool_result.rs b/crates/agents/src/runner/tool_result.rs index a7fc8ada37..6500034cb9 100644 --- a/crates/agents/src/runner/tool_result.rs +++ b/crates/agents/src/runner/tool_result.rs @@ -103,6 +103,74 @@ pub fn sanitize_tool_result(input: &str, max_bytes: usize) -> String { result } +/// Return a stable failure message for tool results that encode logical errors. +#[must_use] +pub fn tool_result_failure(result: &serde_json::Value) -> Option { + let error = result.get("error").filter(|value| !value.is_null()); + if let Some(error) = error { + return Some( + error + .as_str() + .map_or_else(|| error.to_string(), str::to_string), + ); + } + (result.get("success") == Some(&serde_json::Value::Bool(false))) + .then(|| "tool returned success: false".to_string()) +} + +/// Inspect the runner's `{ "result": }` persistence wrapper. +#[must_use] +pub fn persisted_tool_result_failure(result: &serde_json::Value) -> Option { + tool_result_failure(result).or_else(|| result.get("result").and_then(tool_result_failure)) +} + +#[cfg(test)] +mod failure_tests { + use super::{persisted_tool_result_failure, tool_result_failure}; + + #[test] + fn success_false_without_error_has_a_failure_message() { + assert_eq!( + tool_result_failure(&serde_json::json!({"success": false})).as_deref(), + Some("tool returned success: false") + ); + } + + #[test] + fn nested_result_data_is_not_treated_as_a_raw_tool_failure() { + assert_eq!( + tool_result_failure(&serde_json::json!({ + "success": true, + "result": {"error": "page value"} + })), + None + ); + } + + #[test] + fn top_level_failure_wins_even_when_result_is_present() { + assert_eq!( + tool_result_failure(&serde_json::json!({ + "error": "denied", + "result": {"value": 42} + })) + .as_deref(), + Some("denied") + ); + } + + #[test] + fn persisted_tool_errors_are_detected_inside_the_runner_wrapper() { + assert_eq!( + persisted_tool_result_failure(&serde_json::json!({ + "result": {"error": "denied"} + })) + .as_deref(), + Some("denied") + ); + } +} + // ── Multimodal tool result helpers ───────────────────────────────────────── /// Image extracted from a tool result for multimodal handling. diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.toml index d61581fa23..ed2fe38e71 100644 --- a/crates/channels/Cargo.toml +++ b/crates/channels/Cargo.toml @@ -10,14 +10,19 @@ http = { workspace = true } moltis-common = { workspace = true } moltis-config = { workspace = true } moltis-metrics = { optional = true, workspace = true } +moltis-observability = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } +[dev-dependencies] +anyhow = { workspace = true } + [features] default = [] metrics = ["dep:moltis-metrics"] diff --git a/crates/channels/src/feedback.rs b/crates/channels/src/feedback.rs new file mode 100644 index 0000000000..d259d83ede --- /dev/null +++ b/crates/channels/src/feedback.rs @@ -0,0 +1,786 @@ +//! Reaction feedback: recording which trace a reply came from, and turning a +//! later reaction on that reply into a score. +//! +//! The two halves are deliberately in one place because they share the +//! correlation key. The send side writes (channel, account, chat, message) → +//! trace; the reaction side reads it back. + +use std::sync::{Arc, RwLock}; + +use { + moltis_config::FeedbackSettings, + moltis_observability::{ + FeedbackSignal, FeedbackVocabulary, ScoreDeleteRecord, TraceId, feedback_score, + feedback_score_id, + }, + tracing::{debug, warn}, +}; + +use crate::trace_link::{TraceLink, TraceLinkStore}; + +/// What happened to an inbound reaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FeedbackOutcome { + /// A score was recorded for the trace. + Recorded(FeedbackSignal), + /// A previously recorded score was withdrawn. + Retracted, + /// The reaction carried no feedback signal. + NotFeedback, + /// The reply is no longer attributable to a trace. + UnknownMessage, + /// Feedback collection is switched off. + Disabled, +} + +/// The parts of the service that only exist once startup has a database. +struct Active { + links: Arc, + vocabulary: FeedbackVocabulary, + enabled: bool, + environment: Option, + retention_days: u32, +} + +/// Records reply/trace links and converts reactions into scores. +/// +/// Constructed empty and filled in by [`FeedbackService::apply`] once startup +/// has a database pool and the resolved configuration, mirroring how +/// instrumentation itself is installed. Every operation is inert until then, +/// so a reaction arriving mid-startup is ignored rather than panicking. +#[derive(Default)] +pub struct FeedbackService { + active: RwLock>, +} + +impl FeedbackService { + /// Install the link store and settings. + /// + /// Replaces any previous setup so the settings UI can reconfigure feedback + /// without a restart. + pub fn apply( + &self, + links: Arc, + settings: &FeedbackSettings, + environment: Option, + ) { + let next = Some(Active { + links, + vocabulary: FeedbackVocabulary::from_config(&settings.positive, &settings.negative), + enabled: settings.enabled, + environment, + retention_days: settings.link_retention_days, + }); + match self.active.write() { + Ok(mut guard) => *guard = next, + Err(poisoned) => *poisoned.into_inner() = next, + } + } + + /// Whether feedback collection is on. + #[must_use] + pub fn enabled(&self) -> bool { + self.read(|active| active.enabled).unwrap_or(false) + } + + /// Run `f` against the installed configuration, if there is one. + fn read(&self, f: impl FnOnce(&Active) -> T) -> Option { + let guard = self + .active + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.as_ref().map(f) + } + + /// Record that `message_ids` were produced by `trace_id`. + /// + /// Failures are logged, never propagated: losing attribution for a reply + /// must not fail the reply itself, which the user has already received. + pub async fn record_reply( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_ids: &[String], + trace_id: &TraceId, + session_key: Option<&str>, + ) { + let Some((links, enabled)) = + self.read(|active| (Arc::clone(&active.links), active.enabled)) + else { + return; + }; + if !enabled || message_ids.is_empty() { + return; + } + let created_at = now_unix(); + for message_id in message_ids { + let link = TraceLink { + channel_type: channel_type.to_string(), + account_id: account_id.to_string(), + chat_id: chat_id.to_string(), + message_id: message_id.clone(), + trace_id: trace_id.0.clone(), + session_key: session_key.map(str::to_string), + created_at, + }; + if let Err(error) = links.link(link).await { + warn!(%error, channel_type, "failed to record trace link for reply"); + } + } + } + + /// Handle a reaction change on a channel message. + /// + /// `scores_available` says whether a backend that can store a score is + /// running. Without one, both halves of the toggle would be recorded into + /// nothing, so the reaction is reported as disabled instead. + pub async fn on_reaction( + &self, + channel: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + emoji: &str, + user_id: &str, + added: bool, + scores_available: bool, + ) -> FeedbackOutcome { + // Classify before the database lookup: most reactions in a busy chat + // are not feedback, and they should not cost a query each. + let Some((enabled, signal)) = + self.read(|active| (active.enabled, active.vocabulary.classify(emoji))) + else { + return FeedbackOutcome::Disabled; + }; + if !enabled { + return FeedbackOutcome::Disabled; + } + let Some(signal) = signal else { + return FeedbackOutcome::NotFeedback; + }; + + self.submit_signal( + channel, + account_id, + chat_id, + message_id, + added.then_some(signal), + user_id, + Some(format!("{channel} reaction {emoji}")), + scores_available, + ) + .await + } + + /// Submit already-classified feedback from a typed UI boundary. + #[allow(clippy::too_many_arguments)] + pub async fn submit_signal( + &self, + channel: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + signal: Option, + user_id: &str, + comment: Option, + scores_available: bool, + ) -> FeedbackOutcome { + let Some((links, enabled, environment, retention_days)) = self.read(|active| { + ( + Arc::clone(&active.links), + active.enabled, + active.environment.clone(), + active.retention_days, + ) + }) else { + return FeedbackOutcome::Disabled; + }; + if !enabled || !scores_available { + return FeedbackOutcome::Disabled; + } + + let link = match links.lookup(channel, account_id, chat_id, message_id).await { + Ok(Some(link)) => link, + Ok(None) => { + debug!( + channel, + message_id, "reaction on a message with no trace link" + ); + return FeedbackOutcome::UnknownMessage; + }, + Err(error) => { + warn!(%error, channel, "failed to look up trace link for reaction"); + return FeedbackOutcome::UnknownMessage; + }, + }; + if link.created_at < retention_cutoff(retention_days) { + return FeedbackOutcome::UnknownMessage; + } + + let trace_id = TraceId(link.trace_id); + // Namespaced so the same numeric user id on two channels is two people. + let scoped_user = format!("{channel}:{user_id}"); + + if let Some(signal) = signal { + let score = feedback_score(&trace_id, signal, Some(&scoped_user), comment, environment); + moltis_observability::record(moltis_observability::Event::Score(Box::new(score))); + debug!(channel, ?signal, "recorded reaction feedback"); + return FeedbackOutcome::Recorded(signal); + } + + let score_id = feedback_score_id(&trace_id, Some(&scoped_user)); + moltis_observability::record(moltis_observability::Event::ScoreDelete(Box::new( + ScoreDeleteRecord::new(trace_id, score_id), + ))); + FeedbackOutcome::Retracted + } + + /// Drop links older than the active retention window. + pub async fn prune(&self) -> u64 { + let Some((links, retention_days)) = + self.read(|active| (Arc::clone(&active.links), active.retention_days)) + else { + return 0; + }; + let cutoff = retention_cutoff(retention_days); + match links.prune(cutoff).await { + Ok(removed) => removed, + Err(error) => { + warn!(%error, "failed to prune trace links"); + 0 + }, + } + } +} + +fn now_unix() -> i64 { + time::OffsetDateTime::now_utc().unix_timestamp() +} + +fn retention_cutoff(retention_days: u32) -> i64 { + time::OffsetDateTime::now_utc() + .checked_sub(time::Duration::days(i64::from(retention_days))) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH) + .unix_timestamp() +} + +#[allow(clippy::unwrap_used, clippy::expect_used)] +#[cfg(test)] +mod tests { + use {crate::Result as ChannelResult, std::sync::Mutex, tokio::sync::OnceCell}; + + use super::*; + + #[derive(Default)] + struct MemoryLinks { + rows: Mutex>, + } + + #[async_trait::async_trait] + impl TraceLinkStore for MemoryLinks { + async fn link(&self, link: TraceLink) -> ChannelResult<()> { + let mut rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + rows.retain(|r| { + !(r.channel_type == link.channel_type + && r.account_id == link.account_id + && r.chat_id == link.chat_id + && r.message_id == link.message_id) + }); + rows.push(link); + Ok(()) + } + + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> ChannelResult> { + let rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + Ok(rows + .iter() + .find(|r| { + r.channel_type == channel_type + && r.account_id == account_id + && r.chat_id == chat_id + && r.message_id == message_id + }) + .cloned()) + } + + async fn prune(&self, cutoff: i64) -> ChannelResult { + let mut rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + let before = rows.len(); + rows.retain(|r| r.created_at >= cutoff); + Ok((before - rows.len()) as u64) + } + } + + fn service(enabled: bool) -> (FeedbackService, Arc) { + let links = Arc::new(MemoryLinks::default()); + let settings = FeedbackSettings { + enabled, + ..FeedbackSettings::default() + }; + let service = FeedbackService::default(); + service.apply( + Arc::clone(&links) as Arc, + &settings, + Some("production".into()), + ); + (service, links) + } + + /// Collects scores emitted through the global sink. + /// + /// The sink is process-wide, so the tests that assert on emitted scores + /// share one and run behind a lock rather than racing each other. + static SINK_LOCK: OnceCell> = OnceCell::const_new(); + + async fn sink_guard() -> tokio::sync::MutexGuard<'static, ()> { + SINK_LOCK + .get_or_init(|| async { tokio::sync::Mutex::new(()) }) + .await + .lock() + .await + } + + #[derive(Default)] + struct CollectingSink { + events: Mutex>, + } + + #[async_trait::async_trait] + impl moltis_observability::ObservationSink for CollectingSink { + fn name(&self) -> &str { + "collecting" + } + + fn record(&self, event: moltis_observability::Event) { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(event); + } + + async fn flush(&self, _timeout: std::time::Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + async fn link_reply(links: &MemoryLinks, message_id: &str, trace_id: &str) { + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: message_id.into(), + trace_id: trace_id.into(), + session_key: Some("agent:main:main".into()), + created_at: now_unix(), + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn every_chunk_of_a_reply_is_linked() { + // Long replies are split across messages and a reader may react to any + // of them, so all ids must resolve to the turn. + let (service, links) = service(true); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into(), "2".into(), "3".into()], + &TraceId("trace-1".into()), + Some("agent:main:main"), + ) + .await; + + for id in ["1", "2", "3"] { + let found = links + .lookup("telegram", "bot-1", "chat-1", id) + .await + .unwrap(); + assert_eq!(found.expect("linked").trace_id, "trace-1"); + } + } + + #[tokio::test] + async fn a_thumbs_up_records_a_positive_score() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); + + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Recorded(FeedbackSignal::Positive)); + let events = sink.events.lock().unwrap_or_else(|e| e.into_inner()); + let scored = events + .iter() + .filter_map(|e| match e { + moltis_observability::Event::Score(s) => Some(s), + _ => None, + }) + .count(); + assert_eq!(scored, 1); + + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn a_reaction_that_is_not_feedback_costs_no_lookup() { + let (service, _links) = service(true); + // No link recorded at all: if this returned UnknownMessage it would + // mean the vocabulary check ran after the database query. + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f389}", + "99", + true, + false, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::NotFeedback); + } + + #[tokio::test] + async fn a_reaction_on_an_unlinked_message_is_ignored() { + let (service, _links) = service(true); + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "unknown", + "\u{1f44d}", + "99", + true, + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::UnknownMessage); + } + + #[tokio::test] + async fn removing_a_reaction_retracts_rather_than_scoring_again() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + false, + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Retracted); + let events = sink.events.lock().unwrap_or_else(|e| e.into_inner()); + assert!(matches!(events.as_slice(), [ + moltis_observability::Event::ScoreDelete(_) + ])); + drop(events); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn typed_feedback_does_not_depend_on_the_emoji_vocabulary() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); + let links = Arc::new(MemoryLinks::default()); + let service = FeedbackService::default(); + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + positive: vec!["party".into()], + negative: vec!["boo".into()], + ..FeedbackSettings::default() + }, + Some("production".into()), + ); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .submit_signal( + "telegram", + "bot-1", + "chat-1", + "42", + Some(FeedbackSignal::Positive), + "99", + Some("typed feedback".into()), + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Recorded(FeedbackSignal::Positive)); + assert!(matches!( + sink.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_slice(), + [moltis_observability::Event::Score(_)] + )); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn expired_links_are_rejected_even_before_pruning() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "old".into(), + trace_id: "trace-old".into(), + session_key: None, + created_at: retention_cutoff(30) - 1, + }) + .await + .unwrap(); + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "old", + "\u{1f44d}", + "99", + true, + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::UnknownMessage); + } + + #[tokio::test] + async fn score_feedback_is_disabled_without_a_score_backend() { + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + false, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Disabled); + } + + #[tokio::test] + async fn feedback_can_be_switched_off() { + let (service, links) = service(false); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + true, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Disabled); + } + + #[tokio::test] + async fn disabled_feedback_records_no_links() { + let (service, links) = service(false); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into()], + &TraceId("trace-1".into()), + None, + ) + .await; + + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "1") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn an_uninitialized_service_is_inert() { + // A reaction arriving before startup finishes must be ignored rather + // than panicking or scoring against nothing. + let service = FeedbackService::default(); + + assert!(!service.enabled()); + assert_eq!( + service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + true, + ) + .await, + FeedbackOutcome::Disabled + ); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into()], + &TraceId("trace-1".into()), + None, + ) + .await; + assert_eq!(service.prune().await, 0); + } + + #[tokio::test] + async fn reapplying_settings_takes_effect_without_a_restart() { + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + enabled: false, + ..FeedbackSettings::default() + }, + None, + ); + + assert_eq!( + service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + true, + ) + .await, + FeedbackOutcome::Disabled + ); + } + + #[tokio::test] + async fn pruning_removes_links_past_the_retention_window() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "old".into(), + trace_id: "trace-old".into(), + session_key: None, + created_at: now_unix() - time::Duration::days(60).whole_seconds(), + }) + .await + .unwrap(); + link_reply(&links, "new", "trace-new").await; + + let removed = service.prune().await; + + assert_eq!(removed, 1); + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "new") + .await + .unwrap() + .is_some() + ); + } + + #[tokio::test] + async fn pruning_uses_reconfigured_retention_window() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "middle-aged".into(), + trace_id: "trace-middle-aged".into(), + session_key: None, + created_at: now_unix() - time::Duration::days(60).whole_seconds(), + }) + .await + .unwrap(); + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + link_retention_days: 90, + ..FeedbackSettings::default() + }, + None, + ); + + assert_eq!(service.prune().await, 0); + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "middle-aged") + .await + .unwrap() + .is_some() + ); + } +} diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index 46f16ba8ef..62c6bf9334 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -9,6 +9,7 @@ pub mod commands; pub mod config_view; pub mod contract; pub mod error; +pub mod feedback; pub mod gating; pub mod media_download; pub mod message_log; @@ -17,6 +18,7 @@ pub mod plugin; pub mod registry; pub mod slack_api_url; pub mod store; +pub mod trace_link; pub use { channel_webhook_middleware::{ @@ -25,6 +27,7 @@ pub use { }, config_view::ChannelConfigView, error::{Error, Result}, + feedback::{FeedbackOutcome, FeedbackService}, media_download::{InboundMediaDownloader, InboundMediaSource}, plugin::{ ButtonRow, ButtonStyle, ChannelAttachment, ChannelCapabilities, ChannelDescriptor, @@ -37,4 +40,5 @@ pub use { }, registry::{ChannelRegistry, RegistryOutboundRouter}, slack_api_url::{normalize_slack_api_base_url, validate_slack_api_base_url}, + trace_link::{TraceLink, TraceLinkStore, WEB_CHANNEL}, }; diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index e7212befcf..08d846d11c 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -1,3 +1,8 @@ +#[path = "plugin/channel_type.rs"] +mod channel_type; + +pub use self::channel_type::ChannelType; + use std::sync::Arc; use { @@ -10,291 +15,6 @@ use crate::{Error, Result, config_view::ChannelConfigView}; // ── Channel type enum ─────────────────────────────────────────────────────── -/// Supported channel types. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -#[non_exhaustive] -pub enum ChannelType { - Telegram, - Whatsapp, - #[serde(rename = "msteams")] - MsTeams, - Discord, - Slack, - Matrix, - Nostr, - Signal, - Telephony, -} - -impl ChannelType { - /// Returns the channel type identifier as a string slice. - pub fn as_str(&self) -> &'static str { - match self { - Self::Telegram => "telegram", - Self::Whatsapp => "whatsapp", - Self::MsTeams => "msteams", - Self::Discord => "discord", - Self::Slack => "slack", - Self::Matrix => "matrix", - Self::Nostr => "nostr", - Self::Signal => "signal", - Self::Telephony => "telephony", - } - } - - /// Human-readable display name for UI labels. - pub fn display_name(&self) -> &'static str { - match self { - Self::Telegram => "Telegram", - Self::Whatsapp => "WhatsApp", - Self::MsTeams => "Microsoft Teams", - Self::Discord => "Discord", - Self::Slack => "Slack", - Self::Matrix => "Matrix", - Self::Nostr => "Nostr", - Self::Signal => "Signal", - Self::Telephony => "Phone Call", - } - } - - /// Best-effort chat classification for hook and prompt context. - #[must_use] - pub fn classify_chat(&self, chat_id: &str) -> Option { - match self { - Self::Telegram => { - if chat_id.starts_with("-100") { - Some("channel_or_supergroup".to_string()) - } else if chat_id.starts_with('-') { - Some("group".to_string()) - } else { - Some("private".to_string()) - } - }, - Self::Signal => { - if chat_id.starts_with("group:") { - Some("group".to_string()) - } else { - Some("direct".to_string()) - } - }, - Self::Nostr => Some("dm".to_string()), - Self::Telephony => Some("call".to_string()), - _ => None, - } - } - - /// Top-level config fields that must be treated as persisted secrets. - pub fn secret_fields(&self) -> &'static [&'static str] { - match self { - Self::Telegram => &["token"], - Self::Whatsapp => &[], - Self::MsTeams => &["app_password", "webhook_secret"], - Self::Discord => &["token"], - Self::Slack => &["bot_token", "app_token", "signing_secret"], - Self::Matrix => &["access_token", "password"], - Self::Nostr => &["secret_key"], - Self::Signal => &[], - Self::Telephony => &["auth_token"], - } - } -} - -impl std::fmt::Display for ChannelType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl std::str::FromStr for ChannelType { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - "telegram" => Ok(Self::Telegram), - "whatsapp" => Ok(Self::Whatsapp), - "msteams" | "microsoft_teams" | "microsoft-teams" | "teams" => Ok(Self::MsTeams), - "discord" => Ok(Self::Discord), - "slack" => Ok(Self::Slack), - "matrix" | "element" => Ok(Self::Matrix), - "nostr" => Ok(Self::Nostr), - "signal" => Ok(Self::Signal), - "telephony" | "phone" | "voice_call" | "voicecall" => Ok(Self::Telephony), - other => Err(Error::invalid_input(format!( - "unknown channel type: {other}" - ))), - } - } -} - -impl ChannelType { - /// All known channel types. - pub const ALL: &[ChannelType] = &[ - Self::Telegram, - Self::Whatsapp, - Self::MsTeams, - Self::Discord, - Self::Slack, - Self::Matrix, - Self::Nostr, - Self::Signal, - Self::Telephony, - ]; - - /// Returns the static descriptor for this channel type. - #[must_use] - pub fn descriptor(&self) -> ChannelDescriptor { - match self { - Self::Telegram => ChannelDescriptor { - channel_type: *self, - display_name: "Telegram", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Polling, - supports_outbound: true, - supports_streaming: true, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: true, - }, - }, - Self::Whatsapp => ChannelDescriptor { - channel_type: *self, - display_name: "WhatsApp", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: true, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::MsTeams => ChannelDescriptor { - channel_type: *self, - display_name: "Microsoft Teams", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Webhook, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: true, - }, - }, - Self::Discord => ChannelDescriptor { - channel_type: *self, - display_name: "Discord", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: false, - supports_reactions: false, - supports_location: true, - }, - }, - Self::Slack => ChannelDescriptor { - channel_type: *self, - display_name: "Slack", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::SocketMode, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: false, - }, - }, - Self::Matrix => ChannelDescriptor { - channel_type: *self, - display_name: "Matrix", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: true, - }, - }, - Self::Nostr => ChannelDescriptor { - channel_type: *self, - display_name: "Nostr", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::Signal => ChannelDescriptor { - channel_type: *self, - display_name: "Signal", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::Telephony => ChannelDescriptor { - channel_type: *self, - display_name: "Phone Call", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Webhook, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: false, - supports_reactions: false, - supports_location: false, - }, - }, - } - } -} - -// ── Channel capabilities ────────────────────────────────────────────────── - /// How a channel receives inbound messages. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] #[serde(rename_all = "snake_case")] @@ -904,6 +624,29 @@ pub trait ChannelOutbound: Send + Sync { payload: &ReplyPayload, reply_to: Option<&str>, ) -> Result<()>; + /// Send text and report the ids of the messages it produced. + /// + /// Feedback attribution needs the id of the message the user will react + /// to, and most channel APIs return it from the send call and then throw + /// it away. Returns a list rather than one id because channels split long + /// replies into several messages, and a reader may react to any of them — + /// reporting only the last would leave a thumb on an earlier chunk + /// unattributable. + /// + /// Channels that can report ids override this; the default delegates to + /// [`Self::send_text`] and reports none, so a channel that cannot loses + /// feedback attribution rather than losing the message. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text(account_id, to, text, reply_to).await?; + Ok(Vec::new()) + } + /// Send a "typing" indicator. No-op by default. async fn send_typing(&self, _account_id: &str, _to: &str) -> Result<()> { Ok(()) @@ -922,6 +665,27 @@ pub trait ChannelOutbound: Send + Sync { let _ = suffix_html; self.send_text(account_id, to, text, reply_to).await } + + /// [`Self::send_text_with_suffix`], reporting the ids of the messages it + /// produced. + /// + /// A reply that carries an activity logbook is still a reply someone can + /// react to, so it needs the same attribution as a plain one. Kept as a + /// separate method for the same reason as + /// [`Self::send_text_reporting_ids`]: channels that cannot report ids + /// inherit the default and lose attribution, not the message. + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text_with_suffix(account_id, to, text, suffix_html, reply_to) + .await?; + Ok(Vec::new()) + } /// Send pre-formatted HTML without markdown conversion. /// /// Used for content that is already valid Telegram HTML (e.g. the activity @@ -935,6 +699,22 @@ pub trait ChannelOutbound: Send + Sync { ) -> Result<()> { self.send_text(account_id, to, html, reply_to).await } + + /// [`Self::send_html`], reporting the ids of the messages it produced. + /// + /// The activity logbook that follows a streamed reply is delivered this + /// way. It belongs to the same turn, so a reaction on it should score that + /// turn rather than resolve to nothing. + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_html(account_id, to, html, reply_to).await?; + Ok(Vec::new()) + } /// Send a text message without notification (silent). Falls back to send_text by default. async fn send_text_silent( &self, @@ -1055,6 +835,25 @@ pub trait ChannelStreamOutbound: Send + Sync { stream: StreamReceiver, ) -> Result<()>; + /// [`Self::send_stream`], reporting the ids of the messages it left behind. + /// + /// Edit-in-place streaming delivers the final reply itself, so the normal + /// send path never runs and never records a trace link. Without this the + /// message a reader actually reacts to would have no attribution at all. + /// + /// Channels that cannot report ids inherit the default and lose feedback + /// attribution for streamed replies, not the reply. + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(Vec::new()) + } + /// Whether streaming is enabled for this account. async fn is_stream_enabled(&self, _account_id: &str) -> bool { true diff --git a/crates/channels/src/plugin/channel_type.rs b/crates/channels/src/plugin/channel_type.rs new file mode 100644 index 0000000000..e436d19a3b --- /dev/null +++ b/crates/channels/src/plugin/channel_type.rs @@ -0,0 +1,294 @@ +//! The channel type enum and its conversions. +//! +//! Split out of `plugin` to keep both files inside the file-size limit; +//! `ChannelType` and its parsing are a self-contained concern. + +use crate::{ + Error, + plugin::{ChannelCapabilities, ChannelDescriptor, InboundMode}, +}; + +/// Supported channel types. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +#[non_exhaustive] +pub enum ChannelType { + Telegram, + Whatsapp, + #[serde(rename = "msteams")] + MsTeams, + Discord, + Slack, + Matrix, + Nostr, + Signal, + Telephony, +} + +impl ChannelType { + /// Returns the channel type identifier as a string slice. + pub fn as_str(&self) -> &'static str { + match self { + Self::Telegram => "telegram", + Self::Whatsapp => "whatsapp", + Self::MsTeams => "msteams", + Self::Discord => "discord", + Self::Slack => "slack", + Self::Matrix => "matrix", + Self::Nostr => "nostr", + Self::Signal => "signal", + Self::Telephony => "telephony", + } + } + + /// Human-readable display name for UI labels. + pub fn display_name(&self) -> &'static str { + match self { + Self::Telegram => "Telegram", + Self::Whatsapp => "WhatsApp", + Self::MsTeams => "Microsoft Teams", + Self::Discord => "Discord", + Self::Slack => "Slack", + Self::Matrix => "Matrix", + Self::Nostr => "Nostr", + Self::Signal => "Signal", + Self::Telephony => "Phone Call", + } + } + + /// Best-effort chat classification for hook and prompt context. + #[must_use] + pub fn classify_chat(&self, chat_id: &str) -> Option { + match self { + Self::Telegram => { + if chat_id.starts_with("-100") { + Some("channel_or_supergroup".to_string()) + } else if chat_id.starts_with('-') { + Some("group".to_string()) + } else { + Some("private".to_string()) + } + }, + Self::Signal => { + if chat_id.starts_with("group:") { + Some("group".to_string()) + } else { + Some("direct".to_string()) + } + }, + Self::Nostr => Some("dm".to_string()), + Self::Telephony => Some("call".to_string()), + _ => None, + } + } + + /// Top-level config fields that must be treated as persisted secrets. + pub fn secret_fields(&self) -> &'static [&'static str] { + match self { + Self::Telegram => &["token"], + Self::Whatsapp => &[], + Self::MsTeams => &["app_password", "webhook_secret"], + Self::Discord => &["token"], + Self::Slack => &["bot_token", "app_token", "signing_secret"], + Self::Matrix => &["access_token", "password"], + Self::Nostr => &["secret_key"], + Self::Signal => &[], + Self::Telephony => &["auth_token"], + } + } +} + +impl std::fmt::Display for ChannelType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for ChannelType { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "telegram" => Ok(Self::Telegram), + "whatsapp" => Ok(Self::Whatsapp), + "msteams" | "microsoft_teams" | "microsoft-teams" | "teams" => Ok(Self::MsTeams), + "discord" => Ok(Self::Discord), + "slack" => Ok(Self::Slack), + "matrix" | "element" => Ok(Self::Matrix), + "nostr" => Ok(Self::Nostr), + "signal" => Ok(Self::Signal), + "telephony" | "phone" | "voice_call" | "voicecall" => Ok(Self::Telephony), + other => Err(Error::invalid_input(format!( + "unknown channel type: {other}" + ))), + } + } +} + +impl ChannelType { + /// All known channel types. + pub const ALL: &[ChannelType] = &[ + Self::Telegram, + Self::Whatsapp, + Self::MsTeams, + Self::Discord, + Self::Slack, + Self::Matrix, + Self::Nostr, + Self::Signal, + Self::Telephony, + ]; + + /// Returns the static descriptor for this channel type. + #[must_use] + pub fn descriptor(&self) -> ChannelDescriptor { + match self { + Self::Telegram => ChannelDescriptor { + channel_type: *self, + display_name: "Telegram", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Polling, + supports_outbound: true, + supports_streaming: true, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: true, + }, + }, + Self::Whatsapp => ChannelDescriptor { + channel_type: *self, + display_name: "WhatsApp", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: true, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::MsTeams => ChannelDescriptor { + channel_type: *self, + display_name: "Microsoft Teams", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Webhook, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: true, + }, + }, + Self::Discord => ChannelDescriptor { + channel_type: *self, + display_name: "Discord", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: false, + supports_reactions: false, + supports_location: true, + }, + }, + Self::Slack => ChannelDescriptor { + channel_type: *self, + display_name: "Slack", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::SocketMode, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: false, + }, + }, + Self::Matrix => ChannelDescriptor { + channel_type: *self, + display_name: "Matrix", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: true, + }, + }, + Self::Nostr => ChannelDescriptor { + channel_type: *self, + display_name: "Nostr", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::Signal => ChannelDescriptor { + channel_type: *self, + display_name: "Signal", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::Telephony => ChannelDescriptor { + channel_type: *self, + display_name: "Phone Call", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Webhook, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: false, + supports_reactions: false, + supports_location: false, + }, + }, + } + } +} + +// ── Channel capabilities ────────────────────────────────────────────────── diff --git a/crates/channels/src/registry.rs b/crates/channels/src/registry.rs index 0fc8d5a69e..0d208f1dd7 100644 --- a/crates/channels/src/registry.rs +++ b/crates/channels/src/registry.rs @@ -333,6 +333,26 @@ impl ChannelOutbound for RegistryOutboundRouter { outbound.send_text(account_id, to, text, reply_to).await } + /// Delegate rather than inherit the trait default: the default reports no + /// ids, which would silently strip attribution from every channel behind + /// this facade even though the concrete outbound can report them. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_text_reporting_ids(account_id, to, text, reply_to) + .await + } + async fn send_media( &self, account_id: &str, @@ -375,6 +395,24 @@ impl ChannelOutbound for RegistryOutboundRouter { .await } + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_text_with_suffix_reporting_ids(account_id, to, text, suffix_html, reply_to) + .await + } + async fn send_html( &self, account_id: &str, @@ -390,6 +428,23 @@ impl ChannelOutbound for RegistryOutboundRouter { outbound.send_html(account_id, to, html, reply_to).await } + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_html_reporting_ids(account_id, to, html, reply_to) + .await + } + async fn send_text_silent( &self, account_id: &str, @@ -497,6 +552,23 @@ impl ChannelStreamOutbound for RegistryOutboundRouter { .await } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + let stream_out = self + .registry + .resolve_stream(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + stream_out + .send_stream_reporting_ids(account_id, to, reply_to, stream) + .await + } + async fn is_stream_enabled(&self, account_id: &str) -> bool { let Some(stream_out) = self.registry.resolve_stream(account_id).await else { return false; @@ -692,6 +764,29 @@ mod tests { ) -> Result<()> { Ok(()) } + + // Sentinel ids: the trait defaults report none, so a router that + // inherits them instead of delegating produces an empty list. + async fn send_text_reporting_ids( + &self, + _: &str, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result> { + Ok(vec!["text-1".into()]) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + _: &str, + _: &str, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result> { + Ok(vec!["suffix-1".into()]) + } } struct NullStreamOutbound { @@ -716,6 +811,17 @@ mod tests { Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(vec!["stream-1".into()]) + } + async fn streams_final_replies(&self, _: &str) -> bool { self.streams_final_replies } @@ -866,6 +972,51 @@ mod tests { assert!(result.is_ok()); } + /// The id-reporting sends must be delegated, not inherited. + /// + /// Every channel reaches the outbound trait through this router, so a + /// router that fell back to the trait defaults would report no ids for any + /// channel and silently strip feedback attribution from the whole system. + #[tokio::test] + async fn outbound_router_delegates_id_reporting_sends() { + let mut registry = ChannelRegistry::new(); + registry + .register(Arc::new(RwLock::new(TestPlugin::new("telegram")))) + .await; + registry + .start_account("telegram", "bot1", serde_json::json!({})) + .await + .unwrap(); + + let router = RegistryOutboundRouter::new(Arc::new(registry)); + + assert_eq!( + router + .send_text_reporting_ids("bot1", "42", "hello", None) + .await + .unwrap(), + vec!["text-1".to_string()] + ); + assert_eq!( + router + .send_text_with_suffix_reporting_ids("bot1", "42", "hello", "log", None) + .await + .unwrap(), + vec!["suffix-1".to_string()] + ); + + let (tx, rx) = mpsc::channel(8); + tx.send(StreamEvent::Done).await.unwrap(); + drop(tx); + assert_eq!( + router + .send_stream_reporting_ids("bot1", "42", None, rx) + .await + .unwrap(), + vec!["stream-1".to_string()] + ); + } + #[tokio::test] async fn outbound_router_unknown_account_errors() { let registry = Arc::new(ChannelRegistry::new()); diff --git a/crates/channels/src/trace_link.rs b/crates/channels/src/trace_link.rs new file mode 100644 index 0000000000..d15fca8835 --- /dev/null +++ b/crates/channels/src/trace_link.rs @@ -0,0 +1,83 @@ +//! Correlation between a delivered reply and the trace that produced it. +//! +//! A reaction arrives naming a message, not a turn. Attributing it to the +//! agent run that wrote that message needs a record made at send time, because +//! by the time someone reacts the turn is long finished and nothing else in the +//! system remembers which trace a given chat message came from. +//! +//! Deliberately holds identifiers only — no message body. The link table exists +//! to answer "which trace produced this message id", and storing the text again +//! would duplicate content the message log already owns. + +use async_trait::async_trait; + +use crate::Result; + +/// Channel type recorded for replies delivered through the web UI. +pub const WEB_CHANNEL: &str = "web"; + +/// A delivered reply and the trace behind it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceLink { + /// Channel the reply went out on, e.g. `telegram`. [`WEB_CHANNEL`] for the + /// web UI. + pub channel_type: String, + /// Account that sent the reply. + pub account_id: String, + /// Conversation the reply landed in. + pub chat_id: String, + /// The reply's own id in the channel's namespace. This is what a reaction + /// event names. + pub message_id: String, + /// Trace that produced the reply. + pub trace_id: String, + /// Session the turn belonged to. + pub session_key: Option, + /// Unix seconds when the link was recorded. + pub created_at: i64, +} + +/// Store for reply/trace correlation. +#[async_trait] +pub trait TraceLinkStore: Send + Sync { + /// Record a link, replacing any existing link for the same message. + async fn link(&self, link: TraceLink) -> Result<()>; + + /// Find the trace behind a message, if it is still on record. + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> Result>; + + /// Drop links recorded before `cutoff` (unix seconds), returning how many + /// were removed. Without this the table grows for the life of the install. + async fn prune(&self, cutoff: i64) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_link_is_identifiers_only() { + // Guards against a body field creeping in: the message log already owns + // message content, and a second copy here would widen what a feedback + // feature retains without anyone deciding to. + let link = TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "42".into(), + trace_id: "trace-1".into(), + session_key: Some("agent:main:main".into()), + created_at: 1_700_000_000, + }; + + let rendered = format!("{link:?}"); + assert!(rendered.contains("trace-1")); + assert!(!rendered.contains("body")); + } +} diff --git a/crates/chat/Cargo.toml b/crates/chat/Cargo.toml index b2d8b4c5b5..ddc8da53b4 100644 --- a/crates/chat/Cargo.toml +++ b/crates/chat/Cargo.toml @@ -16,6 +16,7 @@ moltis-agents = { workspace = true } moltis-channels = { workspace = true } moltis-common = { workspace = true } moltis-config = { workspace = true } +moltis-observability = { workspace = true } moltis-memory = { workspace = true } moltis-metrics = { optional = true, workspace = true } moltis-plugins = { workspace = true } diff --git a/crates/chat/src/agent_loop.rs b/crates/chat/src/agent_loop.rs index 0f62e43a2b..fa828ecefb 100644 --- a/crates/chat/src/agent_loop.rs +++ b/crates/chat/src/agent_loop.rs @@ -163,6 +163,13 @@ pub(crate) struct ChannelStreamDispatcher { workers: Vec, tasks: Vec>, completed: Arc>>, + /// Ids of the messages each stream left behind, so a reaction on a + /// streamed reply is attributable. The normal send path never runs for + /// these targets, so this is the only place the link can be recorded. + delivered: Arc)>>>, + feedback: Option>, + session_key: String, + trace_correlation_key: String, started: bool, sent_final_delta: bool, } @@ -171,6 +178,7 @@ impl ChannelStreamDispatcher { pub(crate) async fn for_session( state: &Arc, session_key: &str, + trace_correlation_key: &str, ) -> Option { let outbound = state.channel_stream_outbound()?; let targets: Vec = state @@ -187,6 +195,10 @@ impl ChannelStreamDispatcher { workers: Vec::new(), tasks: Vec::new(), completed: Arc::new(Mutex::new(HashSet::new())), + delivered: Arc::new(Mutex::new(Vec::new())), + feedback: state.feedback(), + session_key: session_key.to_string(), + trace_correlation_key: trace_correlation_key.to_string(), started: false, sent_final_delta: false, }; @@ -222,10 +234,12 @@ impl ChannelStreamDispatcher { let (tx, rx) = mpsc::channel(CHANNEL_STREAM_BUFFER_SIZE); let outbound = Arc::clone(&self.outbound); let completed = Arc::clone(&self.completed); + let delivered = Arc::clone(&self.delivered); let account_id = target.account_id.clone(); let to = target.outbound_to().into_owned(); let reply_to = target.message_id.clone(); let key_for_insert = key.clone(); + let target_for_link = target.clone(); let account_for_log = account_id.clone(); let chat_for_log = target.chat_id.clone(); let thread_for_log = target.thread_id.clone(); @@ -236,13 +250,16 @@ impl ChannelStreamDispatcher { }); self.tasks.push(tokio::spawn(async move { match outbound - .send_stream(&account_id, &to, reply_to.as_deref(), rx) + .send_stream_reporting_ids(&account_id, &to, reply_to.as_deref(), rx) .await { - Ok(()) => { + Ok(message_ids) => { if streams_final_replies { completed.lock().await.insert(key_for_insert); } + if !message_ids.is_empty() { + delivered.lock().await.push((target_for_link, message_ids)); + } }, Err(e) => { warn!( @@ -294,6 +311,25 @@ impl ChannelStreamDispatcher { pub(crate) async fn finish(&mut self) { self.send_terminal(moltis_channels::StreamEvent::Done).await; self.join_workers().await; + self.record_delivered_trace_links().await; + } + + /// Attribute the streamed messages to the trace that produced them. + /// + /// Runs after the workers are joined so every id is in hand, and before + /// the caller decides which targets to skip on the normal send path. + async fn record_delivered_trace_links(&self) { + let delivered = std::mem::take(&mut *self.delivered.lock().await); + for (target, message_ids) in delivered { + crate::channel_feedback::record_reply_trace( + self.feedback.as_deref(), + &target, + &message_ids, + &self.session_key, + &self.trace_correlation_key, + ) + .await; + } } async fn send_terminal(&mut self, event: moltis_channels::StreamEvent) { @@ -498,6 +534,7 @@ pub(crate) async fn run_explicit_shell_command( deliver_channel_replies( state, session_key, + run_id, &final_text, ReplyMedium::Text, &streamed_target_keys, @@ -627,6 +664,19 @@ mod tests { Ok(()) } + /// Report an id the way a real edit-in-place channel does, so the + /// dispatcher's trace-link recording is exercised rather than skipped. + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: moltis_channels::StreamReceiver, + ) -> moltis_channels::Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(vec!["streamed-1".into()]) + } + async fn is_stream_enabled(&self, _account_id: &str) -> bool { true } @@ -658,6 +708,10 @@ mod tests { workers: Vec::new(), tasks: Vec::new(), completed: Arc::new(Mutex::new(HashSet::new())), + delivered: Arc::new(Mutex::new(Vec::new())), + feedback: None, + session_key: "session".into(), + trace_correlation_key: "run".into(), started: false, sent_final_delta: false, } @@ -694,6 +748,79 @@ mod tests { assert!(dispatcher.completed_target_keys().await.is_empty()); } + /// Edit-in-place streaming delivers the reply itself, so the normal send + /// path never runs for that target. If the dispatcher did not record the + /// link, a reaction on a streamed reply would resolve as `UnknownMessage`. + #[tokio::test] + async fn streamed_replies_get_a_trace_link() { + use { + async_trait::async_trait as link_async_trait, + moltis_channels::trace_link::{TraceLink, TraceLinkStore}, + }; + + #[derive(Default)] + struct RecordingLinks { + links: std::sync::Mutex>, + } + + #[link_async_trait] + impl TraceLinkStore for RecordingLinks { + async fn link(&self, link: TraceLink) -> moltis_channels::Result<()> { + self.links + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(link); + Ok(()) + } + + async fn lookup( + &self, + _channel_type: &str, + _account_id: &str, + _chat_id: &str, + _message_id: &str, + ) -> moltis_channels::Result> { + Ok(None) + } + + async fn prune(&self, _cutoff: i64) -> moltis_channels::Result { + Ok(0) + } + } + + let session_key = "streamed-link-test-session"; + let links = Arc::new(RecordingLinks::default()); + let feedback = Arc::new(moltis_channels::FeedbackService::default()); + feedback.apply( + Arc::clone(&links) as Arc, + &moltis_config::FeedbackSettings::default(), + None, + ); + moltis_observability::remember_trace( + session_key, + &moltis_observability::TraceId("trace-streamed".into()), + ); + + let outbound = Arc::new(RecordingStreamOutbound { + streams_final_replies: true, + receives_progress_deltas: false, + events: Arc::new(Mutex::new(Vec::new())), + }); + let mut dispatcher = dispatcher_with(outbound); + dispatcher.feedback = Some(feedback); + dispatcher.session_key = session_key.to_string(); + dispatcher.trace_correlation_key = session_key.to_string(); + + dispatcher.send_delta("final").await; + dispatcher.finish().await; + + let recorded = links.links.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].message_id, "streamed-1"); + assert_eq!(recorded[0].chat_id, "chat"); + assert_eq!(recorded[0].trace_id, "trace-streamed"); + } + #[tokio::test] async fn final_stream_workers_receive_progress_and_final_deltas_and_complete_targets() { let events = Arc::new(Mutex::new(Vec::new())); diff --git a/crates/chat/src/channel_compaction.rs b/crates/chat/src/channel_compaction.rs new file mode 100644 index 0000000000..530b1921a8 --- /dev/null +++ b/crates/chat/src/channel_compaction.rs @@ -0,0 +1,101 @@ +//! Channel notices for session compaction. +//! +//! Split from the main channel delivery module to keep both inside the +//! file-size limit; the compaction notice is a self-contained concern. + +use {std::sync::Arc, tracing::warn}; + +use crate::{compaction_run, runtime::ChatRuntime}; + +fn format_channel_compaction_notice( + outcome: &compaction_run::CompactionOutcome, + include_settings_hint: bool, +) -> String { + let mode_label = match outcome.effective_mode { + moltis_config::CompactionMode::Deterministic => "Deterministic", + moltis_config::CompactionMode::RecencyPreserving => "Recency preserving", + moltis_config::CompactionMode::Structured => "Structured", + moltis_config::CompactionMode::LlmReplace => "LLM replace", + }; + let total = outcome.total_tokens(); + let token_line = if total == 0 { + // Any strategy that made no LLM calls ends up here: Deterministic, + // RecencyPreserving, or a Structured run that fell back to + // recency_preserving before the LLM call landed. Report the + // actual effective mode so users don't see "deterministic + // strategy" when they picked recency_preserving. + format!( + "No LLM tokens used ({} strategy)", + mode_label.to_lowercase() + ) + } else { + format!( + "Used {total} tokens ({input} in + {output} out)", + total = total, + input = outcome.input_tokens, + output = outcome.output_tokens, + ) + }; + let body = format!( + "🧹 Conversation compacted\n\ + Mode: {mode_label}\n\ + {token_line}", + ); + if include_settings_hint { + format!("{body}\n{hint}", hint = compaction_run::SETTINGS_HINT) + } else { + body + } +} + +/// Send a silent "session compacted" notice to pending channel targets +/// without draining them. +/// +/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, +/// not drained, so the in-flight agent run can still deliver its final +/// reply to them afterward. Uses `send_text_silent` so the channel +/// integration doesn't count it toward user-visible interactive replies +/// (no TTS, no delivery receipts beyond the channel's own). +pub(crate) async fn notify_channels_of_compaction( + state: &Arc, + session_key: &str, + outcome: &compaction_run::CompactionOutcome, + include_settings_hint: bool, +) { + let targets = state.peek_channel_replies(session_key).await; + if targets.is_empty() { + return; + } + + let Some(outbound) = state.channel_outbound() else { + return; + }; + + let message = format_channel_compaction_notice(outcome, include_settings_hint); + let mut tasks = Vec::with_capacity(targets.len()); + for target in targets { + let outbound = Arc::clone(&outbound); + let message = message.clone(); + let to = target.outbound_to().into_owned(); + tasks.push(tokio::spawn(async move { + let reply_to = target.message_id.as_deref(); + if let Err(e) = outbound + .send_text_silent(&target.account_id, &to, &message, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send compaction notice to channel: {e}" + ); + } + })); + } + + for task in tasks { + if let Err(e) = task.await { + warn!(error = %e, "channel compaction notice task join failed"); + } + } +} diff --git a/crates/chat/src/channel_feedback.rs b/crates/chat/src/channel_feedback.rs new file mode 100644 index 0000000000..01d9bba3a5 --- /dev/null +++ b/crates/chat/src/channel_feedback.rs @@ -0,0 +1,181 @@ +//! Correlating delivered replies with the traces that produced them. +//! +//! Feedback attribution is recorded at send time because nothing else in the +//! system remembers which agent run wrote a given chat message, and by the +//! time someone reacts the turn is long finished. + +use std::sync::Arc; + +use crate::runtime::ChatRuntime; + +/// Link a web-delivered assistant message to the trace that produced it. +/// +/// The web is treated as one more channel so a thumb in the browser resolves +/// through the same correlation table as a Telegram reaction. The run id is +/// the message identity: unlike a message index it does not shift when older +/// history is compacted away. +pub(crate) async fn record_web_reply_trace( + state: &Arc, + session_key: &str, + run_id: &str, +) { + let Some(feedback) = state.feedback() else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(run_id) else { + return; + }; + feedback + .record_reply( + moltis_channels::trace_link::WEB_CHANNEL, + moltis_channels::trace_link::WEB_CHANNEL, + session_key, + std::slice::from_ref(&run_id.to_string()), + &trace_id, + Some(session_key), + ) + .await; +} + +/// Link the messages a reply produced to the trace that generated it. +/// +/// Keyed on [`ChannelReplyTarget::chat_id`] rather than the outbound address: +/// for a Telegram forum topic the outbound address carries a `:thread` suffix, +/// but a `message_reaction` update reports only the bare chat id, so a link +/// stored under the suffixed form is one no reaction could ever find. +/// +/// Best-effort by design: a missing link costs feedback attribution for one +/// reply, and the user already has the message. +/// +/// [`ChannelReplyTarget::chat_id`]: moltis_channels::ChannelReplyTarget::chat_id +pub(crate) async fn record_reply_trace( + feedback: Option<&moltis_channels::FeedbackService>, + target: &moltis_channels::ChannelReplyTarget, + message_ids: &[String], + session_key: &str, + trace_correlation_key: &str, +) { + if message_ids.is_empty() { + return; + } + let Some(feedback) = feedback else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(trace_correlation_key) else { + return; + }; + feedback + .record_reply( + target.channel_type.as_str(), + &target.account_id, + &target.chat_id, + message_ids, + &trace_id, + Some(session_key), + ) + .await; +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use std::sync::Mutex; + + use { + async_trait::async_trait, + moltis_channels::trace_link::{TraceLink, TraceLinkStore}, + moltis_config::FeedbackSettings, + moltis_observability::TraceId, + }; + + use super::*; + + #[derive(Default)] + struct RecordingLinks { + links: Mutex>, + } + + #[async_trait] + impl TraceLinkStore for RecordingLinks { + async fn link(&self, link: TraceLink) -> moltis_channels::Result<()> { + self.links.lock().unwrap().push(link); + Ok(()) + } + + async fn lookup( + &self, + _channel_type: &str, + _account_id: &str, + _chat_id: &str, + _message_id: &str, + ) -> moltis_channels::Result> { + Ok(None) + } + + async fn prune(&self, _cutoff: i64) -> moltis_channels::Result { + Ok(0) + } + } + + fn telegram_topic_target() -> moltis_channels::ChannelReplyTarget { + moltis_channels::ChannelReplyTarget { + ack_message_id: None, + channel_type: moltis_channels::ChannelType::Telegram, + account_id: "bot1".into(), + chat_id: "-1001234".into(), + message_id: Some("77".into()), + thread_id: Some("42".into()), + } + } + + /// A reply into a Telegram forum topic goes out to `chat:thread`, but the + /// `message_reaction` update that follows reports only `chat`. Storing the + /// composite address would make every topic reaction unresolvable. + #[tokio::test] + async fn topic_replies_are_keyed_on_the_bare_chat_id() { + let session_key = "topic-key-test-session"; + let links = Arc::new(RecordingLinks::default()); + let feedback = moltis_channels::FeedbackService::default(); + feedback.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings::default(), + None, + ); + moltis_observability::remember_trace(session_key, &TraceId("trace-abc".into())); + + let target = telegram_topic_target(); + assert_eq!(target.outbound_to().as_ref(), "-1001234:42"); + + record_reply_trace( + Some(&feedback), + &target, + &["901".to_string(), "902".to_string()], + session_key, + session_key, + ) + .await; + + let recorded = links.links.lock().unwrap(); + assert_eq!(recorded.len(), 2); + for link in recorded.iter() { + assert_eq!(link.chat_id, "-1001234"); + assert_eq!(link.account_id, "bot1"); + assert_eq!(link.channel_type, "telegram"); + assert_eq!(link.trace_id, "trace-abc"); + } + assert_eq!(recorded[0].message_id, "901"); + assert_eq!(recorded[1].message_id, "902"); + } + + #[tokio::test] + async fn nothing_is_recorded_without_a_feedback_service() { + record_reply_trace( + None, + &telegram_topic_target(), + &["1".to_string()], + "none", + "none", + ) + .await; + } +} diff --git a/crates/chat/src/channel_reply_delivery.rs b/crates/chat/src/channel_reply_delivery.rs new file mode 100644 index 0000000000..e3171bd103 --- /dev/null +++ b/crates/chat/src/channel_reply_delivery.rs @@ -0,0 +1,579 @@ +//! Fanning a finished reply out to the channels that asked for it. +//! +//! Split from the main channel module to stay inside the file-size limit; the +//! reply fan-out is a self-contained concern, and keeping it together is what +//! makes the "every delivery path records a trace link" rule checkable by +//! reading one file. + +use std::{collections::HashSet, sync::Arc}; + +use tracing::warn; + +use crate::{ + agent_loop::ChannelReplyTargetKey, + channels::{build_tts_payload, format_logbook_html}, + runtime::ChatRuntime, + types::ReplyMedium, +}; + +/// Send the activity logbook as a follow-up message and link it to the trace. +/// +/// Reached from every branch where the answer itself went out some other way — +/// as voice audio, or already streamed edit-in-place — which leaves this as the +/// only text message of the turn and therefore a likely reaction target. Goes +/// through the id-reporting send for the same reason +/// [`deliver_text_reply`] does: the non-reporting one silently loses +/// attribution. +/// +/// A no-op when there is no logbook, so callers need no emptiness check. +async fn deliver_logbook_follow_up( + outbound: &Arc, + feedback: Option<&moltis_channels::FeedbackService>, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + logbook_html: &str, + session_key: &str, + trace_correlation_key: &str, +) { + if logbook_html.is_empty() { + return; + } + match outbound + .send_html_reporting_ids(&target.account_id, to, logbook_html, None) + .await + { + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback, + target, + &message_ids, + session_key, + trace_correlation_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + }, + } +} + +/// Send the reply text (optionally carrying an activity logbook) and link the +/// messages it produced to the trace that wrote them. +/// +/// One helper for both the Telegram and generic branches so a delivery path +/// cannot quietly lose feedback attribution by picking the non-reporting send: +/// the reporting variants are the only ones reachable from here. Channels that +/// cannot report ids return an empty list and lose attribution, not the reply. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn deliver_text_reply( + outbound: &Arc, + feedback: Option<&moltis_channels::FeedbackService>, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + text: &str, + logbook_html: &str, + reply_to: Option<&str>, + session_key: &str, + trace_correlation_key: &str, +) { + let result = if logbook_html.is_empty() { + outbound + .send_text_reporting_ids(&target.account_id, to, text, reply_to) + .await + } else { + outbound + .send_text_with_suffix_reporting_ids( + &target.account_id, + to, + text, + logbook_html, + reply_to, + ) + .await + }; + match result { + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback, + target, + &message_ids, + session_key, + trace_correlation_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel reply: {e}" + ); + }, + } +} + +pub(crate) async fn deliver_channel_replies_to_targets( + outbound: Arc, + targets: Vec, + session_key: &str, + trace_correlation_key: &str, + text: &str, + state: Arc, + desired_reply_medium: ReplyMedium, + status_log: Vec, + streamed_target_keys: &HashSet, +) { + let session_key = session_key.to_string(); + let trace_correlation_key = trace_correlation_key.to_string(); + let text = text.to_string(); + let logbook_html = format_logbook_html(&status_log); + // Resolved once rather than per target: it is the same service either way. + let feedback = state.feedback(); + let mut tasks = Vec::with_capacity(targets.len()); + for target in targets { + let outbound = Arc::clone(&outbound); + let state = Arc::clone(&state); + let feedback = feedback.clone(); + let session_key = session_key.clone(); + let trace_correlation_key = trace_correlation_key.clone(); + let text = text.clone(); + let logbook_html = logbook_html.clone(); + // Text was already delivered via edit-in-place streaming — skip text + // caption/follow-up and only send the TTS voice audio. + let text_already_streamed = + streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); + let to = target.outbound_to().into_owned(); + tasks.push(tokio::spawn(async move { + let tts_payload = match desired_reply_medium { + ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, + ReplyMedium::Text => None, + }; + let reply_to = target.message_id.as_deref(); + match target.channel_type { + moltis_channels::ChannelType::Telegram => match tts_payload { + Some(mut payload) => { + let transcript = std::mem::take(&mut payload.text); + + if text_already_streamed { + // Text was already streamed — send voice audio only. + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + &trace_correlation_key, + ) + .await; + } else { + // Check if transcript fits as Telegram caption (when feature enabled). + // When telegram feature is disabled, this evaluates to false and we + // send voice + follow-up text. + #[cfg(feature = "telegram")] + let fits_in_caption = transcript.len() + <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; + #[cfg(not(feature = "telegram"))] + let fits_in_caption = false; + + if fits_in_caption { + // Short transcript fits as a caption on the voice message. + payload.text = transcript; + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + &trace_correlation_key, + ) + .await; + } else { + // Transcript too long for a caption — send voice + // without caption, then the full text as a follow-up. + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + // The transcript follow-up is the readable form + // of this turn's answer, so it is as much a + // reaction target as a plain text reply and + // goes out through the same attributing path. + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &transcript, + &logbook_html, + None, + &session_key, + &trace_correlation_key, + ) + .await; + } + } + }, + None if text_already_streamed => { + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + &trace_correlation_key, + ) + .await; + }, + None => { + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + &session_key, + &trace_correlation_key, + ) + .await; + }, + }, + _ => match tts_payload { + Some(payload) => { + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + }, + None if text_already_streamed => { + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + &trace_correlation_key, + ) + .await; + }, + None => { + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + &session_key, + &trace_correlation_key, + ) + .await; + }, + }, + } + })); + } + + for task in tasks { + if let Err(e) = task.await { + warn!(error = %e, "channel reply task join failed"); + } + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + async_trait::async_trait, + moltis_channels::{ChannelReplyTarget, ChannelType}, + moltis_common::types::ReplyPayload, + std::sync::Mutex, + }; + + /// Records which send method was used and hands back ids, so a test can + /// tell an attributable delivery from one that silently drops the ids. + #[derive(Default)] + struct ReportingOutbound { + calls: Mutex>, + } + + #[async_trait] + impl moltis_channels::ChannelOutbound for ReportingOutbound { + async fn send_text( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text"); + Ok(()) + } + + async fn send_media( + &self, + _account_id: &str, + _to: &str, + _payload: &ReplyPayload, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + Ok(()) + } + + async fn send_text_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_reporting_ids"); + Ok(vec!["plain-1".into()]) + } + + async fn send_text_with_suffix( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _suffix_html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_with_suffix"); + Ok(()) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _suffix_html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_with_suffix_reporting_ids"); + Ok(vec!["logbook-1".into()]) + } + + async fn send_html( + &self, + _account_id: &str, + _to: &str, + _html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_html"); + Ok(()) + } + + async fn send_html_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_html_reporting_ids"); + Ok(vec!["html-1".into()]) + } + } + + fn reply_target() -> ChannelReplyTarget { + ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Discord, + account_id: "acct".into(), + chat_id: "chan".into(), + message_id: None, + thread_id: None, + } + } + + /// A reply carrying an activity logbook is still a reply someone can react + /// to, so it must go out through the id-reporting suffix send rather than + /// the plain one, which reports nothing and loses attribution. + /// The logbook follow-up is the only text message on the branches that + /// reach it, so it must go out through the id-reporting send. `send_html` + /// reports nothing and would leave the reaction unresolvable. + #[tokio::test] + async fn a_logbook_follow_up_uses_the_id_reporting_html_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_logbook_follow_up( + &outbound, + None, + &reply_target(), + "chan", + "
log
", + "session", + "run", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_html_reporting_ids"]); + } + + #[tokio::test] + async fn an_absent_logbook_sends_nothing() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_logbook_follow_up( + &outbound, + None, + &reply_target(), + "chan", + "", + "session", + "run", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert!(calls.is_empty(), "expected no send, got {calls:?}"); + } + + /// Guards the invariant the whole module exists to hold: no delivery path + /// may reach for a send that drops message ids. A new branch that calls + /// `send_text`, `send_text_with_suffix` or `send_html` directly loses + /// feedback attribution silently, which is exactly the class of bug this + /// file kept reintroducing. + #[test] + fn no_delivery_branch_uses_a_non_reporting_send() { + let source = include_str!("channel_reply_delivery.rs"); + let body = source + .split("#[cfg(test)]") + .next() + .unwrap_or_default() + .to_string(); + for forbidden in [".send_text(", ".send_text_with_suffix(", ".send_html("] { + assert!( + !body.contains(forbidden), + "delivery code calls {forbidden} — use the *_reporting_ids variant \ + so the reply keeps its trace link" + ); + } + } + + #[tokio::test] + async fn a_logbook_reply_uses_the_id_reporting_suffix_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_text_reply( + &outbound, + None, + &reply_target(), + "chan", + "the answer", + "
log
", + None, + "session", + "run", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_text_with_suffix_reporting_ids"]); + } + + #[tokio::test] + async fn a_plain_reply_uses_the_id_reporting_text_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_text_reply( + &outbound, + None, + &reply_target(), + "chan", + "the answer", + "", + None, + "session", + "run", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_text_reporting_ids"]); + } +} diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 378471b1f8..8883c9cf4f 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -12,9 +12,7 @@ use { use moltis_sessions::store::SessionStore; -use crate::{ - agent_loop::ChannelReplyTargetKey, compaction_run, error, runtime::ChatRuntime, types::*, -}; +use crate::{agent_loop::ChannelReplyTargetKey, error, runtime::ChatRuntime, types::*}; /// Drain any pending channel reply targets for a session and send the /// response text back to each originating channel via outbound. @@ -23,6 +21,7 @@ use crate::{ pub(crate) async fn deliver_channel_replies( state: &Arc, session_key: &str, + trace_correlation_key: &str, text: &str, desired_reply_medium: ReplyMedium, streamed_target_keys: &HashSet, @@ -100,6 +99,9 @@ pub(crate) async fn deliver_channel_replies( Arc::clone(&outbound), streamed_targets, &logbook_html, + state.feedback(), + session_key, + trace_correlation_key, ) .await; } @@ -114,10 +116,11 @@ pub(crate) async fn deliver_channel_replies( } return; } - deliver_channel_replies_to_targets( + crate::channel_reply_delivery::deliver_channel_replies_to_targets( outbound, targets, session_key, + trace_correlation_key, text, Arc::clone(state), desired_reply_medium, @@ -129,7 +132,7 @@ pub(crate) async fn deliver_channel_replies( /// Format buffered status log entries into a Telegram expandable blockquote HTML. /// Returns an empty string if there are no entries. -fn format_logbook_html(entries: &[String]) -> String { +pub(crate) fn format_logbook_html(entries: &[String]) -> String { if entries.is_empty() { return String::new(); } @@ -146,32 +149,57 @@ fn format_logbook_html(entries: &[String]) -> String { html } +/// Send the activity logbook after a streamed reply. +/// +/// The stream delivered the answer itself, so this follow-up is the only +/// message this path creates — and a reader rating the turn may well land on +/// it. It is therefore linked to the trace like any other reply message. async fn send_channel_logbook_follow_up_to_targets( outbound: Arc, targets: Vec, logbook_html: &str, + feedback: Option>, + session_key: &str, + trace_correlation_key: &str, ) { if targets.is_empty() || logbook_html.is_empty() { return; } let html = logbook_html.to_string(); + let session_key = session_key.to_string(); + let trace_correlation_key = trace_correlation_key.to_string(); let mut tasks = Vec::with_capacity(targets.len()); for target in targets { let outbound = Arc::clone(&outbound); let html = html.clone(); + let feedback = feedback.clone(); + let session_key = session_key.clone(); + let trace_correlation_key = trace_correlation_key.clone(); let to = target.outbound_to().into_owned(); tasks.push(tokio::spawn(async move { - if let Err(e) = outbound - .send_html(&target.account_id, &to, &html, None) + match outbound + .send_html_reporting_ids(&target.account_id, &to, &html, None) .await { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback.as_deref(), + &target, + &message_ids, + &session_key, + &trace_correlation_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + }, } })); } @@ -219,99 +247,6 @@ fn format_channel_error_message(error_obj: &Value) -> String { /// `chat.compaction.show_settings_hint = false` don't see the repetitive /// hint on every compaction. Mode + token lines are always included. /// The LLM retry path never sees this text regardless. -fn format_channel_compaction_notice( - outcome: &compaction_run::CompactionOutcome, - include_settings_hint: bool, -) -> String { - let mode_label = match outcome.effective_mode { - moltis_config::CompactionMode::Deterministic => "Deterministic", - moltis_config::CompactionMode::RecencyPreserving => "Recency preserving", - moltis_config::CompactionMode::Structured => "Structured", - moltis_config::CompactionMode::LlmReplace => "LLM replace", - }; - let total = outcome.total_tokens(); - let token_line = if total == 0 { - // Any strategy that made no LLM calls ends up here: Deterministic, - // RecencyPreserving, or a Structured run that fell back to - // recency_preserving before the LLM call landed. Report the - // actual effective mode so users don't see "deterministic - // strategy" when they picked recency_preserving. - format!( - "No LLM tokens used ({} strategy)", - mode_label.to_lowercase() - ) - } else { - format!( - "Used {total} tokens ({input} in + {output} out)", - total = total, - input = outcome.input_tokens, - output = outcome.output_tokens, - ) - }; - let body = format!( - "🧹 Conversation compacted\n\ - Mode: {mode_label}\n\ - {token_line}", - ); - if include_settings_hint { - format!("{body}\n{hint}", hint = compaction_run::SETTINGS_HINT) - } else { - body - } -} - -/// Send a silent "session compacted" notice to pending channel targets -/// without draining them. -/// -/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, -/// not drained, so the in-flight agent run can still deliver its final -/// reply to them afterward. Uses `send_text_silent` so the channel -/// integration doesn't count it toward user-visible interactive replies -/// (no TTS, no delivery receipts beyond the channel's own). -pub(crate) async fn notify_channels_of_compaction( - state: &Arc, - session_key: &str, - outcome: &compaction_run::CompactionOutcome, - include_settings_hint: bool, -) { - let targets = state.peek_channel_replies(session_key).await; - if targets.is_empty() { - return; - } - - let Some(outbound) = state.channel_outbound() else { - return; - }; - - let message = format_channel_compaction_notice(outcome, include_settings_hint); - let mut tasks = Vec::with_capacity(targets.len()); - for target in targets { - let outbound = Arc::clone(&outbound); - let message = message.clone(); - let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { - let reply_to = target.message_id.as_deref(); - if let Err(e) = outbound - .send_text_silent(&target.account_id, &to, &message, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send compaction notice to channel: {e}" - ); - } - })); - } - - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel compaction notice task join failed"); - } - } -} - /// Send a short retry status update to pending channel targets without draining /// them. The final reply (or terminal error) will still use the same targets. pub(crate) async fn send_retry_status_to_channels( @@ -419,254 +354,6 @@ pub(crate) async fn deliver_channel_error( } } -async fn deliver_channel_replies_to_targets( - outbound: Arc, - targets: Vec, - session_key: &str, - text: &str, - state: Arc, - desired_reply_medium: ReplyMedium, - status_log: Vec, - streamed_target_keys: &HashSet, -) { - let session_key = session_key.to_string(); - let text = text.to_string(); - let logbook_html = format_logbook_html(&status_log); - let mut tasks = Vec::with_capacity(targets.len()); - for target in targets { - let outbound = Arc::clone(&outbound); - let state = Arc::clone(&state); - let session_key = session_key.clone(); - let text = text.clone(); - let logbook_html = logbook_html.clone(); - // Text was already delivered via edit-in-place streaming — skip text - // caption/follow-up and only send the TTS voice audio. - let text_already_streamed = - streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); - let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { - let tts_payload = match desired_reply_medium { - ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, - ReplyMedium::Text => None, - }; - let reply_to = target.message_id.as_deref(); - match target.channel_type { - moltis_channels::ChannelType::Telegram => match tts_payload { - Some(mut payload) => { - let transcript = std::mem::take(&mut payload.text); - - if text_already_streamed { - // Text was already streamed — send voice audio only. - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - // Check if transcript fits as Telegram caption (when feature enabled). - // When telegram feature is disabled, this evaluates to false and we - // send voice + follow-up text. - #[cfg(feature = "telegram")] - let fits_in_caption = transcript.len() - <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; - #[cfg(not(feature = "telegram"))] - let fits_in_caption = false; - - if fits_in_caption { - // Short transcript fits as a caption on the voice message. - payload.text = transcript; - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - // Transcript too long for a caption — send voice - // without caption, then the full text as a follow-up. - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - let text_result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &transcript, None) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &transcript, - &logbook_html, - None, - ) - .await - }; - if let Err(e) = text_result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send transcript follow-up: {e}" - ); - } - } - } - }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - }, - None => { - let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - } - }, - }, - _ => match tts_payload { - Some(payload) => { - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - }, - None => { - let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - } - }, - }, - } - })); - } - - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel reply task join failed"); - } - } -} - #[derive(Debug, Deserialize)] struct TtsStatusResponse { enabled: bool, @@ -747,7 +434,7 @@ pub(crate) async fn generate_tts_audio( .map_err(|_| error::Error::message("invalid base64 audio returned by TTS provider")) } -async fn build_tts_payload( +pub(crate) async fn build_tts_payload( state: &Arc, session_key: &str, target: &moltis_channels::ChannelReplyTarget, diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index 62875ef293..67ad5b986e 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,6 +1,9 @@ mod agent_loop; +mod channel_compaction; +mod channel_feedback; #[cfg(any(feature = "push-notifications", test))] mod channel_push; +mod channel_reply_delivery; mod channels; mod compaction; mod compaction_run; diff --git a/crates/chat/src/run_with_tools.rs b/crates/chat/src/run_with_tools.rs index b525f93c9f..70c6d63374 100644 --- a/crates/chat/src/run_with_tools.rs +++ b/crates/chat/src/run_with_tools.rs @@ -36,11 +36,12 @@ use crate::{ ChannelStreamDispatcher, clear_unsupported_model, compact_session, mark_unsupported_model, ordered_runner_event_callback, }, + channel_compaction::notify_channels_of_compaction, channels::{ deliver_channel_error, deliver_channel_replies, dispatch_document_to_channels, document_payload_from_data_uri, document_payload_from_ref, generate_tts_audio, - notify_channels_of_compaction, send_location_to_channels, send_retry_status_to_channels, - send_screenshot_to_channels, send_tool_result_to_channels, send_tool_status_to_channels, + send_location_to_channels, send_retry_status_to_channels, send_screenshot_to_channels, + send_tool_result_to_channels, send_tool_status_to_channels, }, chat_error::parse_chat_error, memory_tools::{effective_tool_mode, install_agent_scoped_memory_tools}, @@ -277,9 +278,10 @@ pub(crate) async fn run_with_tools( let provider_name_for_events = provider_name.to_string(); let active_partial_for_events = active_partial_assistant.as_ref().map(Arc::clone); let (on_event, mut event_rx) = ordered_runner_event_callback(); - let channel_stream_dispatcher = ChannelStreamDispatcher::for_session(state, session_key) - .await - .map(|dispatcher| Arc::new(Mutex::new(dispatcher))); + let channel_stream_dispatcher = + ChannelStreamDispatcher::for_session(state, session_key, run_id) + .await + .map(|dispatcher| Arc::new(Mutex::new(dispatcher))); let channel_stream_for_events = channel_stream_dispatcher.as_ref().map(Arc::clone); let event_forwarder = tokio::spawn(async move { // Track tool call arguments from ToolCallStart so they can be persisted in ToolCallEnd. @@ -893,6 +895,7 @@ pub(crate) async fn run_with_tools( conn_id.as_deref(), runtime_context, ); + tool_context["_trace_correlation_key"] = serde_json::json!(run_id); if let Some(controls) = tool_controls { if let Some(active_tools) = controls.active_tools { tool_context["active_tools"] = serde_json::json!(active_tools); @@ -1227,6 +1230,7 @@ pub(crate) async fn run_with_tools( deliver_channel_replies( state, session_key, + run_id, &display_text, desired_reply_medium, &streamed_target_keys, diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index 6e8face729..b370ce7053 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -49,6 +49,14 @@ pub trait ChatRuntime: Send + Sync { /// Broadcast a WebSocket event to all connected clients. async fn broadcast(&self, topic: &str, payload: Value); + /// Reaction-feedback service, when one is configured. + /// + /// Defaulted to `None` so runtimes that do not collect feedback — tests, + /// embedders — need no change. + fn feedback(&self) -> Option> { + None + } + // ── Channel reply queue ────────────────────────────────────────────── /// Push a reply target for a session (channel message triggered a chat run). diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index 72d7c0a5ac..d24d890c64 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -32,7 +32,7 @@ use { use crate::{ agent_loop::effective_tool_mode, - channels::notify_channels_of_compaction, + channel_compaction::notify_channels_of_compaction, compaction_run, memory_tools::AgentScopedMemoryWriter, message::{ @@ -303,6 +303,8 @@ impl ChatService for LiveChatService { { warn!("send_sync: failed to persist assistant message: {e}"); } + crate::channel_feedback::record_web_reply_trace(&self.state, &session_key, &run_id) + .await; // Update metadata message count. if let Ok(count) = self.session_store.count(&session_key).await { self.session_metadata.touch(&session_key, count).await; diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index b8d33a0e34..02975263f8 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -1286,6 +1286,12 @@ impl LiveChatService { { warn!("failed to persist assistant message: {e}"); } + crate::channel_feedback::record_web_reply_trace( + &state_for_drain, + &session_key_clone, + &run_id_clone, + ) + .await; // Update metadata counts. if let Ok(count) = session_store.count(&session_key_clone).await { session_metadata.touch(&session_key_clone, count).await; diff --git a/crates/chat/src/streaming.rs b/crates/chat/src/streaming.rs index 8008c6bd91..f310f40589 100644 --- a/crates/chat/src/streaming.rs +++ b/crates/chat/src/streaming.rs @@ -17,7 +17,8 @@ use { moltis_agents::{ ChatMessage, UserContent, model::{ - StreamEvent, push_capped_provider_raw_event, + AgentToolControls, ProviderAttemptEvent, ProviderIdentity, StreamEvent, + TrackedStreamEvent, push_capped_provider_raw_event, values_to_chat_messages_with_tool_result_limit, }, prompt::{PromptRuntimeContext, build_system_prompt_minimal_runtime_details}, @@ -34,7 +35,7 @@ use crate::{ chat_error::parse_chat_error, message::apply_voice_reply_suffix, models::DisabledModelsStore, - prompt::prompt_build_limits_from_config, + prompt::{channel_binding_from_runtime_context, prompt_build_limits_from_config}, runtime::ChatRuntime, service::ActiveAssistantDraft, types::*, @@ -226,337 +227,459 @@ pub(crate) async fn run_streaming( let mut rate_limit_retries_remaining: u8 = STREAM_RATE_LIMIT_MAX_RETRIES; let mut rate_limit_backoff_ms: Option = None; let mut channel_stream_dispatcher = - ChannelStreamDispatcher::for_session(state, session_key).await; + ChannelStreamDispatcher::for_session(state, session_key, run_id).await; + let turn_recorder = moltis_agents::runner::instrumentation::begin_turn( + session_key, + Some(run_id), + channel_binding_from_runtime_context(runtime_context).as_ref(), + provider_name, + model_id, + user_content, + moltis_agents::runner::instrumentation::recorder_settings(&persona.config.instrumentation), + persona.config.instrumentation.environment.clone(), + moltis_agents::runner::instrumentation::release(&persona.config.instrumentation), + ); 'attempts: loop { #[cfg(feature = "metrics")] let stream_start = Instant::now(); - let mut stream = provider.stream(messages.clone()); + let mut generation_step: Option = None; + let mut selected_identity = ProviderIdentity::new(provider_name, model_id); + let mut stream = provider.stream_with_tools_and_options_tracked( + messages.clone(), + Vec::new(), + AgentToolControls::default(), + ); let mut accumulated = String::new(); let mut accumulated_reasoning = String::new(); let mut raw_llm_responses: Vec = Vec::new(); while let Some(event) = stream.next().await { match event { - StreamEvent::Delta(delta) => { - accumulated.push_str(&delta); - if let Some(ref map) = active_partial_assistant - && let Some(draft) = map.write().await.get_mut(session_key) - { - draft.append_text(&delta); + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) => { + if let Some(mut previous) = generation_step.take() { + previous.fail("provider attempt superseded before completion"); + previous.finish(); } - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.send_delta(&delta).await; + selected_identity = identity; + if let Some(recorder) = turn_recorder.as_ref() { + recorder.set_metadata( + "provider", + Value::String(selected_identity.provider.clone()), + ); + recorder + .set_metadata("model", Value::String(selected_identity.model.clone())); + let mut step = recorder.step( + moltis_observability::ObservationKind::Generation, + moltis_agents::runner::instrumentation::generation_name( + &selected_identity.provider, + &selected_identity.model, + ), + ); + step.set_model(selected_identity.model.clone()); + step.set_metadata( + "provider", + Value::String(selected_identity.provider.clone()), + ); + step.set_input(Value::Array( + messages.iter().map(ChatMessage::to_openai_value).collect(), + )); + generation_step = Some(step); } - broadcast( - state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "delta", - "text": delta, - }), - BroadcastOpts::default(), - ) - .await; }, - StreamEvent::ReasoningDelta(delta) => { - accumulated_reasoning.push_str(&delta); - if let Some(ref map) = active_partial_assistant - && let Some(draft) = map.write().await.get_mut(session_key) - { - draft.set_reasoning(&accumulated_reasoning); + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { error, .. }) => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + step.finish(); } - broadcast( - state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "thinking_text", - "text": accumulated_reasoning.clone(), - }), - BroadcastOpts::default(), - ) - .await; }, - StreamEvent::ProviderRaw(raw) => { - push_capped_provider_raw_event(&mut raw_llm_responses, raw); - }, - StreamEvent::Done(usage) => { - clear_unsupported_model(state, model_store, model_id).await; - - // Record streaming completion metrics (mirroring provider_chain.rs) - #[cfg(feature = "metrics")] - { - let duration = stream_start.elapsed().as_secs_f64(); - counter!( - llm_metrics::COMPLETIONS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(1); - counter!( - llm_metrics::INPUT_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.input_tokens)); - counter!( - llm_metrics::OUTPUT_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.output_tokens)); - counter!( - llm_metrics::CACHE_READ_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.cache_read_tokens)); - counter!( - llm_metrics::CACHE_WRITE_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() + TrackedStreamEvent::Event(event) => match event { + StreamEvent::Delta(delta) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } + accumulated.push_str(&delta); + if let Some(ref map) = active_partial_assistant + && let Some(draft) = map.write().await.get_mut(session_key) + { + draft.append_text(&delta); + } + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.send_delta(&delta).await; + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "delta", + "text": delta, + }), + BroadcastOpts::default(), ) - .increment(u64::from(usage.cache_write_tokens)); - histogram!( - llm_metrics::COMPLETION_DURATION_SECONDS, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() + .await; + }, + StreamEvent::ReasoningDelta(delta) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } + accumulated_reasoning.push_str(&delta); + if let Some(ref map) = active_partial_assistant + && let Some(draft) = map.write().await.get_mut(session_key) + { + draft.set_reasoning(&accumulated_reasoning); + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "thinking_text", + "text": accumulated_reasoning.clone(), + }), + BroadcastOpts::default(), ) - .record(duration); - } + .await; + }, + StreamEvent::ProviderRaw(raw) => { + push_capped_provider_raw_event(&mut raw_llm_responses, raw); + }, + StreamEvent::Done(usage) => { + clear_unsupported_model(state, model_store, &selected_identity.model).await; + + // Record streaming completion metrics (mirroring provider_chain.rs) + #[cfg(feature = "metrics")] + { + let duration = stream_start.elapsed().as_secs_f64(); + counter!( + llm_metrics::COMPLETIONS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(1); + counter!( + llm_metrics::INPUT_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.input_tokens)); + counter!( + llm_metrics::OUTPUT_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.output_tokens)); + counter!( + llm_metrics::CACHE_READ_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.cache_read_tokens)); + counter!( + llm_metrics::CACHE_WRITE_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.cache_write_tokens)); + histogram!( + llm_metrics::COMPLETION_DURATION_SECONDS, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .record(duration); + } - let is_silent = accumulated.trim().is_empty(); - let reasoning = { - let trimmed = accumulated_reasoning.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_string()) - }; - let streamed_target_keys = - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - dispatcher.completed_target_keys().await - } else { - HashSet::new() + let is_silent = accumulated.trim().is_empty(); + let reasoning = { + let trimmed = accumulated_reasoning.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) }; - - info!( - run_id, - input_tokens = usage.input_tokens, - output_tokens = usage.output_tokens, - response = %accumulated, - silent = is_silent, - "chat stream done" - ); - - // Detect provider failures: silent stream with zero tokens - // means the LLM never produced output (e.g. network_error). - if is_silent && usage.output_tokens == 0 { - warn!( + let streamed_target_keys = + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.finish().await; + dispatcher.completed_target_keys().await + } else { + HashSet::new() + }; + + info!( run_id, - "empty stream with zero tokens — treating as provider error" + input_tokens = usage.input_tokens, + output_tokens = usage.output_tokens, + response = %accumulated, + silent = is_silent, + "chat stream done" ); - let error_obj = parse_chat_error( - "The provider returned an empty response (possible network error). Please try again.", - Some(provider_name), - ); - deliver_channel_error(state, session_key, &error_obj).await; - let error_payload = ChatErrorBroadcast { - run_id: run_id.to_string(), - session_key: session_key.to_string(), - state: "error", - error: error_obj, - seq: client_seq, - }; - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; - return None; - } - let assistant_message_index = user_message_index + 1; - - // Generate & persist TTS audio for voice-medium web UI replies. - let mut audio_warning: Option = None; - let audio_path = if !is_silent && desired_reply_medium == ReplyMedium::Voice { - match generate_tts_audio(state, session_key, &accumulated).await { - Ok(bytes) => { - let filename = format!("{run_id}.ogg"); - if let Some(store) = session_store { - match store.save_media(session_key, &filename, &bytes).await { - Ok(path) => Some(path), - Err(e) => { - let warning = format!( - "TTS audio generated but failed to save: {e}" - ); - warn!(run_id, error = %warning, "failed to save TTS audio to media dir"); - audio_warning = Some(warning); - None - }, - } - } else { - audio_warning = Some( + // Detect provider failures: silent stream with zero tokens + // means the LLM never produced output (e.g. network_error). + if is_silent && usage.output_tokens == 0 { + warn!( + run_id, + "empty stream with zero tokens — treating as provider error" + ); + let error_obj = parse_chat_error( + "The provider returned an empty response (possible network error). Please try again.", + Some(&selected_identity.provider), + ); + let message = + "provider returned an empty response with zero output tokens"; + if let Some(mut step) = generation_step.take() { + step.set_usage( + moltis_agents::runner::instrumentation::to_token_usage(&usage), + ); + step.fail(message); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(message); + } + deliver_channel_error(state, session_key, &error_obj).await; + let error_payload = ChatErrorBroadcast { + run_id: run_id.to_string(), + session_key: session_key.to_string(), + state: "error", + error: error_obj, + seq: client_seq, + }; + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&error_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + return None; + } + + if let Some(mut step) = generation_step.take() { + step.set_usage(moltis_agents::runner::instrumentation::to_token_usage( + &usage, + )); + step.set_output(Value::String(accumulated.clone())); + if !accumulated_reasoning.is_empty() { + step.set_output_metadata( + "reasoning", + Value::String(accumulated_reasoning.clone()), + ); + } + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.set_output(Value::String(accumulated.clone())); + recorder.finish(); + } + + let assistant_message_index = user_message_index + 1; + + // Generate & persist TTS audio for voice-medium web UI replies. + let mut audio_warning: Option = None; + let audio_path = if !is_silent && desired_reply_medium == ReplyMedium::Voice + { + match generate_tts_audio(state, session_key, &accumulated).await { + Ok(bytes) => { + let filename = format!("{run_id}.ogg"); + if let Some(store) = session_store { + match store.save_media(session_key, &filename, &bytes).await + { + Ok(path) => Some(path), + Err(e) => { + let warning = format!( + "TTS audio generated but failed to save: {e}" + ); + warn!(run_id, error = %warning, "failed to save TTS audio to media dir"); + audio_warning = Some(warning); + None + }, + } + } else { + audio_warning = Some( "TTS audio generated but session media storage is unavailable" .to_string(), ); + None + } + }, + Err(error) => { + let error = error.to_string(); + warn!(run_id, error = %error, "voice reply generation skipped"); + audio_warning = Some(error); None - } - }, - Err(error) => { - let error = error.to_string(); - warn!(run_id, error = %error, "voice reply generation skipped"); - audio_warning = Some(error); - None - }, - } - } else { - None - }; - - let final_payload = build_chat_final_broadcast( - run_id, - session_key, - accumulated.clone(), - provider.id().to_string(), - provider_name.to_string(), - UsageSnapshot::new(usage.clone(), Some(usage.clone())), - run_started.elapsed().as_millis() as u64, - assistant_message_index, - desired_reply_medium, - None, - None, - audio_path.clone(), - audio_warning, - reasoning.clone(), - client_seq, - ); - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&final_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + }, + } + } else { + None + }; - if !is_silent { - #[cfg(feature = "push-notifications")] - { - tracing::info!("push: checking push notification"); - let push_state = Arc::clone(state); - let push_session_key = session_key.to_string(); - let push_text = accumulated.clone(); - let push_order = crate::channel_push::next_push_notification_order(); - tokio::spawn(async move { - send_chat_push_notification( - &push_state, - &push_session_key, - &push_text, - push_order, - ) - .await; - }); - } - deliver_channel_replies( - state, + let final_payload = build_chat_final_broadcast( + run_id, session_key, - &accumulated, + accumulated.clone(), + selected_identity.model.clone(), + selected_identity.provider.clone(), + UsageSnapshot::new(usage.clone(), Some(usage.clone())), + run_started.elapsed().as_millis() as u64, + assistant_message_index, desired_reply_medium, - &streamed_target_keys, - ) - .await; - } - let llm_api_response = - (!raw_llm_responses.is_empty()).then_some(Value::Array(raw_llm_responses)); - return Some(build_assistant_turn_output( - accumulated, - UsageSnapshot::new(usage.clone(), Some(usage)), - run_started.elapsed().as_millis() as u64, - audio_path, - reasoning, - llm_api_response, - )); - }, - StreamEvent::Error(msg) => { - let error_obj = parse_chat_error(&msg, Some(provider_name)); - let has_no_streamed_content = accumulated.trim().is_empty() - && accumulated_reasoning.trim().is_empty() - && raw_llm_responses.is_empty(); - if has_no_streamed_content - && let Some(delay_ms) = next_stream_retry_delay_ms( - &msg, - &error_obj, - &mut server_retries_remaining, - &mut rate_limit_retries_remaining, - &mut rate_limit_backoff_ms, - ) - { - warn!( - run_id, - error = %msg, - delay_ms, - server_retries_remaining, - rate_limit_retries_remaining, - "chat stream transient error, retrying after delay" + None, + None, + audio_path.clone(), + audio_warning, + reasoning.clone(), + client_seq, ); - if error_obj.get("type").and_then(Value::as_str) - == Some("rate_limit_exceeded") - { - send_retry_status_to_channels( + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&final_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + + if !is_silent { + // Send push notification when chat response completes + #[cfg(feature = "push-notifications")] + { + tracing::info!("push: checking push notification"); + let push_state = Arc::clone(state); + let push_session_key = session_key.to_string(); + let push_text = accumulated.clone(); + let push_order = + crate::channel_push::next_push_notification_order(); + tokio::spawn(async move { + send_chat_push_notification( + &push_state, + &push_session_key, + &push_text, + push_order, + ) + .await; + }); + } + deliver_channel_replies( state, session_key, + run_id, + &accumulated, + desired_reply_medium, + &streamed_target_keys, + ) + .await; + } + let llm_api_response = (!raw_llm_responses.is_empty()) + .then_some(Value::Array(raw_llm_responses)); + return Some(build_assistant_turn_output( + accumulated, + UsageSnapshot::new(usage.clone(), Some(usage)), + run_started.elapsed().as_millis() as u64, + audio_path, + reasoning, + llm_api_response, + )); + }, + StreamEvent::Error(msg) => { + let error_obj = parse_chat_error(&msg, Some(&selected_identity.provider)); + let has_no_streamed_content = accumulated.trim().is_empty() + && accumulated_reasoning.trim().is_empty() + && raw_llm_responses.is_empty(); + if has_no_streamed_content + && let Some(delay_ms) = next_stream_retry_delay_ms( + &msg, &error_obj, - Duration::from_millis(delay_ms), + &mut server_retries_remaining, + &mut rate_limit_retries_remaining, + &mut rate_limit_backoff_ms, + ) + { + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } + warn!( + run_id, + error = %msg, + delay_ms, + server_retries_remaining, + rate_limit_retries_remaining, + "chat stream transient error, retrying after delay" + ); + if error_obj.get("type").and_then(Value::as_str) + == Some("rate_limit_exceeded") + { + send_retry_status_to_channels( + state, + session_key, + &error_obj, + Duration::from_millis(delay_ms), + ) + .await; + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "retrying", + "error": error_obj, + "retryAfterMs": delay_ms, + "seq": client_seq, + }), + BroadcastOpts::default(), ) .await; + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + continue 'attempts; } - broadcast( + + warn!(run_id, error = %msg, "chat stream error"); + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(msg.clone()); + } + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.finish().await; + } + state.set_run_error(run_id, msg.clone()).await; + mark_unsupported_model( state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "retrying", - "error": error_obj, - "retryAfterMs": delay_ms, - "seq": client_seq, - }), - BroadcastOpts::default(), + model_store, + &selected_identity.model, + &selected_identity.provider, + &error_obj, ) .await; - tokio::time::sleep(Duration::from_millis(delay_ms)).await; - continue 'attempts; - } - - warn!(run_id, error = %msg, "chat stream error"); - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - } - state.set_run_error(run_id, msg.clone()).await; - mark_unsupported_model(state, model_store, model_id, provider_name, &error_obj) - .await; - deliver_channel_error(state, session_key, &error_obj).await; - let error_payload = ChatErrorBroadcast { - run_id: run_id.to_string(), - session_key: session_key.to_string(), - state: "error", - error: error_obj, - seq: client_seq, - }; - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; - return None; + deliver_channel_error(state, session_key, &error_obj).await; + let error_payload = ChatErrorBroadcast { + run_id: run_id.to_string(), + session_key: session_key.to_string(), + state: "error", + error: error_obj, + seq: client_seq, + }; + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&error_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + return None; + }, + // Tool events not expected in stream-only mode. + StreamEvent::ToolCallStart { .. } + | StreamEvent::ToolCallArgumentsDelta { .. } + | StreamEvent::ToolCallComplete { .. } => {}, }, - // Tool events not expected in stream-only mode. - StreamEvent::ToolCallStart { .. } - | StreamEvent::ToolCallArgumentsDelta { .. } - | StreamEvent::ToolCallComplete { .. } => {}, } } // Stream ended unexpectedly without Done/Error. + let message = "provider stream ended without a terminal event"; + if let Some(mut step) = generation_step.take() { + step.fail(message); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(message); + } return None; } } diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 6aa8a59e82..3de28ce585 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -158,6 +158,7 @@ full = [ "lang-rust", "lang-toml", "lang-typescript", + "langfuse", "llm-compaction", "local-embeddings", "local-llm", @@ -170,6 +171,7 @@ full = [ "ngrok", "nostr", "openclaw-import", + "otlp", "prometheus", "provider-github-copilot", "provider-kimi-code", @@ -221,8 +223,13 @@ lang-typescript = ["moltis-memory/lang-typescript"] # jemalloc is intentionally unavailable on linux/aarch64 to avoid runtime # crashes on 16 KiB page kernels (e.g. Raspberry Pi OS variants). claude-import = ["dep:moltis-claude-import", "moltis-gateway/claude-import"] -codex-import = ["dep:moltis-codex-import", "moltis-gateway/codex-import"] +codex-import = ["dep:moltis-codex-import", "moltis-gateway/codex-import"] hermes-import = ["dep:moltis-hermes-import", "moltis-gateway/hermes-import"] +# Instrumentation exporters. Default-on so the `[instrumentation]` config can be +# switched on without a rebuild; enabling the config, not the feature, is what +# starts sending data. `langfuse` implies `otlp`: Langfuse ingests traces over +# the OTLP wire format. +langfuse = ["moltis-gateway/langfuse"] lightweight = ["jemalloc", "tls", "web-ui"] local-embeddings = ["moltis-gateway/local-embeddings"] local-llm = ["moltis-gateway/local-llm"] @@ -237,6 +244,7 @@ netbird = ["moltis-httpd/netbird", "moltis-web?/netbird"] ngrok = ["moltis-httpd/ngrok", "moltis-web?/ngrok"] nostr = ["moltis-httpd/nostr"] openclaw-import = ["dep:moltis-openclaw-import", "moltis-gateway/openclaw-import"] +otlp = ["moltis-gateway/otlp"] prometheus = ["moltis-httpd/prometheus"] provider-github-copilot = [ "moltis-gateway/provider-github-copilot", diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index d161d84124..d3c60c29d8 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -56,15 +56,16 @@ pub use { AgentIdentity, AgentMemoryWriteMode, AgentPreset, AgentRuntimeLimitSource, AgentRuntimeLimits, AgentsConfig, AuthConfig, CacheRetention, CalDavAccountConfig, CalDavConfig, ChannelToolPolicyOverride, ChannelsConfig, ChatConfig, CodeIndexTomlConfig, - CompactionConfig, CompactionMode, GeoLocation, GroupToolPolicy, HeartbeatConfig, - HomeAssistantAccountConfig, HomeAssistantConfig, MemoryBackend, MemoryCitationsMode, - MemoryProvider, MemoryScope, MemorySearchMergeStrategy, MemoryStyle, MessageQueueMode, - ModePreset, ModesConfig, MoltisConfig, NgrokConfig, PresetMemoryConfig, PresetToolPolicy, - PromptMemoryMode, ResolvedIdentity, SessionAccessPolicyConfig, SessionExportMode, Timezone, - ToolMode, ToolPolicyConfig, ToolRegistryMode, UserProfile, UserProfileWriteMode, - VoiceConfig, VoiceElevenLabsConfig, VoiceOpenAiConfig, VoiceSttConfig, VoiceSttProvider, - VoiceTtsConfig, VoiceTtsProvider, VoiceWhisperConfig, VoiceWhisperLocalConfig, WireApi, - parse_byte_size, + CompactionConfig, CompactionMode, ContentCaptureMode, DatadogSettings, FeedbackSettings, + GeoLocation, GroupToolPolicy, HeartbeatConfig, HomeAssistantAccountConfig, + HomeAssistantConfig, InstrumentationConfig, LangfuseSettings, MemoryBackend, + MemoryCitationsMode, MemoryProvider, MemoryScope, MemorySearchMergeStrategy, MemoryStyle, + MessageQueueMode, ModePreset, ModesConfig, MoltisConfig, NgrokConfig, OtlpSettings, + PresetMemoryConfig, PresetToolPolicy, PromptMemoryMode, ResolvedIdentity, + SessionAccessPolicyConfig, SessionExportMode, Timezone, ToolMode, ToolPolicyConfig, + ToolRegistryMode, UserProfile, UserProfileWriteMode, VoiceConfig, VoiceElevenLabsConfig, + VoiceOpenAiConfig, VoiceSttConfig, VoiceSttProvider, VoiceTtsConfig, VoiceTtsProvider, + VoiceWhisperConfig, VoiceWhisperLocalConfig, WireApi, parse_byte_size, }, validate::{Diagnostic, Severity, ValidationResult}, }; diff --git a/crates/config/src/schema.rs b/crates/config/src/schema.rs index eb4339d136..044907666b 100644 --- a/crates/config/src/schema.rs +++ b/crates/config/src/schema.rs @@ -15,6 +15,8 @@ mod chat; mod code_index; #[path = "schema/hooks.rs"] mod hooks; + +mod instrumentation; #[path = "schema/memory.rs"] mod memory; #[path = "schema/modes.rs"] @@ -33,8 +35,8 @@ mod tools; mod voice; pub use { - agents::*, chat::*, code_index::*, hooks::*, memory::*, modes::*, phone::*, providers::*, - runtime::*, system::*, tools::*, voice::*, + agents::*, chat::*, code_index::*, hooks::*, instrumentation::*, memory::*, modes::*, phone::*, + providers::*, runtime::*, system::*, tools::*, voice::*, }; // ── Reasoning effort ────────────────────────────────────────────────────── @@ -305,6 +307,8 @@ pub struct MoltisConfig { pub auth: AuthConfig, pub graphql: GraphqlConfig, pub metrics: MetricsConfig, + /// Agent instrumentation exported to Langfuse, OTLP collectors and Datadog. + pub instrumentation: InstrumentationConfig, pub identity: AgentIdentity, pub user: UserProfile, pub hooks: Option, diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs new file mode 100644 index 0000000000..ed39b78693 --- /dev/null +++ b/crates/config/src/schema/instrumentation.rs @@ -0,0 +1,496 @@ +//! Instrumentation (tracing/observability) configuration. +//! +//! One `[instrumentation]` section configures every backend, because they are +//! fed from a single instrumentation pass in the agent runtime. Each backend +//! has its own sub-table and its own content policy — see the `profile` module +//! in `moltis-observability` for why an LLM observability product and an +//! infrastructure APM must not receive the same payload. + +use { + secrecy::Secret, + serde::{Deserialize, Serialize}, +}; + +use crate::schema::{deserialize_option_secret, serialize_option_secret}; + +fn default_true() -> bool { + true +} + +fn default_sample_rate() -> f64 { + 1.0 +} + +fn default_flush_interval_ms() -> u64 { + 5_000 +} + +fn default_queue_capacity() -> usize { + 10_000 +} + +fn default_max_batch_bytes() -> usize { + 3_000_000 +} + +fn default_timeout_secs() -> u64 { + 10 +} + +fn default_langfuse_host() -> String { + "https://cloud.langfuse.com".to_string() +} + +/// How much conversation content a backend receives. +/// +/// Mirrors `moltis_observability::profile::ContentCapture`; kept as a separate +/// type so `moltis-config` does not depend on the observability crate, and +/// converted at the boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContentCaptureMode { + /// Send prompts, completions, tool arguments and tool results. + Full, + /// Send structural metadata only: sizes, counts, model, timings. + #[default] + MetadataOnly, + /// Send neither content nor content-derived metadata. + None, +} + +impl ContentCaptureMode { + /// Accepted values, for config validation messages. + pub const ALL: &'static [&'static str] = &["full", "metadata_only", "none"]; +} + +fn default_link_retention_days() -> u32 { + 30 +} + +/// End-user feedback collected from chat reactions. +/// +/// A thumb on a reply is the cheapest quality signal available, but attributing +/// one to the turn that produced it needs the reply's trace to still be on +/// record when the reaction arrives — hence the retention window. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct FeedbackSettings { + /// Collect reaction feedback. On by default once instrumentation is + /// enabled: a scoring pipeline with no scores in it is not useful. + #[serde(default = "default_true")] + pub enabled: bool, + /// Reaction tokens counted as approval. Empty uses the built-in list. + /// + /// Accepts raw emoji or shortcodes; skin tone and presentation selectors + /// are ignored when matching. + #[serde(default)] + pub positive: Vec, + /// Reaction tokens counted as disapproval. Empty uses the built-in list. + #[serde(default)] + pub negative: Vec, + /// How long a reply stays attributable to its trace, in days. + /// + /// Reactions often arrive long after the reply. Kept bounded because the + /// link table would otherwise grow without limit. + #[serde(default = "default_link_retention_days")] + pub link_retention_days: u32, +} + +impl Default for FeedbackSettings { + fn default() -> Self { + Self { + enabled: true, + positive: Vec::new(), + negative: Vec::new(), + link_retention_days: default_link_retention_days(), + } + } +} + +/// Top-level instrumentation configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct InstrumentationConfig { + /// Master switch. Disabled by default: every backend here ships data off + /// the machine, so it must be an explicit choice rather than something a + /// user discovers after the fact. + #[serde(default)] + pub enabled: bool, + /// Deployment environment reported to every backend. + #[serde(default = "default_environment")] + pub environment: String, + /// Build release identifier. Defaults to the running Moltis version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub release: Option, + /// Fraction of turns to trace, in `0.0..=1.0`. + #[serde(default = "default_sample_rate")] + pub sample_rate: f64, + /// Extra object keys whose values are redacted before export. Merged with + /// the built-in list; it cannot be narrowed. + #[serde(default)] + pub redact: Vec, + /// Nonzero bounded export queue depth. Events are dropped once this fills + /// rather than blocking the agent loop. + #[serde(default = "default_queue_capacity")] + pub queue_capacity: usize, + /// Nonzero maximum time an event waits before being flushed. + #[serde(default = "default_flush_interval_ms")] + pub flush_interval_ms: u64, + /// Nonzero maximum estimated batch size in bytes before a forced flush. + #[serde(default = "default_max_batch_bytes")] + pub max_batch_bytes: usize, + /// Langfuse backend. + #[serde(default)] + pub langfuse: LangfuseSettings, + /// Generic OTLP backend (Grafana Tempo/Alloy, Honeycomb, a collector). + #[serde(default)] + pub otlp: OtlpSettings, + /// Datadog backend, via Datadog's OTLP intake. + #[serde(default)] + pub datadog: DatadogSettings, + /// End-user reaction feedback. + #[serde(default)] + pub feedback: FeedbackSettings, +} + +fn default_environment() -> String { + "production".to_string() +} + +impl Default for InstrumentationConfig { + fn default() -> Self { + Self { + enabled: false, + environment: default_environment(), + release: None, + sample_rate: default_sample_rate(), + redact: Vec::new(), + queue_capacity: default_queue_capacity(), + flush_interval_ms: default_flush_interval_ms(), + max_batch_bytes: default_max_batch_bytes(), + langfuse: LangfuseSettings::default(), + otlp: OtlpSettings::default(), + datadog: DatadogSettings::default(), + feedback: FeedbackSettings::default(), + } + } +} + +impl InstrumentationConfig { + /// Whether any backend is switched on. + #[must_use] + pub fn any_backend_enabled(&self) -> bool { + self.enabled && (self.langfuse.enabled || self.otlp.enabled || self.datadog.enabled) + } +} + +/// Langfuse-specific settings. +/// +/// Langfuse is LLM-native: it wants the whole conversation, and its cost, +/// session and prompt-version views are built on that. Content capture is +/// therefore on by default here, unlike the APM backends. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct LangfuseSettings { + /// Whether to export to Langfuse. + #[serde(default)] + pub enabled: bool, + /// Base host. Set to a self-hosted URL to keep data on-premises. + #[serde(default = "default_langfuse_host")] + pub host: String, + /// Project public key. + #[serde(default)] + pub public_key: String, + /// Project secret key. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_option_secret", + deserialize_with = "deserialize_option_secret" + )] + pub secret_key: Option>, + /// Whether to send turn and step inputs. + #[serde(default = "default_true")] + pub capture_input: bool, + /// Whether to send turn and step outputs. + #[serde(default = "default_true")] + pub capture_output: bool, + /// Whether to send tool arguments and results. Separate from the input and + /// output switches because tool arguments are the likeliest place for + /// credentials to appear. + #[serde(default = "default_true")] + pub capture_tool_io: bool, + /// Nonzero per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +impl Default for LangfuseSettings { + fn default() -> Self { + Self { + enabled: false, + host: default_langfuse_host(), + public_key: String::new(), + secret_key: None, + capture_input: true, + capture_output: true, + capture_tool_io: true, + timeout_secs: default_timeout_secs(), + } + } +} + +/// Generic OTLP backend settings. +/// +/// Targets an OpenTelemetry collector, Grafana Alloy/Tempo, Honeycomb, or +/// anything else speaking OTLP/HTTP. Content capture defaults to +/// `metadata_only`: these are operational tools, and prompt bodies there mean +/// unbounded span size, cardinality pressure, per-byte ingest billing, and +/// conversation content sitting in a system nobody scoped for it. +#[derive(Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct OtlpSettings { + /// Whether to export over OTLP. + #[serde(default)] + pub enabled: bool, + /// Full traces endpoint, e.g. `http://localhost:4318/v1/traces`. + #[serde(default)] + pub endpoint: String, + /// Extra headers, typically for authentication. + #[serde(default)] + pub headers: std::collections::BTreeMap, + /// How much conversation content this backend receives. + #[serde(default)] + pub content: ContentCaptureMode, + /// Whether to attach end-user identity. Off by default: high-cardinality + /// in an APM's index and often a compliance question. + #[serde(default)] + pub emit_user_id: bool, + /// Nonzero per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +impl std::fmt::Debug for OtlpSettings { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OtlpSettings") + .field("enabled", &self.enabled) + .field("endpoint", &self.endpoint) + .field("header_names", &self.headers.keys().collect::>()) + .field("content", &self.content) + .field("emit_user_id", &self.emit_user_id) + .field("timeout_secs", &self.timeout_secs) + .finish() + } +} + +impl Default for OtlpSettings { + fn default() -> Self { + Self { + enabled: false, + endpoint: String::new(), + headers: std::collections::BTreeMap::new(), + content: ContentCaptureMode::MetadataOnly, + emit_user_id: false, + timeout_secs: default_timeout_secs(), + } + } +} + +/// Datadog backend settings. +/// +/// Datadog ingests OTLP either through the Datadog Agent (default +/// `http://localhost:4318/v1/traces`) or its direct intake. Tags are dropped +/// for this backend because Datadog indexes span tags and bills on custom +/// metric cardinality. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct DatadogSettings { + /// Whether to export to Datadog. + #[serde(default)] + pub enabled: bool, + /// OTLP endpoint exposed by the Datadog Agent or intake. + #[serde(default = "default_datadog_endpoint")] + pub endpoint: String, + /// Datadog API key, required only when posting to the intake directly + /// rather than through a local Agent. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_option_secret", + deserialize_with = "deserialize_option_secret" + )] + pub api_key: Option>, + /// Datadog service name. + #[serde(default = "default_datadog_service")] + pub service: String, + /// How much conversation content this backend receives. + #[serde(default)] + pub content: ContentCaptureMode, + /// Nonzero per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +fn default_datadog_endpoint() -> String { + "http://localhost:4318/v1/traces".to_string() +} + +fn default_datadog_service() -> String { + "moltis".to_string() +} + +impl Default for DatadogSettings { + fn default() -> Self { + Self { + enabled: false, + endpoint: default_datadog_endpoint(), + api_key: None, + service: default_datadog_service(), + content: ContentCaptureMode::MetadataOnly, + timeout_secs: default_timeout_secs(), + } + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use secrecy::ExposeSecret; + + use super::*; + + #[test] + fn instrumentation_is_off_by_default() { + // Unlike every other Moltis feature flag: enabling this ships + // conversation content to a third party. + let config = InstrumentationConfig::default(); + assert!(!config.enabled); + assert!(!config.langfuse.enabled); + assert!(!config.otlp.enabled); + assert!(!config.datadog.enabled); + assert!(!config.any_backend_enabled()); + } + + #[test] + fn master_switch_gates_every_backend() { + let mut config = InstrumentationConfig { + enabled: false, + ..Default::default() + }; + config.langfuse.enabled = true; + + assert!( + !config.any_backend_enabled(), + "master switch must override per-backend flags" + ); + + config.enabled = true; + assert!(config.any_backend_enabled()); + } + + #[test] + fn langfuse_captures_content_by_default_but_apms_do_not() { + // The asymmetry is the point: Langfuse is useless without content, + // an APM is actively harmed by it. + assert!(LangfuseSettings::default().capture_input); + assert!(LangfuseSettings::default().capture_tool_io); + assert_eq!( + OtlpSettings::default().content, + ContentCaptureMode::MetadataOnly + ); + assert_eq!( + DatadogSettings::default().content, + ContentCaptureMode::MetadataOnly + ); + } + + #[test] + fn apm_backends_do_not_emit_user_id_by_default() { + assert!(!OtlpSettings::default().emit_user_id); + } + + #[test] + fn secret_key_round_trips_through_toml() { + let toml = r#" + enabled = true + host = "https://self-hosted.example.com" + public_key = "pk-lf-1" + secret_key = "sk-lf-1" + "#; + let parsed: LangfuseSettings = toml::from_str(toml).expect("valid toml"); + + assert_eq!(parsed.host, "https://self-hosted.example.com"); + assert_eq!( + parsed.secret_key.as_ref().map(ExposeSecret::expose_secret), + Some(&"sk-lf-1".to_string()) + ); + } + + #[test] + fn debug_output_does_not_leak_the_secret_key() { + let settings = LangfuseSettings { + secret_key: Some(Secret::new("sk-lf-supersecret".to_string())), + ..Default::default() + }; + let rendered = format!("{settings:?}"); + + assert!( + !rendered.contains("sk-lf-supersecret"), + "secret leaked into Debug: {rendered}" + ); + } + + #[test] + fn debug_output_does_not_leak_otlp_header_values() { + let settings = OtlpSettings { + headers: std::collections::BTreeMap::from([( + "Authorization".into(), + "Bearer super-secret".into(), + )]), + ..Default::default() + }; + let rendered = format!("{settings:?}"); + + assert!(rendered.contains("Authorization")); + assert!(!rendered.contains("super-secret")); + } + + #[test] + fn datadog_defaults_to_the_local_agent_endpoint() { + // The overwhelmingly common deployment: Agent on localhost, no API key + // needed in Moltis at all. + let settings = DatadogSettings::default(); + assert_eq!(settings.endpoint, "http://localhost:4318/v1/traces"); + assert!(settings.api_key.is_none()); + } + + #[test] + fn content_mode_parses_every_documented_value() { + for value in ContentCaptureMode::ALL { + let toml = format!("content = \"{value}\""); + toml::from_str::(&toml) + .unwrap_or_else(|e| panic!("{value} should parse: {e}")); + } + } + + #[test] + fn partial_config_fills_in_defaults() { + // Operators write a two-line section; everything else must default. + let config: InstrumentationConfig = toml::from_str( + r#" + enabled = true + [langfuse] + enabled = true + public_key = "pk" + "#, + ) + .expect("valid toml"); + + assert!(config.enabled); + assert_eq!(config.sample_rate, 1.0); + assert_eq!(config.environment, "production"); + assert_eq!(config.langfuse.host, default_langfuse_host()); + assert_eq!(config.queue_capacity, 10_000); + } +} diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index 93e3f46ec7..4677892390 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -569,6 +569,64 @@ port = {port} # Port number (auto-generated for this i # enabled = true # Enable metrics collection # prometheus_endpoint = true # Expose /metrics endpoint +# ══════════════════════════════════════════════════════════════════════════════ +# INSTRUMENTATION (Langfuse / OpenTelemetry / Datadog) +# ══════════════════════════════════════════════════════════════════════════════ +# +# Exports completed agent runs — LLM calls, tool calls and retrievals — to an +# external backend. Observations are immutable and sent once, after completion. +# Disabled by default: enabling it sends conversation data to a third party. +# See docs/src/instrumentation.md. +# +# Backends deliberately receive different data. Langfuse gets the full +# conversation, token usage and session context for LLM observability and cost +# inference. Prompt Management, datasets, evaluators and media uploads are not +# integrated. OTLP and Datadog receive operational shape only. + +# [instrumentation] +# enabled = false # Master switch, gates every backend +# environment = "production" # Reported to every backend +# sample_rate = 1.0 # Fraction of turns traced (0.0-1.0) +# redact = ["customer_ref"] # Extra keys to redact; extends the defaults +# queue_capacity = 10000 # Must be nonzero; full queues drop events +# flush_interval_ms = 5000 # Must be nonzero +# max_batch_bytes = 3000000 # Must be nonzero + +# [instrumentation.langfuse] +# enabled = false +# host = "https://cloud.langfuse.com" # Or a self-hosted URL +# public_key = "pk-lf-..." +# Prefer MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY in the process environment. +# A secret_key config value is also accepted. +# capture_input = true # Turn and LLM inputs +# capture_output = true # Turn and LLM outputs +# capture_tool_io = true # Tool arguments and results +# timeout_secs = 10 # Must be nonzero + +# [instrumentation.otlp] # Grafana Tempo/Alloy, Honeycomb, a collector +# enabled = false +# endpoint = "http://localhost:4318/v1/traces" +# content = "metadata_only" # "full" | "metadata_only" | "none" +# emit_user_id = false # High-cardinality in an APM index +# timeout_secs = 10 # Must be nonzero + +# [instrumentation.datadog] # Via the Datadog Agent's OTLP intake +# enabled = false +# endpoint = "http://localhost:4318/v1/traces" +# service = "moltis" +# content = "metadata_only" +# timeout_secs = 10 # Must be nonzero + +# Reaction feedback. A thumbs up/down on a reply in Telegram, Discord or Slack +# becomes a BOOLEAN "user-feedback" score through Langfuse's dedicated Scores +# API. Score creates/replacements and deletions retain queue order. Lists accept +# raw emoji or shortcodes; empty means the built-in vocabulary. +# [instrumentation.feedback] +# enabled = true +# positive = ["\U0001F44D", "+1", "thumbsup"] +# negative = ["\U0001F44E", "-1", "thumbsdown"] +# link_retention_days = 30 # How long a reply stays attributable + # ══════════════════════════════════════════════════════════════════════════════ # CRON # ══════════════════════════════════════════════════════════════════════════════ diff --git a/crates/config/src/validate/schema_map.rs b/crates/config/src/validate/schema_map.rs index 6416856ea3..fb94b48377 100644 --- a/crates/config/src/validate/schema_map.rs +++ b/crates/config/src/validate/schema_map.rs @@ -509,6 +509,63 @@ pub(super) fn build_schema_map() -> KnownKeys { ("labels", Map(Box::new(Leaf))), ])), ), + ( + "instrumentation", + Struct(HashMap::from([ + ("enabled", Leaf), + ("environment", Leaf), + ("release", Leaf), + ("sample_rate", Leaf), + ("redact", Leaf), + ("queue_capacity", Leaf), + ("flush_interval_ms", Leaf), + ("max_batch_bytes", Leaf), + ( + "langfuse", + Struct(HashMap::from([ + ("enabled", Leaf), + ("host", Leaf), + ("public_key", Leaf), + ("secret_key", Leaf), + ("capture_input", Leaf), + ("capture_output", Leaf), + ("capture_tool_io", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ( + "otlp", + Struct(HashMap::from([ + ("enabled", Leaf), + ("endpoint", Leaf), + ("headers", Map(Box::new(Leaf))), + ("content", Leaf), + ("emit_user_id", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ( + "datadog", + Struct(HashMap::from([ + ("enabled", Leaf), + ("endpoint", Leaf), + ("api_key", Leaf), + ("service", Leaf), + ("content", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ( + "feedback", + Struct(HashMap::from([ + ("enabled", Leaf), + ("positive", Leaf), + ("negative", Leaf), + ("link_retention_days", Leaf), + ])), + ), + ])), + ), ( "identity", Struct(HashMap::from([ diff --git a/crates/config/src/validate/semantic.rs b/crates/config/src/validate/semantic.rs index 276c2bc513..4551f9e7e1 100644 --- a/crates/config/src/validate/semantic.rs +++ b/crates/config/src/validate/semantic.rs @@ -409,6 +409,8 @@ pub(super) fn check_semantic_warnings(config: &MoltisConfig, diagnostics: &mut V }); } + validate_instrumentation(&config.instrumentation, diagnostics); + for (name, server) in &config.mcp.servers { if server.request_timeout_secs == Some(0) { diagnostics.push(Diagnostic { @@ -987,6 +989,113 @@ pub(super) fn check_semantic_warnings(config: &MoltisConfig, diagnostics: &mut V } } +fn validate_instrumentation( + config: &crate::schema::InstrumentationConfig, + diagnostics: &mut Vec, +) { + if !config.sample_rate.is_finite() { + diagnostics.push(Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: "instrumentation.sample_rate".into(), + message: "instrumentation.sample_rate must be finite and between 0.0 and 1.0".into(), + }); + } else if !(0.0..=1.0).contains(&config.sample_rate) { + diagnostics.push(Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: "instrumentation.sample_rate".into(), + message: format!( + "instrumentation.sample_rate must be between 0.0 and 1.0 (got {})", + config.sample_rate + ), + }); + } + + let zero_values = [ + (config.queue_capacity == 0, "instrumentation.queue_capacity"), + ( + config.flush_interval_ms == 0, + "instrumentation.flush_interval_ms", + ), + ( + config.max_batch_bytes == 0, + "instrumentation.max_batch_bytes", + ), + ( + config.langfuse.timeout_secs == 0, + "instrumentation.langfuse.timeout_secs", + ), + ( + config.otlp.timeout_secs == 0, + "instrumentation.otlp.timeout_secs", + ), + ( + config.datadog.timeout_secs == 0, + "instrumentation.datadog.timeout_secs", + ), + ( + config.feedback.link_retention_days == 0, + "instrumentation.feedback.link_retention_days", + ), + ]; + diagnostics.extend( + zero_values + .into_iter() + .filter(|(is_zero, _)| *is_zero) + .map(|(_, path)| Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: path.into(), + message: format!("{path} must be at least 1"), + }), + ); + + let unusually_large = [ + ( + config.queue_capacity > 1_000_000, + "instrumentation.queue_capacity", + "a queue above 1,000,000 events can consume excessive memory", + ), + ( + config.flush_interval_ms > 300_000, + "instrumentation.flush_interval_ms", + "an interval above 5 minutes delays delivery and increases shutdown loss risk", + ), + ( + config.max_batch_bytes > 100_000_000, + "instrumentation.max_batch_bytes", + "a batch above 100 MB can exceed collector and proxy request limits", + ), + ( + config.langfuse.timeout_secs > 300, + "instrumentation.langfuse.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ( + config.otlp.timeout_secs > 300, + "instrumentation.otlp.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ( + config.datadog.timeout_secs > 300, + "instrumentation.datadog.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ]; + diagnostics.extend( + unusually_large + .into_iter() + .filter(|(is_large, ..)| *is_large) + .map(|(_, path, message)| Diagnostic { + severity: Severity::Warning, + category: "invalid-value", + path: path.into(), + message: message.into(), + }), + ); +} + /// Validate a `context_window` override value (optional field). fn validate_context_window(value: Option, path: &str, diagnostics: &mut Vec) { let Some(cw) = value else { diff --git a/crates/config/src/validate/tests.rs b/crates/config/src/validate/tests.rs index ab741b24d5..acdbe0fb40 100644 --- a/crates/config/src/validate/tests.rs +++ b/crates/config/src/validate/tests.rs @@ -13,6 +13,8 @@ mod channels; mod common; #[path = "tests/defaults.rs"] mod defaults; +#[path = "tests/instrumentation.rs"] +mod instrumentation; #[path = "tests/memory.rs"] mod memory; #[path = "tests/providers.rs"] diff --git a/crates/config/src/validate/tests/instrumentation.rs b/crates/config/src/validate/tests/instrumentation.rs new file mode 100644 index 0000000000..07c1eb33bd --- /dev/null +++ b/crates/config/src/validate/tests/instrumentation.rs @@ -0,0 +1,104 @@ +use std::collections::BTreeSet; + +use super::*; + +fn diagnostics_for(toml: &str, severity: Severity) -> BTreeSet { + validate_toml_str(toml) + .diagnostics + .into_iter() + .filter(|diagnostic| diagnostic.severity == severity) + .map(|diagnostic| diagnostic.path) + .filter(|path| path.starts_with("instrumentation.")) + .collect() +} + +#[test] +fn sample_rate_must_be_finite_and_in_range() { + for sample_rate in ["nan", "inf", "-inf", "-0.01", "1.01"] { + let result = + validate_toml_str(&format!("[instrumentation]\nsample_rate = {sample_rate}\n")); + let diagnostic = result.diagnostics.iter().find(|diagnostic| { + diagnostic.severity == Severity::Error + && diagnostic.path == "instrumentation.sample_rate" + }); + + assert!( + diagnostic.is_some(), + "sample_rate {sample_rate} should be rejected: {:?}", + result.diagnostics + ); + } + + for sample_rate in ["0.0", "0.25", "1.0"] { + let errors = diagnostics_for( + &format!("[instrumentation]\nsample_rate = {sample_rate}\n"), + Severity::Error, + ); + assert!( + !errors.contains("instrumentation.sample_rate"), + "sample_rate {sample_rate} should be accepted" + ); + } +} + +#[test] +fn batching_and_backend_timeouts_must_be_nonzero() { + let errors = diagnostics_for( + r#" +[instrumentation] +queue_capacity = 0 +flush_interval_ms = 0 +max_batch_bytes = 0 + +[instrumentation.langfuse] +timeout_secs = 0 + +[instrumentation.otlp] +timeout_secs = 0 + +[instrumentation.datadog] +timeout_secs = 0 + +[instrumentation.feedback] +link_retention_days = 0 +"#, + Severity::Error, + ); + + assert_eq!( + errors, + BTreeSet::from([ + "instrumentation.datadog.timeout_secs".to_string(), + "instrumentation.flush_interval_ms".to_string(), + "instrumentation.feedback.link_retention_days".to_string(), + "instrumentation.langfuse.timeout_secs".to_string(), + "instrumentation.max_batch_bytes".to_string(), + "instrumentation.otlp.timeout_secs".to_string(), + "instrumentation.queue_capacity".to_string(), + ]) + ); +} + +#[test] +fn operationally_extreme_delivery_settings_are_warned() { + let warnings = diagnostics_for( + r#" +[instrumentation] +queue_capacity = 1000001 +flush_interval_ms = 300001 +max_batch_bytes = 100000001 + +[instrumentation.langfuse] +timeout_secs = 301 + +[instrumentation.otlp] +timeout_secs = 301 + +[instrumentation.datadog] +timeout_secs = 301 +"#, + Severity::Warning, + ); + + assert_eq!(warnings.len(), 6, "unexpected warnings: {warnings:?}"); +} diff --git a/crates/discord/src/handler.rs b/crates/discord/src/handler.rs index 25dcf5c430..94a6d7345d 100644 --- a/crates/discord/src/handler.rs +++ b/crates/discord/src/handler.rs @@ -2,5 +2,7 @@ #[path = "handler/implementation.rs"] mod implementation; +#[path = "handler/reactions.rs"] +mod reactions; pub use self::implementation::*; diff --git a/crates/discord/src/handler/implementation.rs b/crates/discord/src/handler/implementation.rs index 08f8759f3f..debae7d002 100644 --- a/crates/discord/src/handler/implementation.rs +++ b/crates/discord/src/handler/implementation.rs @@ -657,6 +657,14 @@ fn set_bot_presence(ctx: &Context, account_id: &str, config: &DiscordAccountConf #[async_trait] impl EventHandler for Handler { + async fn reaction_add(&self, ctx: Context, reaction: serenity::all::Reaction) { + self.emit_reaction_change(&ctx, &reaction, true).await; + } + + async fn reaction_remove(&self, ctx: Context, reaction: serenity::all::Reaction) { + self.emit_reaction_change(&ctx, &reaction, false).await; + } + async fn message(&self, ctx: Context, msg: Message) { // Ignore messages from bots (including ourselves). if msg.author.bot { @@ -1362,12 +1370,28 @@ pub async fn send_discord_message( text: &str, reference: Option, ) -> Result { + let mut sent = send_discord_message_all(http, channel_id, text, reference).await?; + // Always `Some` because `text` is non-empty and every chunk is sent. + sent.pop().ok_or_else(|| "no chunks produced".into()) +} + +/// Send `text`, returning every message the send produced. +/// +/// Long replies are split across several Discord messages and a reader may +/// react to any of them, so feedback attribution needs all the ids rather than +/// just the last. +pub async fn send_discord_message_all( + http: &serenity::http::Http, + channel_id: serenity::all::ChannelId, + text: &str, + reference: Option, +) -> Result, String> { if text.is_empty() { return Err("empty message".into()); } let chunks = chunk_message(text, 2000); - let mut last_msg = None; + let mut sent = Vec::with_capacity(chunks.len()); for (i, chunk) in chunks.iter().enumerate() { let mut create = CreateMessage::new().content(*chunk); // Only the first chunk gets the reply reference. @@ -1376,15 +1400,14 @@ pub async fn send_discord_message( { create = create.reference_message((channel_id, ref_id)); } - last_msg = Some( + sent.push( channel_id .send_message(http, create) .await .map_err(|e| format!("Discord send: {e}"))?, ); } - // `last_msg` is always `Some` because `text` is non-empty. - last_msg.ok_or_else(|| "no chunks produced".into()) + Ok(sent) } /// Split a message into chunks of at most `max_len` characters. diff --git a/crates/discord/src/handler/reactions.rs b/crates/discord/src/handler/reactions.rs new file mode 100644 index 0000000000..6302add041 --- /dev/null +++ b/crates/discord/src/handler/reactions.rs @@ -0,0 +1,113 @@ +//! Inbound Discord reaction handling. +//! +//! Split from the main handler so both files stay inside the file-size limit. + +use serenity::all::{Context, Reaction, ReactionType}; + +use { + moltis_channels::{ChannelEvent, ChannelType}, + moltis_common::types::ChatType, +}; + +use {super::implementation::Handler, crate::access}; + +impl Handler { + /// Surface a reaction change so the gateway can score it as feedback. + /// + /// The bot's own acknowledgement reactions are skipped: it marks incoming + /// messages with 👀/✅/❌, and treating those as user opinions would score + /// every turn the bot itself handled. + pub(super) async fn emit_reaction_change( + &self, + ctx: &Context, + reaction: &Reaction, + added: bool, + ) { + let Some(user_id) = reaction.user_id else { + return; + }; + if user_id == ctx.cache.current_user().id { + return; + } + + let (config, sink) = { + let accts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); + match accts.get(&self.account_id) { + Some(state) => (state.config.clone(), state.event_sink.clone()), + None => return, + } + }; + let Some(sink) = sink else { + return; + }; + + let chat_type = if reaction.guild_id.is_some() { + ChatType::Group + } else { + ChatType::Dm + }; + let guild_id = reaction.guild_id.map(|id| id.to_string()); + let username = if added { + reaction.user(&ctx.http).await.ok().map(|user| user.name) + } else { + None + }; + let basic_access = !added + || access::check_access( + &config, + &chat_type, + &user_id.to_string(), + username.as_deref(), + guild_id.as_deref(), + // A reaction is intentional interaction and does not need a fresh + // mention, while the account and guild allowlists still apply. + true, + ) + .is_ok(); + let filter_active = + !config.channel_name_patterns.is_empty() || !config.category_allowlist.is_empty(); + let channel_allowed = if !added || !filter_active { + true + } else { + reaction + .guild_id + .and_then(|guild_id| { + ctx.cache.guild(guild_id).and_then(|guild| { + guild.channels.get(&reaction.channel_id).map(|channel| { + access::channel_matches_filter( + &config, + Some(&channel.name), + channel.parent_id.map(|id| id.to_string()).as_deref(), + ) + }) + }) + }) + .unwrap_or(false) + }; + if !basic_access || !channel_allowed { + return; + } + + // Custom guild emoji have no unicode form; their name is what a + // feedback vocabulary can match against. + let emoji = match &reaction.emoji { + ReactionType::Unicode(raw) => raw.clone(), + ReactionType::Custom { name, .. } => name.clone().unwrap_or_default(), + _ => return, + }; + if emoji.is_empty() { + return; + } + + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Discord, + account_id: self.account_id.clone(), + chat_id: reaction.channel_id.to_string(), + message_id: reaction.message_id.to_string(), + user_id: user_id.to_string(), + emoji, + added, + }) + .await; + } +} diff --git a/crates/discord/src/outbound.rs b/crates/discord/src/outbound.rs index 6ef8ca9eb4..50722c8887 100644 --- a/crates/discord/src/outbound.rs +++ b/crates/discord/src/outbound.rs @@ -23,7 +23,7 @@ use { }; use crate::{ - handler::{send_discord_message, send_discord_text}, + handler::{send_discord_message, send_discord_message_all}, state::AccountStateMap, }; @@ -336,9 +336,9 @@ impl DiscordOutbound { channel_id: ChannelId, suffix_html: &str, reply_to: Option<&str>, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let Some(rendered) = render_activity_log_for_discord(suffix_html) else { - return Ok(()); + return Ok(None); }; let color: u32 = if rendered.has_errors { @@ -355,14 +355,19 @@ impl DiscordOutbound { msg = msg.reference_message((channel_id, reference)); } - channel_id.send_message(http, msg).await.map_err(|e| { + let sent = channel_id.send_message(http, msg).await.map_err(|e| { ChannelError::external("Discord send embed", std::io::Error::other(e.to_string())) })?; - Ok(()) + Ok(Some(sent.id)) } /// Inner implementation for `send_text_with_suffix` that does not handle /// ack reaction removal -- the caller is responsible for that. + /// + /// Returns every message id the reply produced, the activity-log embed + /// included: a reader rating "the bot's answer" may land on any of them, + /// and each maps back to the same turn, so being forgiving here costs + /// nothing and recovers scores that would otherwise be dropped. async fn send_text_with_suffix_inner( &self, account_id: &str, @@ -371,18 +376,21 @@ impl DiscordOutbound { text: &str, suffix_html: &str, reply_to: Option<&str>, - ) -> ChannelResult<()> { + ) -> ChannelResult> { // Send main response text. let reference = self.resolve_reference(account_id, reply_to); - send_discord_message(http, channel_id, text, reference) + let sent = send_discord_message_all(http, channel_id, text, reference) .await .map_err(|e| ChannelError::external("Discord send", std::io::Error::other(e)))?; // Send the activity log as a separate embed message. - self.send_activity_log_embed(account_id, http, channel_id, suffix_html, None) + let embed_id = self + .send_activity_log_embed(account_id, http, channel_id, suffix_html, None) .await?; - Ok(()) + let mut ids: Vec = sent.into_iter().map(|m| m.id.to_string()).collect(); + ids.extend(embed_id.map(|id| id.to_string())); + Ok(ids) } } @@ -419,6 +427,31 @@ impl ChannelOutbound for DiscordOutbound { Ok(()) } + /// Discord splits long replies across messages, so every id is reported. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let http = self.resolve_http(account_id)?; + let channel_id = Self::parse_channel_id(to)?; + let reference = self.resolve_reference(account_id, reply_to); + let sent = send_discord_message_all(&http, channel_id, text, reference) + .await + .map_err(|e| ChannelError::external("Discord send", std::io::Error::other(e)))?; + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + moltis_metrics::channels::MESSAGES_SENT_TOTAL, + moltis_metrics::labels::CHANNEL => "discord" + ) + .increment(1); + + Ok(sent.into_iter().map(|m| m.id.to_string()).collect()) + } + async fn send_media( &self, account_id: &str, @@ -541,6 +574,19 @@ impl ChannelOutbound for DiscordOutbound { suffix_html: &str, reply_to: Option<&str>, ) -> ChannelResult<()> { + self.send_text_with_suffix_reporting_ids(account_id, to, text, suffix_html, reply_to) + .await?; + Ok(()) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; @@ -573,7 +619,23 @@ impl ChannelOutbound for DiscordOutbound { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; self.send_activity_log_embed(account_id, &http, channel_id, html, reply_to) - .await + .await?; + Ok(()) + } + + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let http = self.resolve_http(account_id)?; + let channel_id = Self::parse_channel_id(to)?; + let id = self + .send_activity_log_embed(account_id, &http, channel_id, html, reply_to) + .await?; + Ok(id.map(|id| id.to_string()).into_iter().collect()) } async fn send_location( @@ -666,15 +728,19 @@ fn truncate_at_char_boundary(s: &str, max: usize) -> &str { &s[..end] } -#[async_trait] -impl ChannelStreamOutbound for DiscordOutbound { - async fn send_stream( +impl DiscordOutbound { + /// Drive an edit-in-place stream, returning the ids of the messages the + /// final reply ended up occupying. + /// + /// Both trait entry points share this so the streaming state machine + /// exists once; [`ChannelStreamOutbound::send_stream`] discards the ids. + async fn send_stream_ids( &self, account_id: &str, to: &str, reply_to: Option<&str>, mut stream: StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; let reference = self.resolve_reference(account_id, reply_to); @@ -758,6 +824,9 @@ impl ChannelStreamOutbound for DiscordOutbound { } } + // Ids of messages this stream created besides the edited one. + let mut extra_ids: Vec = Vec::new(); + // Phase 3: final update with the complete text. if !accumulated.is_empty() { if accumulated.len() <= DISCORD_MAX_MESSAGE_LEN { @@ -773,11 +842,12 @@ impl ChannelStreamOutbound for DiscordOutbound { ); } } else { - send_discord_message(&http, channel_id, &accumulated, None) + let msg = send_discord_message(&http, channel_id, &accumulated, None) .await .map_err(|e| { ChannelError::external("Discord send", std::io::Error::other(e)) })?; + extra_ids.push(msg.id.to_string()); } } else { // Content overflows -- edit the first message with the first 2000 chars, @@ -786,13 +856,15 @@ impl ChannelStreamOutbound for DiscordOutbound { if let Some(msg_id) = sent_message_id { let edit = EditMessage::new().content(first); let _ = channel_id.edit_message(&http, msg_id, edit).await; - } else { - let _ = send_discord_text(&http, channel_id, first).await; + } else if let Ok(sent) = + send_discord_message_all(&http, channel_id, first, None).await + { + extra_ids.extend(sent.into_iter().map(|m| m.id.to_string())); } let rest = &accumulated[first.len()..]; if !rest.is_empty() { - send_discord_text(&http, channel_id, rest) + let sent = send_discord_message_all(&http, channel_id, rest, None) .await .map_err(|e| { ChannelError::external( @@ -800,6 +872,7 @@ impl ChannelStreamOutbound for DiscordOutbound { std::io::Error::other(e), ) })?; + extra_ids.extend(sent.into_iter().map(|m| m.id.to_string())); } } } @@ -815,9 +888,37 @@ impl ChannelStreamOutbound for DiscordOutbound { streamed = sent_message_id.is_some(), "discord stream completed" ); + let mut ids = Vec::with_capacity(extra_ids.len() + 1); + ids.extend(sent_message_id.map(|id| id.to_string())); + ids.extend(extra_ids); + Ok(ids) + } +} + +#[async_trait] +impl ChannelStreamOutbound for DiscordOutbound { + async fn send_stream( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> ChannelResult<()> { + self.send_stream_ids(account_id, to, reply_to, stream) + .await?; Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> ChannelResult> { + self.send_stream_ids(account_id, to, reply_to, stream).await + } + async fn is_stream_enabled(&self, _account_id: &str) -> bool { true } diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index e04ad842d9..a4368292bc 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -49,6 +49,7 @@ moltis-netbird = { optional = true, workspace = true } moltis-network-filter = { features = ["proxy", "service"], optional = true, workspace = true } moltis-nostr = { optional = true, workspace = true } moltis-oauth = { workspace = true } +moltis-observability = { workspace = true } moltis-onboarding = { workspace = true } moltis-openclaw-import = { optional = true, workspace = true } moltis-plugins = { workspace = true } @@ -126,6 +127,7 @@ default = [ "graphql", "hermes-import", "home-assistant", + "langfuse", "local-llm", "local-llm-metal", "matrix", @@ -135,6 +137,7 @@ default = [ "netbird", "nostr", "openclaw-import", + "otlp", "prometheus", "provider-github-copilot", "provider-kimi-code", @@ -166,6 +169,7 @@ fs-tools = ["moltis-tools/fs-tools"] graphql = [] hermes-import = ["dep:moltis-hermes-import"] home-assistant = ["dep:moltis-home-assistant"] +langfuse = ["moltis-observability/langfuse"] llm-compaction = ["moltis-chat/llm-compaction"] local-embeddings = ["moltis-memory/local-embeddings"] local-llm = [ @@ -186,12 +190,14 @@ metrics = [ "moltis-metrics/sqlite", "moltis-msteams?/metrics", "moltis-nostr?/metrics", + "moltis-observability/metrics", "moltis-signal?/metrics", ] msteams = ["dep:moltis-msteams"] netbird = ["dep:moltis-netbird"] nostr = ["dep:moltis-nostr"] openclaw-import = ["dep:moltis-openclaw-import"] +otlp = ["moltis-observability/otlp"] prometheus = ["metrics", "moltis-metrics/prometheus"] provider-github-copilot = [ "moltis-provider-setup/provider-github-copilot", diff --git a/crates/gateway/migrations/20260726120000_trace_links.sql b/crates/gateway/migrations/20260726120000_trace_links.sql new file mode 100644 index 0000000000..739df53226 --- /dev/null +++ b/crates/gateway/migrations/20260726120000_trace_links.sql @@ -0,0 +1,19 @@ +-- Correlates a delivered reply with the trace that produced it, so a reaction +-- arriving later can be scored against the right agent run. +-- +-- Identifiers only: message bodies stay in message_log rather than being +-- duplicated here. +CREATE TABLE IF NOT EXISTS trace_links ( + channel_type TEXT NOT NULL, + account_id TEXT NOT NULL, + chat_id TEXT NOT NULL, + message_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + session_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (channel_type, account_id, chat_id, message_id) +); + +-- Pruning scans by age. +CREATE INDEX IF NOT EXISTS idx_trace_links_created + ON trace_links (created_at); diff --git a/crates/gateway/src/channel_events/sink.rs b/crates/gateway/src/channel_events/sink.rs index 563763dcda..85d2672897 100644 --- a/crates/gateway/src/channel_events/sink.rs +++ b/crates/gateway/src/channel_events/sink.rs @@ -14,11 +14,50 @@ impl GatewayChannelEventSink { } } +/// Route a reaction change into the feedback pipeline. +/// +/// Runs before the broadcast so a slow WebSocket client cannot delay scoring. +async fn handle_reaction_feedback(state: &Arc, event: &ChannelEvent) { + let ChannelEvent::ReactionChange { + channel_type, + account_id, + chat_id, + message_id, + user_id, + emoji, + added, + } = event + else { + return; + }; + + let outcome = state + .feedback + .on_reaction( + channel_type.as_str(), + account_id, + chat_id, + message_id, + emoji, + user_id, + *added, + state.instrumentation.scores_available(), + ) + .await; + debug!( + channel = channel_type.as_str(), + ?outcome, + "reaction feedback handled" + ); +} + pub(super) async fn emit( state: &Arc>>, event: ChannelEvent, ) { if let Some(state) = state.get() { + handle_reaction_feedback(state, &event).await; + let payload = match serde_json::to_value(&event) { Ok(v) => v, Err(e) => { diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index 42c9b5d106..e8713eff3d 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -39,6 +39,10 @@ impl ChatRuntime for GatewayChatRuntime { .await; } + fn feedback(&self) -> Option> { + Some(Arc::clone(&self.state.feedback)) + } + // ── Channel reply queue ───────────────────────────────────────────────── async fn push_channel_reply(&self, session_key: &str, target: ChannelReplyTarget) { diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 5d1a72ca89..6dc8e12ac3 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -58,6 +58,7 @@ pub mod state; pub mod tailscale; #[cfg(feature = "msteams")] pub mod teams_agent_tools; +pub mod trace_link_store; pub mod tts_phrases; pub mod update_check; pub mod updater; diff --git a/crates/gateway/src/methods/dispatch.rs b/crates/gateway/src/methods/dispatch.rs index 4d0c8f95b1..0fc2b4f640 100644 --- a/crates/gateway/src/methods/dispatch.rs +++ b/crates/gateway/src/methods/dispatch.rs @@ -95,6 +95,8 @@ const READ_METHODS: &[&str] = &[ "webhooks.delivery.get", "webhooks.delivery.payload", "webhooks.delivery.actions", + "feedback.status", + "instrumentation.status", "heartbeat.status", "heartbeat.runs", "system-presence", @@ -255,6 +257,9 @@ const WRITE_METHODS: &[&str] = &[ "webhooks.create", "webhooks.update", "webhooks.delete", + // Writes a score against a trace: a mutation, not a read. + "feedback.submit", + "instrumentation.test", "heartbeat.update", "heartbeat.run", "voice.config.save_key", @@ -500,6 +505,22 @@ mod tests { ); } + #[test] + fn feedback_status_requires_read_but_submit_requires_write() { + // Submitting writes a score against a trace. A read-scoped client must + // not be able to record or retract one. + assert!( + authorize_method("feedback.status", "operator", &scopes(&["operator.read"])).is_none() + ); + assert_error_code( + authorize_method("feedback.submit", "operator", &scopes(&["operator.read"])), + "UNAUTHORIZED", + ); + assert!( + authorize_method("feedback.submit", "operator", &scopes(&["operator.write"])).is_none() + ); + } + #[test] fn senders_approve_requires_write() { assert!( @@ -755,6 +776,54 @@ mod tests { ); } + /// Guards the fix end to end rather than only at `authorize_method`: the + /// method was reachable with `operator.read` while writing a score, so a + /// read-scoped client could forge or retract feedback. + #[test] + fn feedback_submit_is_rejected_for_read_scoped_clients() { + use crate::{ + auth::{AuthMode, ResolvedAuth}, + services::GatewayServices, + state::GatewayState, + }; + + let reg = MethodRegistry::new(); + let ctx = MethodContext { + request_id: "test".into(), + method: "feedback.submit".into(), + params: serde_json::json!({ + "sessionKey": "someone-elses-session", + "messageId": "run-1", + "signal": "positive", + // Ignored by the handler, which attributes to the + // authenticated operator; present here to prove the request + // cannot choose whose vote it writes. + "userId": "victim", + }), + client_conn_id: "conn-1".into(), + client_role: "operator".into(), + client_scopes: scopes(&["operator.read"]), + state: GatewayState::new( + ResolvedAuth { + mode: AuthMode::Token, + token: None, + password: None, + }, + GatewayServices::noop(), + ), + channel: None, + }; + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime"); + let resp = rt.block_on(reg.dispatch(ctx)); + assert!(!resp.ok); + assert_eq!( + resp.error.as_ref().map(|e| e.code.as_str()), + Some("UNAUTHORIZED") + ); + } + #[test] fn chat_send_sync_is_registered_and_authorized() { let reg = MethodRegistry::new(); diff --git a/crates/gateway/src/methods/services.rs b/crates/gateway/src/methods/services.rs index 450901cf4f..dae836c50c 100644 --- a/crates/gateway/src/methods/services.rs +++ b/crates/gateway/src/methods/services.rs @@ -506,6 +506,8 @@ mod admin; mod agents; mod channels; mod core; +mod feedback; +mod instrumentation; mod modes; mod sessions; mod system; @@ -521,6 +523,8 @@ pub(super) fn register(reg: &mut MethodRegistry) { core::register(reg); system::register(reg); admin::register(reg); + feedback::register(reg); + instrumentation::register(reg); #[cfg(feature = "voice")] voice_personas::register(reg); voicecall::register(reg); diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs new file mode 100644 index 0000000000..05d774ddd4 --- /dev/null +++ b/crates/gateway/src/methods/services/feedback.rs @@ -0,0 +1,137 @@ +//! `feedback.*` RPC methods backing the thumbs up/down control in the web UI. +//! +//! The web is treated as one more channel: a reply gets a link keyed on the +//! session and the message index, and a thumb resolves that link the same way +//! a Telegram reaction does. +//! +//! `feedback.status` is a read. `feedback.submit` writes a score and is +//! authorized as a write, with the scoring identity taken from the +//! authenticated operator rather than from the request. + +use { + moltis_channels::FeedbackOutcome, + moltis_observability::FeedbackSignal, + moltis_protocol::{ErrorShape, error_codes}, +}; + +use super::MethodRegistry; + +/// Register the `feedback.*` namespace. +pub(super) fn register(reg: &mut MethodRegistry) { + reg.register( + "feedback.status", + Box::new(|ctx| { + Box::pin(async move { + let config = &ctx.state.config.instrumentation; + Ok(serde_json::json!({ + // The control is only useful when something is collecting + // the result, so the UI hides it unless both are on. + "enabled": config.enabled + && config.feedback.enabled + && ctx.state.instrumentation.scores_available(), + "instrumentation_active": ctx.state.instrumentation.status().active, + "retention_days": config.feedback.link_retention_days, + })) + }) + }), + ); + + reg.register( + "feedback.submit", + Box::new(|ctx| { + Box::pin(async move { + let session_key = ctx + .params + .get("sessionKey") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + if session_key.is_empty() { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + "sessionKey is required", + )); + } + + let message_id = ctx + .params + .get("messageId") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + if message_id.is_empty() { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + "messageId is required", + )); + } + + // `signal` rather than a raw score so the wire format cannot + // smuggle an arbitrary value into the metric. + let signal = ctx.params.get("signal").and_then(|v| v.as_str()); + let signal = match signal { + Some("positive") => Some(FeedbackSignal::Positive), + Some("negative") => Some(FeedbackSignal::Negative), + // Clicking an active thumb clears it. + Some("clear") => None, + other => { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + format!( + "signal must be \"positive\", \"negative\" or \"clear\", got {other:?}" + ), + )); + }, + }; + + // The score is attributed to the authenticated operator, never + // to an identity named in the request: a caller-chosen `userId` + // would let one client overwrite or retract another's vote, + // since the score id is derived from (trace, name, user). A + // stable value is required for that upsert to keep working + // across reconnects, which also rules out the connection id. + // + // The web surface has exactly one identity because Moltis has + // exactly one account: a single password, no user table, and + // passkeys registered as "owner". `AuthIdentity` carries only a + // method and scopes, so there is no per-person id to derive + // here — everyone who authenticates *is* the owner, and one + // constant states that honestly rather than manufacturing a + // per-connection identity that would break the toggle. + // + // If Moltis ever grows real accounts, this is the line that has + // to change, and the score id derivation will follow. + let user_id = moltis_channels::trace_link::WEB_CHANNEL; + + let outcome = ctx + .state + .feedback + .submit_signal( + moltis_channels::trace_link::WEB_CHANNEL, + moltis_channels::trace_link::WEB_CHANNEL, + session_key, + message_id, + signal, + user_id, + Some("web feedback".to_string()), + ctx.state.instrumentation.scores_available(), + ) + .await; + + Ok(serde_json::json!({ + "ok": matches!( + outcome, + FeedbackOutcome::Recorded(_) | FeedbackOutcome::Retracted + ), + "outcome": match outcome { + FeedbackOutcome::Recorded(_) => "recorded", + FeedbackOutcome::Retracted => "retracted", + FeedbackOutcome::NotFeedback => "not_feedback", + // The turn is older than the retention window, or + // predates instrumentation being switched on. + FeedbackOutcome::UnknownMessage => "unknown_message", + FeedbackOutcome::Disabled => "disabled", + }, + })) + }) + }), + ); +} diff --git a/crates/gateway/src/methods/services/instrumentation.rs b/crates/gateway/src/methods/services/instrumentation.rs new file mode 100644 index 0000000000..cf553a9de8 --- /dev/null +++ b/crates/gateway/src/methods/services/instrumentation.rs @@ -0,0 +1,234 @@ +//! `instrumentation.*` RPC methods backing the Instrumentation settings page. + +use moltis_protocol::{ErrorShape, error_codes}; + +use super::MethodRegistry; + +/// Register the `instrumentation.*` namespace. +pub(super) fn register(reg: &mut MethodRegistry) { + reg.register( + "instrumentation.status", + Box::new(|ctx| { + Box::pin(async move { + let status = ctx.state.instrumentation.status(); + let config = &ctx.state.config.instrumentation; + + // The secret key is deliberately absent: the UI shows whether + // one is configured, never its value. + serde_json::to_value(serde_json::json!({ + "active": status.active, + "backends": status.backends, + "skipped": status.skipped, + "delivery": status.delivery, + "config": { + "enabled": config.enabled, + "environment": config.environment, + "sample_rate": config.sample_rate, + "queue_capacity": config.queue_capacity, + "flush_interval_ms": config.flush_interval_ms, + "max_batch_bytes": config.max_batch_bytes, + "langfuse": { + "enabled": config.langfuse.enabled, + "host": config.langfuse.host, + "public_key": config.langfuse.public_key, + "secret_key_set": config.langfuse.secret_key.is_some(), + "capture_input": config.langfuse.capture_input, + "capture_output": config.langfuse.capture_output, + "capture_tool_io": config.langfuse.capture_tool_io, + "timeout_secs": config.langfuse.timeout_secs, + }, + "otlp": { + "enabled": config.otlp.enabled, + "endpoint": config.otlp.endpoint, + "content": config.otlp.content, + "emit_user_id": config.otlp.emit_user_id, + "timeout_secs": config.otlp.timeout_secs, + }, + "datadog": { + "enabled": config.datadog.enabled, + "endpoint": config.datadog.endpoint, + "service": config.datadog.service, + "api_key_set": config.datadog.api_key.is_some(), + "content": config.datadog.content, + "timeout_secs": config.datadog.timeout_secs, + }, + }, + })) + .map_err(|e| ErrorShape::new(error_codes::INTERNAL, e.to_string())) + }) + }), + ); + + reg.register( + "instrumentation.test", + Box::new(|ctx| { + Box::pin(async move { + let backend = ctx + .params + .get("backend") + .and_then(|v| v.as_str()) + .unwrap_or("langfuse"); + + match backend { + #[cfg(feature = "langfuse")] + "langfuse" => { + let Some(client) = ctx.state.instrumentation.langfuse() else { + return Ok(serde_json::json!({ + "ok": false, + "error": "Langfuse is not enabled, or it failed to start. \ + Save valid credentials first.", + })); + }; + match client.test_connection().await { + Ok(()) => Ok(serde_json::json!({ "ok": true })), + Err(error) => Ok(serde_json::json!({ + "ok": false, + "error": error.to_string(), + })), + } + }, + // Say so rather than falling through to the generic "no test + // available" answer, which would read as a Langfuse quirk. + #[cfg(not(feature = "langfuse"))] + "langfuse" => Ok(serde_json::json!({ + "ok": false, + "error": "this build was compiled without the `langfuse` feature", + })), + // OTLP collectors have no standard health endpoint, and + // POSTing a probe span would pollute the operator's traces + // with fake data. Live delivery counters are the honest + // signal once the exporter has received real events. + other => Ok(serde_json::json!({ + "ok": false, + "error": format!( + "no connection test available for `{other}`; check its \ + delivery status after the next agent run" + ), + })), + } + }) + }), + ); +} + +#[cfg(test)] +mod tests { + use { + super::*, + crate::{ + auth::{AuthMode, ResolvedAuth}, + methods::MethodContext, + services::GatewayServices, + state::GatewayState, + }, + moltis_config::{InstrumentationConfig, LangfuseSettings}, + }; + + async fn dispatch( + state: std::sync::Arc, + method: &str, + params: serde_json::Value, + ) -> serde_json::Value { + let mut registry = MethodRegistry::default(); + register(&mut registry); + let response = registry + .dispatch(MethodContext { + request_id: "test".into(), + method: method.into(), + params, + client_conn_id: "conn-1".into(), + client_role: "operator".into(), + client_scopes: vec!["operator.read".into(), "operator.write".into()], + state, + channel: None, + }) + .await; + + assert!(response.ok, "method failed: {:?}", response.error); + response + .payload + .unwrap_or_else(|| panic!("{method} returned no payload")) + } + + fn gateway_state() -> std::sync::Arc { + GatewayState::new( + ResolvedAuth { + mode: AuthMode::Token, + token: None, + password: None, + }, + GatewayServices::noop(), + ) + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn status_preserves_the_latest_skipped_backend_failure() { + let state = gateway_state(); + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + ..Default::default() + }, + ..Default::default() + }; + state.instrumentation.apply(&config, "test"); + + let payload = dispatch(state, "instrumentation.status", serde_json::json!({})).await; + + assert_eq!(payload["active"], false); + assert_eq!(payload["skipped"][0]["name"], "langfuse"); + assert_eq!(payload["config"]["flush_interval_ms"], 5_000); + assert_eq!(payload["config"]["max_batch_bytes"], 3_000_000); + assert_eq!(payload["config"]["langfuse"]["timeout_secs"], 10); + assert!( + payload["skipped"][0]["reason"] + .as_str() + .is_some_and(|reason| reason.contains("public_key")) + ); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn unsupported_connection_test_points_to_live_delivery_health() { + let payload = dispatch( + gateway_state(), + "instrumentation.test", + serde_json::json!({ "backend": "otlp" }), + ) + .await; + + assert_eq!(payload["ok"], false); + let error = payload["error"].as_str().unwrap_or_default(); + assert!(error.contains("delivery status")); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn status_surfaces_each_exporter_delivery_path() { + let state = gateway_state(); + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + public_key: "pk-test".into(), + secret_key: Some(secrecy::Secret::new("sk-test".to_string())), + ..Default::default() + }, + ..Default::default() + }; + state.instrumentation.apply(&config, "test"); + + let payload = dispatch(state, "instrumentation.status", serde_json::json!({})).await; + + assert_eq!(payload["delivery"][0]["name"], "langfuse"); + assert_eq!(payload["delivery"][0]["accepted"], 0); + assert_eq!(payload["delivery"][0]["delivered"], 0); + assert_eq!(payload["delivery"][0]["dropped_queue_full"], 0); + assert_eq!(payload["delivery"][0]["dropped_failed"], 0); + assert_eq!(payload["delivery"][0]["retries"], 0); + assert_eq!(payload["delivery"][1]["name"], "langfuse-scores"); + moltis_observability::clear_global_sink(); + } +} diff --git a/crates/gateway/src/server/instrumentation.rs b/crates/gateway/src/server/instrumentation.rs new file mode 100644 index 0000000000..5adf9e1715 --- /dev/null +++ b/crates/gateway/src/server/instrumentation.rs @@ -0,0 +1,519 @@ +//! Startup wiring for agent instrumentation. +//! +//! Builds every configured backend and installs the resulting fanout as the +//! process-wide sink, which the agent runner then discovers without any +//! plumbing through its call signatures. + +use std::{ + sync::{Arc, RwLock}, + time::Duration, +}; + +use { + moltis_config::InstrumentationConfig, + moltis_observability::{ObservationSink, SinkStatsSnapshot, SkippedBackend}, + tracing::{info, warn}, +}; + +#[cfg(feature = "langfuse")] +use moltis_observability::exporters::langfuse::LangfuseClient; + +/// Live instrumentation state, surfaced by the `instrumentation.*` RPC methods. +#[derive(Default)] +pub struct InstrumentationState { + inner: RwLock, +} + +/// The latest applied outcome, including failures when no sink could start. +#[derive(Default)] +struct InstrumentationSnapshot { + sink: Option>, + backends: Vec, + skipped: Vec, + scores: bool, + #[cfg(feature = "langfuse")] + langfuse: Option>, +} + +/// Status reported to the settings UI. +#[derive(Debug, Clone, serde::Serialize)] +pub struct InstrumentationStatus { + /// Whether any backend is actively exporting. + pub active: bool, + /// Backends that are running. + pub backends: Vec, + /// Backends that were enabled in config but could not start, with reasons. + /// Surfaced rather than only logged: a silently disabled exporter is + /// indistinguishable from a broken one. + pub skipped: Vec, + /// Live delivery health for each exporter path. + pub delivery: Vec, +} + +impl InstrumentationState { + /// Build from config and install the sink. Replaces any previous setup and + /// flushes the replaced sink in the background; this synchronous method + /// never waits on I/O. + pub fn apply(&self, config: &InstrumentationConfig, release: &str) -> InstrumentationStatus { + let outcome = moltis_observability::build(config, release); + let snapshot = match outcome.built { + Some(built) => { + moltis_observability::set_global_sink(Arc::clone(&built.sink)); + info!(backends = ?built.backends, "agent instrumentation active"); + InstrumentationSnapshot { + sink: Some(built.sink), + backends: built.backends, + skipped: outcome.skipped, + scores: built.scores, + #[cfg(feature = "langfuse")] + langfuse: built.langfuse, + } + }, + None => { + // Tear down rather than leaving a stale sink installed, so + // disabling instrumentation in the UI takes effect immediately. + moltis_observability::clear_global_sink(); + if !outcome.skipped.is_empty() { + warn!( + skipped = ?outcome.skipped, + "instrumentation configured but no backend could start" + ); + } + InstrumentationSnapshot { + skipped: outcome.skipped, + ..Default::default() + } + }, + }; + + let status = snapshot.status(); + if let Some(old_sink) = self.replace(snapshot) { + tokio::spawn(async move { + if let Err(error) = old_sink.flush(Duration::from_secs(5)).await { + warn!(%error, "replaced instrumentation sink flush failed"); + } + }); + } + + status + } + + /// Current status. + #[must_use] + pub fn status(&self) -> InstrumentationStatus { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.status() + } + + /// Whether a running backend can store scores. + /// + /// The feedback surfaces ask this instead of naming a backend: a thumb that + /// nothing records is worse than no thumb at all. + #[must_use] + pub fn scores_available(&self) -> bool { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.scores + } + + /// The Langfuse client, when that backend is running. + #[cfg(feature = "langfuse")] + #[must_use] + pub fn langfuse(&self) -> Option> { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.langfuse.clone() + } + + /// Flush every backend, for a clean shutdown. + pub async fn flush(&self, timeout: Duration) { + let sink = { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.sink.clone() + }; + let Some(sink) = sink else { + return; + }; + if let Err(error) = sink.flush(timeout).await { + warn!(%error, "instrumentation flush failed during shutdown"); + } + } + + /// Stop accepting new events and flush the currently installed sink. + pub async fn shutdown(&self, timeout: Duration) { + moltis_observability::clear_global_sink(); + let started = tokio::time::Instant::now(); + let flush_reserve = timeout.min(Duration::from_millis(100)); + let wait_budget = timeout.saturating_sub(flush_reserve); + if !moltis_observability::wait_for_active_turns(wait_budget).await { + warn!("instrumentation shutdown timed out waiting for active turns"); + } + self.flush(timeout.saturating_sub(started.elapsed())).await; + } + + fn replace(&self, value: InstrumentationSnapshot) -> Option> { + match self.inner.write() { + Ok(mut guard) => std::mem::replace(&mut *guard, value).sink, + Err(poisoned) => std::mem::replace(&mut *poisoned.into_inner(), value).sink, + } + } +} + +impl InstrumentationSnapshot { + fn status(&self) -> InstrumentationStatus { + InstrumentationStatus { + active: self.sink.is_some(), + backends: self.backends.clone(), + skipped: self.skipped.clone(), + delivery: self + .sink + .as_ref() + .map_or_else(Vec::new, |sink| sink.delivery_stats()), + } + } +} + +#[cfg(test)] +#[allow(clippy::expect_used)] +mod tests { + use { + async_trait::async_trait, + moltis_config::LangfuseSettings, + secrecy::Secret, + std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + tokio::sync::oneshot, + }; + + use super::*; + + #[cfg(feature = "langfuse")] + fn valid_langfuse_config() -> InstrumentationConfig { + InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + } + } + + struct FlushTrackingSink { + flushed: Mutex>>, + } + + struct TimeoutAwareFlushSink { + delivered: Mutex>>, + queued: AtomicUsize, + } + + #[async_trait] + impl ObservationSink for FlushTrackingSink { + fn name(&self) -> &str { + "tracking" + } + + fn record(&self, _event: moltis_observability::Event) {} + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + if let Some(flushed) = self + .flushed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = flushed.send(()); + } + Ok(()) + } + } + + #[async_trait] + impl ObservationSink for TimeoutAwareFlushSink { + fn name(&self) -> &str { + "timeout-aware" + } + + fn record(&self, _event: moltis_observability::Event) { + self.queued.fetch_add(1, Ordering::Relaxed); + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + anyhow::ensure!( + timeout >= Duration::from_millis(25), + "insufficient flush budget" + ); + let queued = self.queued.swap(0, Ordering::Relaxed); + if let Some(delivered) = self + .delivered + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = delivered.send(queued); + } + Ok(()) + } + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn disabled_config_reports_inactive_and_installs_no_sink() { + let state = InstrumentationState::default(); + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + assert!(status.backends.is_empty()); + assert!(!moltis_observability::is_enabled()); + + moltis_observability::clear_global_sink(); + } + + #[cfg(feature = "langfuse")] + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn applying_a_valid_config_installs_the_sink() { + let state = InstrumentationState::default(); + let status = state.apply(&valid_langfuse_config(), "20260726.01"); + + assert!(status.active); + assert_eq!(status.backends, vec!["langfuse"]); + assert_eq!( + status + .delivery + .iter() + .map(|stats| stats.name.as_str()) + .collect::>(), + vec!["langfuse", "langfuse-scores"] + ); + assert!(status.delivery.iter().all(|stats| stats.accepted == 0)); + assert!(moltis_observability::is_enabled()); + assert!(state.langfuse().is_some()); + + moltis_observability::clear_global_sink(); + } + + #[cfg(feature = "langfuse")] + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn reapplying_a_disabled_config_tears_the_sink_down() { + let state = InstrumentationState::default(); + state.apply(&valid_langfuse_config(), "test"); + assert!(moltis_observability::is_enabled()); + + // Turning instrumentation off in the UI must take effect immediately + // rather than leaving the previous sink exporting. + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + assert!(!moltis_observability::is_enabled()); + assert!(state.langfuse().is_none()); + } + + #[cfg(feature = "langfuse")] + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn skipped_backends_are_reported_not_just_logged() { + let mut config = valid_langfuse_config(); + config.langfuse.public_key = String::new(); + + let state = InstrumentationState::default(); + let status = state.apply(&config, "test"); + + assert!(!status.active); + assert_eq!(status.skipped.len(), 1); + assert_eq!(status.skipped[0].name, "langfuse"); + assert!(status.skipped[0].reason.contains("public_key")); + + let current = state.status(); + assert_eq!(current.skipped, status.skipped); + + moltis_observability::clear_global_sink(); + } + + #[cfg(not(feature = "langfuse"))] + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn a_backend_this_build_lacks_is_reported_as_skipped() { + // Without the exporter compiled in, enabling it in config must explain + // itself rather than looking like a credential mistake. + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + }; + + let state = InstrumentationState::default(); + let status = state.apply(&config, "test"); + + assert!(!status.active); + assert!(!state.scores_available()); + assert_eq!(status.skipped.len(), 1); + assert_eq!(status.skipped[0].name, "langfuse"); + assert_eq!(status.skipped[0].reason, "not compiled into this build"); + } + + #[tokio::test] + async fn status_before_any_apply_is_inactive() { + let state = InstrumentationState::default(); + let status = state.status(); + + assert!(!status.active); + assert!(status.backends.is_empty()); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn flush_without_a_sink_is_a_no_op() { + moltis_observability::clear_global_sink(); + let state = InstrumentationState::default(); + state.flush(Duration::from_millis(10)).await; + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn reconfiguration_flushes_the_replaced_sink_without_blocking_apply() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let old_sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&old_sink)); + state.replace(InstrumentationSnapshot { + sink: Some(old_sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + tokio::time::timeout(Duration::from_secs(1), flushed_rx) + .await + .expect("replaced sink flush should be scheduled") + .expect("flush notification should be sent"); + assert!(!moltis_observability::is_enabled()); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_stops_new_events_and_flushes_the_active_sink() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&sink)); + state.replace(InstrumentationSnapshot { + sink: Some(sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + + state.shutdown(Duration::from_secs(1)).await; + + flushed_rx + .await + .expect("active sink should be flushed during shutdown"); + assert!(!moltis_observability::is_enabled()); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_waits_for_active_turns_before_flushing() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&sink)); + state.replace(InstrumentationSnapshot { + sink: Some(sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + let recorder = moltis_observability::TurnRecorder::begin( + "active", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + + let shutdown = state.shutdown(Duration::from_secs(1)); + tokio::pin!(shutdown); + assert!( + tokio::time::timeout(Duration::from_millis(10), &mut shutdown) + .await + .is_err(), + "shutdown must wait for the active recorder" + ); + drop(recorder); + shutdown.await; + + flushed_rx + .await + .expect("sink should flush after the turn closes"); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_flushes_queued_data_when_active_turns_time_out() { + let (delivered_tx, delivered_rx) = oneshot::channel(); + let sink = Arc::new(TimeoutAwareFlushSink { + delivered: Mutex::new(Some(delivered_tx)), + queued: AtomicUsize::new(0), + }); + let installed_sink: Arc = sink; + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&installed_sink)); + state.replace(InstrumentationSnapshot { + sink: Some(installed_sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + let completed = moltis_observability::TurnRecorder::begin( + "completed", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + completed.finish(); + let recorder = moltis_observability::TurnRecorder::begin( + "active", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + + state.shutdown(Duration::from_millis(100)).await; + + let delivered = tokio::time::timeout(Duration::from_secs(1), delivered_rx) + .await + .expect("shutdown should retain enough time to flush") + .expect("queued data should flush despite the active turn"); + assert!(delivered > 0, "the completed turn should reach the sink"); + drop(recorder); + } +} diff --git a/crates/gateway/src/server/mod.rs b/crates/gateway/src/server/mod.rs index ff32319dc4..26db8a8522 100644 --- a/crates/gateway/src/server/mod.rs +++ b/crates/gateway/src/server/mod.rs @@ -32,6 +32,8 @@ mod tests_legacy; // ── Re-exports ─────────────────────────────────────────────────────────────── // Preserves the original public API surface of `crate::server::*`. +pub mod instrumentation; + pub(crate) use hooks::discover_and_build_hooks; pub use { helpers::approval_manager_from_config, diff --git a/crates/gateway/src/server/prepare_core.rs b/crates/gateway/src/server/prepare_core.rs index 45c9e19a92..d8efc6ddd1 100644 --- a/crates/gateway/src/server/prepare_core.rs +++ b/crates/gateway/src/server/prepare_core.rs @@ -34,6 +34,7 @@ use { std::{path::PathBuf, sync::Arc}, tracing::{debug, info, warn}, }; +mod feedback; mod log_persistence; mod post_state; mod sandbox; diff --git a/crates/gateway/src/server/prepare_core/feedback.rs b/crates/gateway/src/server/prepare_core/feedback.rs new file mode 100644 index 0000000000..aa4803e852 --- /dev/null +++ b/crates/gateway/src/server/prepare_core/feedback.rs @@ -0,0 +1,40 @@ +//! Installing the reaction-feedback service during startup. +//! +//! Split out of `post_state` so that file stays inside the size limit; the +//! wiring is self-contained. + +use std::sync::Arc; + +use sqlx::SqlitePool; + +use crate::state::GatewayState; + +/// Install reply/trace correlation and start the retention prune. +/// +/// Needs the database pool, which does not exist at state construction, so it +/// runs here rather than in `GatewayState::new`. +pub(super) fn install_feedback(state: &Arc, db_pool: &SqlitePool) { + let links: Arc = Arc::new( + crate::trace_link_store::SqliteTraceLinkStore::new(db_pool.clone()), + ); + state.feedback.apply( + links, + &state.config.instrumentation.feedback, + Some(state.config.instrumentation.environment.clone()), + ); + + // Links accumulate one row per delivered reply; drop the ones too old + // to attribute a reaction to. + let feedback = Arc::clone(&state.feedback); + tokio::spawn(async move { + let prune_interval = std::time::Duration::try_from(time::Duration::days(1)) + .unwrap_or(std::time::Duration::MAX); + loop { + let removed = feedback.prune().await; + if removed > 0 { + tracing::debug!(removed, "pruned expired trace links"); + } + tokio::time::sleep(prune_interval).await; + } + }); +} diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index da4c2d95bc..afe634b5de 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -1,10 +1,10 @@ -use std::{ - path::PathBuf, - sync::{Arc, atomic::Ordering}, -}; - use { + super::feedback::install_feedback, secrecy::Secret, + std::{ + path::PathBuf, + sync::{Arc, atomic::Ordering}, + }, tracing::{debug, info, warn}, }; @@ -405,6 +405,28 @@ pub(super) async fn complete_startup( vault.clone(), ); + // ── Agent instrumentation ───────────────────────────────────────────── + // Applied on the state's own instance so RPC and lifecycle callers observe + // the same backends that were installed here. Runs before any agent can be + // invoked, so the first turn is traced. + { + let status = state + .instrumentation + .apply(&state.config.instrumentation, &state.version); + if status.active { + tracing::info!(backends = ?status.backends, "agent instrumentation enabled"); + } + for skipped in &status.skipped { + tracing::warn!( + backend = %skipped.name, + reason = %skipped.reason, + "instrumentation backend could not start" + ); + } + } + + install_feedback(&state, &db_pool); + { let (webhook_tx, webhook_rx) = tokio::sync::mpsc::channel::(256); let webhook_store: Arc = { diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index 857f9f56a3..77992a0de0 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -415,6 +415,11 @@ pub struct GatewayState { /// Code index for workspace codebase intelligence (discover, filter, status, peek). /// Always initialized in config-only mode; search is deferred to QMD backend. pub code_index: Arc, + /// Live agent instrumentation (Langfuse / OTLP / Datadog). + /// `Arc` because the RPC handlers and the shutdown path both reach it. + pub instrumentation: Arc, + /// Reaction feedback: reply/trace correlation and score submission. + pub feedback: Arc, /// Whether the server is bound to a loopback address (localhost/127.0.0.1/::1). pub localhost_only: bool, /// Whether the server is known to be behind a reverse proxy. @@ -568,6 +573,11 @@ impl GatewayState { pairing_store, memory_manager, code_index, + // Constructed empty; `apply` runs later from `prepare_core`, which + // is inside a Tokio runtime (each backend spawns an export task). + instrumentation: Arc::default(), + // Filled in by `prepare_core`, which has the database pool. + feedback: Arc::default(), localhost_only, behind_proxy, tls_active, diff --git a/crates/gateway/src/trace_link_store.rs b/crates/gateway/src/trace_link_store.rs new file mode 100644 index 0000000000..45bd929cf8 --- /dev/null +++ b/crates/gateway/src/trace_link_store.rs @@ -0,0 +1,314 @@ +//! SQLite-backed reply/trace correlation. + +use { + async_trait::async_trait, + moltis_channels::{ + Error as ChannelError, Result as ChannelResult, + trace_link::{TraceLink, TraceLinkStore}, + }, + sqlx::SqlitePool, +}; + +fn db_error(context: &'static str, source: sqlx::Error) -> ChannelError { + ChannelError::external(context, source) +} + +/// SQLite-backed [`TraceLinkStore`]. +pub struct SqliteTraceLinkStore { + pool: SqlitePool, +} + +impl SqliteTraceLinkStore { + /// Build a store over `pool`. + #[must_use] + pub const fn new(pool: SqlitePool) -> Self { + Self { pool } + } + + /// Create the schema directly. + /// + /// **Deprecated**: schema is managed by sqlx migrations. Retained for tests + /// on in-memory databases, matching the other channel stores. + #[doc(hidden)] + pub async fn init(pool: &SqlitePool) -> ChannelResult<()> { + sqlx::query( + "CREATE TABLE IF NOT EXISTS trace_links ( + channel_type TEXT NOT NULL, + account_id TEXT NOT NULL, + chat_id TEXT NOT NULL, + message_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + session_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (channel_type, account_id, chat_id, message_id) + )", + ) + .execute(pool) + .await + .map_err(|e| db_error("init trace_links table", e))?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_trace_links_created + ON trace_links (created_at)", + ) + .execute(pool) + .await + .map_err(|e| db_error("init trace_links index", e))?; + + Ok(()) + } +} + +#[async_trait] +impl TraceLinkStore for SqliteTraceLinkStore { + async fn link(&self, link: TraceLink) -> ChannelResult<()> { + // Replace rather than ignore: an edited or resent reply keeps the same + // message id but belongs to the newer turn, and a reaction on it should + // score that turn. + sqlx::query( + "INSERT INTO trace_links + (channel_type, account_id, chat_id, message_id, trace_id, session_key, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (channel_type, account_id, chat_id, message_id) + DO UPDATE SET trace_id = excluded.trace_id, + session_key = excluded.session_key, + created_at = excluded.created_at", + ) + .bind(&link.channel_type) + .bind(&link.account_id) + .bind(&link.chat_id) + .bind(&link.message_id) + .bind(&link.trace_id) + .bind(&link.session_key) + .bind(link.created_at) + .execute(&self.pool) + .await + .map_err(|e| db_error("record trace link", e))?; + Ok(()) + } + + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> ChannelResult> { + let row = sqlx::query_as::<_, (String, Option, i64)>( + "SELECT trace_id, session_key, created_at + FROM trace_links + WHERE channel_type = ? AND account_id = ? AND chat_id = ? AND message_id = ?", + ) + .bind(channel_type) + .bind(account_id) + .bind(chat_id) + .bind(message_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| db_error("look up trace link", e))?; + + Ok(row.map(|(trace_id, session_key, created_at)| TraceLink { + channel_type: channel_type.to_string(), + account_id: account_id.to_string(), + chat_id: chat_id.to_string(), + message_id: message_id.to_string(), + trace_id, + session_key, + created_at, + })) + } + + async fn prune(&self, cutoff: i64) -> ChannelResult { + let result = sqlx::query("DELETE FROM trace_links WHERE created_at < ?") + .bind(cutoff) + .execute(&self.pool) + .await + .map_err(|e| db_error("prune trace links", e))?; + Ok(result.rows_affected()) + } +} + +#[allow(clippy::unwrap_used, clippy::expect_used)] +#[cfg(test)] +mod tests { + use super::*; + + async fn store() -> SqliteTraceLinkStore { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + SqliteTraceLinkStore::init(&pool).await.unwrap(); + SqliteTraceLinkStore::new(pool) + } + + fn link(message_id: &str, trace_id: &str) -> TraceLink { + TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: message_id.into(), + trace_id: trace_id.into(), + session_key: Some("agent:main:main".into()), + created_at: 1_700_000_000, + } + } + + #[tokio::test] + async fn a_linked_reply_resolves_to_its_trace() { + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("link should resolve"); + + assert_eq!(found.trace_id, "trace-1"); + assert_eq!(found.session_key.as_deref(), Some("agent:main:main")); + } + + #[tokio::test] + async fn an_unknown_message_resolves_to_nothing() { + let store = store().await; + let found = store + .lookup("telegram", "bot-1", "chat-1", "missing") + .await + .unwrap(); + + assert!(found.is_none()); + } + + #[tokio::test] + async fn message_ids_are_scoped_per_channel_and_chat() { + // Message ids are only unique within a chat; Telegram numbers restart + // per conversation, so a global lookup would attribute a reaction in + // one chat to a completely unrelated turn in another. + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + + let other_chat = TraceLink { + chat_id: "chat-2".into(), + trace_id: "trace-2".into(), + ..link("42", "trace-2") + }; + store.link(other_chat).await.unwrap(); + + let first = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + let second = store + .lookup("telegram", "bot-1", "chat-2", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(first.trace_id, "trace-1"); + assert_eq!(second.trace_id, "trace-2"); + } + + #[tokio::test] + async fn the_same_message_id_on_another_channel_is_a_separate_link() { + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + store + .link(TraceLink { + channel_type: "discord".into(), + trace_id: "trace-discord".into(), + ..link("42", "trace-discord") + }) + .await + .unwrap(); + + let discord = store + .lookup("discord", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(discord.trace_id, "trace-discord"); + } + + #[tokio::test] + async fn relinking_a_message_points_at_the_newer_turn() { + let store = store().await; + store.link(link("42", "trace-old")).await.unwrap(); + store + .link(TraceLink { + created_at: 1_700_000_500, + ..link("42", "trace-new") + }) + .await + .unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(found.trace_id, "trace-new"); + assert_eq!(found.created_at, 1_700_000_500); + } + + #[tokio::test] + async fn pruning_drops_only_links_older_than_the_cutoff() { + let store = store().await; + store.link(link("old", "trace-old")).await.unwrap(); + store + .link(TraceLink { + created_at: 1_700_009_000, + ..link("new", "trace-new") + }) + .await + .unwrap(); + + let removed = store.prune(1_700_005_000).await.unwrap(); + + assert_eq!(removed, 1); + assert!( + store + .lookup("telegram", "bot-1", "chat-1", "old") + .await + .unwrap() + .is_none() + ); + assert!( + store + .lookup("telegram", "bot-1", "chat-1", "new") + .await + .unwrap() + .is_some(), + "pruning must not drop links still inside the retention window" + ); + } + + #[tokio::test] + async fn pruning_an_empty_table_is_a_no_op() { + let store = store().await; + assert_eq!(store.prune(1_700_000_000).await.unwrap(), 0); + } + + #[tokio::test] + async fn a_link_without_a_session_is_still_usable() { + // Channel replies that are not bound to an agent session still deserve + // feedback attribution. + let store = store().await; + store + .link(TraceLink { + session_key: None, + ..link("42", "trace-1") + }) + .await + .unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(found.trace_id, "trace-1"); + assert!(found.session_key.is_none()); + } +} diff --git a/crates/httpd/src/server/runtime.rs b/crates/httpd/src/server/runtime.rs index 0ce6156681..8da43d8b80 100644 --- a/crates/httpd/src/server/runtime.rs +++ b/crates/httpd/src/server/runtime.rs @@ -1371,6 +1371,7 @@ pub async fn start_gateway( // - SIGINT (ctrl-c): immediate exit { let browser_for_shutdown = Arc::clone(&banner.browser_for_lifecycle); + let instrumentation_for_shutdown = Arc::clone(&state.instrumentation); #[cfg(feature = "tailscale")] let reset_tailscale_on_exit = banner.tailscale_mode != TailscaleMode::Off && banner.tailscale_reset_on_exit; @@ -1406,19 +1407,18 @@ pub async fn start_gateway( } "SIGINT" }; - if signal_name == "SIGINT" { - info!("received SIGINT, exiting immediately"); + info!("received SIGINT, flushing instrumentation before exit"); + instrumentation_for_shutdown + .shutdown(std::time::Duration::from_secs(1)) + .await; std::process::exit(0); } - info!(signal = signal_name, "starting graceful shutdown"); - #[cfg(feature = "mdns")] if let Some(ref daemon) = _mdns_daemon { moltis_gateway::mdns::shutdown(daemon); } - #[cfg(feature = "tailscale")] if reset_tailscale_on_exit { info!("shutting down tailscale {ts_mode}"); @@ -1427,8 +1427,8 @@ pub async fn start_gateway( warn!("failed to reset tailscale on exit: {e}"); } } - let shutdown_grace = std::time::Duration::from_secs(5); + instrumentation_for_shutdown.shutdown(shutdown_grace).await; info!( grace_secs = shutdown_grace.as_secs(), "shutting down browser pool" diff --git a/crates/observability/Cargo.toml b/crates/observability/Cargo.toml new file mode 100644 index 0000000000..05374f281a --- /dev/null +++ b/crates/observability/Cargo.toml @@ -0,0 +1,36 @@ +[package] +edition.workspace = true +name = "moltis-observability" +version.workspace = true + +[features] +default = ["langfuse", "otlp"] +langfuse = ["otlp"] +metrics = ["dep:moltis-metrics"] +otlp = [] + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +base64 = { workspace = true } +futures = { workspace = true } +moltis-common = { workspace = true } +moltis-config = { workspace = true } +moltis-metrics = { optional = true, workspace = true } +rand = { workspace = true } +reqwest = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +tokio = { features = ["full", "test-util"], workspace = true } +wiremock = { workspace = true } + +[lints] +workspace = true diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs new file mode 100644 index 0000000000..8ab37445b5 --- /dev/null +++ b/crates/observability/src/builder.rs @@ -0,0 +1,787 @@ +//! Construction of live sinks from `[instrumentation]` configuration. + +use std::sync::Arc; + +use { + moltis_config::{ContentCaptureMode, InstrumentationConfig}, + tracing::{info, warn}, +}; + +use crate::{ + profile::ContentCapture, + recorder::RecorderSettings, + redact::RedactionPolicy, + sink::{ObservationSink, SinkFanout}, +}; + +// Everything below is needed only by an exporter that was compiled in. A build +// with no exporter feature still parses `[instrumentation]` and reports each +// enabled backend as skipped, so the config never silently does nothing. +#[cfg(feature = "langfuse")] +use { + crate::exporters::langfuse::{LangfuseClient, LangfuseConfig, ScoreSink}, + moltis_config::LangfuseSettings, +}; +#[cfg(any(feature = "langfuse", feature = "otlp"))] +use { + crate::runtime::{BatchConfig, BatchSink}, + secrecy::ExposeSecret, + std::time::Duration, +}; +#[cfg(feature = "otlp")] +use { + crate::{ + exporters::otlp::{OtlpConfig, OtlpTransport}, + profile::ExportProfile, + }, + moltis_config::{DatadogSettings, OtlpSettings}, +}; + +/// Convert the config-level content mode to the profile-level one. +impl From for ContentCapture { + fn from(mode: ContentCaptureMode) -> Self { + match mode { + ContentCaptureMode::Full => Self::Full, + ContentCaptureMode::MetadataOnly => Self::MetadataOnly, + ContentCaptureMode::None => Self::None, + } + } +} + +/// The result of building instrumentation from config. +pub struct BuiltInstrumentation { + /// Fanout over every enabled backend. + pub sink: Arc, + /// Settings the agent runtime applies when recording. + pub recorder_settings: RecorderSettings, + /// Whether a backend that accepts scores is running. + /// + /// Scores are the one event type most backends have no representation for, + /// so the feedback surfaces ask this rather than naming a backend. + pub scores: bool, + /// Langfuse REST client, present when the Langfuse backend is enabled. + /// Used for the settings UI connection test. + #[cfg(feature = "langfuse")] + pub langfuse: Option>, + /// Names of the backends that were built. + pub backends: Vec, +} + +/// Reasons a backend was skipped, surfaced in the settings UI rather than only +/// in logs — a silently disabled exporter is indistinguishable from a broken one. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct SkippedBackend { + /// Backend name. + pub name: String, + /// Why it was not built. + pub reason: String, +} + +/// Outcome of a build attempt. +pub struct BuildOutcome { + /// Built instrumentation, or `None` when nothing was enabled. + pub built: Option, + /// Backends that were enabled but could not be built. + pub skipped: Vec, +} + +/// Batching parameters shared by every backend. +#[cfg(any(feature = "langfuse", feature = "otlp"))] +fn batch_config(config: &InstrumentationConfig) -> BatchConfig { + BatchConfig { + max_batch_bytes: config.max_batch_bytes, + flush_interval: Duration::from_millis(config.flush_interval_ms), + queue_capacity: config.queue_capacity, + ..Default::default() + } +} + +fn validate_batch_config(config: &InstrumentationConfig) -> Result<(), String> { + let invalid = [ + (config.queue_capacity == 0, "queue_capacity"), + (config.flush_interval_ms == 0, "flush_interval_ms"), + (config.max_batch_bytes == 0, "max_batch_bytes"), + ]; + if let Some((_, field)) = invalid.into_iter().find(|(invalid, _)| *invalid) { + return Err(format!("{field} must be at least 1")); + } + Ok(()) +} + +/// Validate a backend endpoint. +/// +/// Plaintext HTTP is refused for non-loopback hosts: these payloads carry +/// conversation content and credentials in a header, and shipping them +/// unencrypted across a network is not a trade-off an operator should make by +/// typing a URL. Loopback is allowed for a local Agent or collector. +#[cfg(any(feature = "langfuse", feature = "otlp"))] +fn validate_endpoint(raw: &str) -> Result<(), String> { + let Ok(url) = reqwest::Url::parse(raw) else { + return Err("endpoint is not a valid URL".to_string()); + }; + if !url.username().is_empty() || url.password().is_some() { + return Err("endpoint must not contain credentials; use headers instead".to_string()); + } + match url.scheme() { + "https" => Ok(()), + "http" => { + let host = url.host_str().unwrap_or_default(); + let is_loopback = host == "localhost" + || host == "::1" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()); + if is_loopback { + Ok(()) + } else { + Err(format!( + "refusing plaintext http for non-loopback host `{host}`; use https" + )) + } + }, + other => Err(format!("unsupported URL scheme `{other}`")), + } +} + +/// Build the Langfuse backend. +/// +/// Returns two sinks, not one. Traces go over OTLP, but the OTLP mapping has +/// no representation for a score and drops those events, so scores need their +/// own sink onto the ingestion API — otherwise every score recorded would be +/// silently discarded on the way out. +#[cfg(feature = "langfuse")] +fn build_langfuse( + settings: &LangfuseSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result<(Vec>, Arc), String> { + if settings.public_key.trim().is_empty() { + return Err("public_key is not set".to_string()); + } + let Some(secret) = settings.secret_key.as_ref() else { + return Err("secret_key is not set".to_string()); + }; + if secret.expose_secret().trim().is_empty() { + return Err("secret_key is empty".to_string()); + } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } + validate_endpoint(&settings.host)?; + + let langfuse_config = LangfuseConfig { + host: settings.host.clone(), + public_key: settings.public_key.clone(), + secret_key: secrecy::SecretString::new(secret.expose_secret().clone()), + environment: Some(config.environment.clone()), + release: Some(release.to_string()), + timeout: Duration::from_secs(settings.timeout_secs), + }; + + let transport = Arc::new(langfuse_config.build_transport(release.to_string())); + let traces = Arc::new(BatchSink::spawn(transport, batch_config(config))); + let client = Arc::new(LangfuseClient::new(langfuse_config)); + let scores = Arc::new(ScoreSink::spawn(Arc::clone(&client), batch_config(config))); + Ok((vec![traces, scores], client)) +} + +/// Build a generic OTLP backend. +#[cfg(feature = "otlp")] +fn build_otlp( + settings: &OtlpSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result, String> { + if settings.endpoint.trim().is_empty() { + return Err("endpoint is not set".to_string()); + } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } + validate_endpoint(&settings.endpoint)?; + + let profile = ExportProfile { + content: settings.content.into(), + emit_user_id: settings.emit_user_id, + ..ExportProfile::otel_generic() + }; + + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "otlp".to_string(), + endpoint: settings.endpoint.clone(), + headers: settings.headers.clone(), + timeout: Duration::from_secs(settings.timeout_secs), + service_name: "moltis".to_string(), + service_version: release.to_string(), + environment: Some(config.environment.clone()), + profile, + })); + Ok(Arc::new(BatchSink::spawn(transport, batch_config(config)))) +} + +/// Build the Datadog backend, over the same OTLP wire format. +#[cfg(feature = "otlp")] +fn build_datadog( + settings: &DatadogSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result, String> { + if settings.endpoint.trim().is_empty() { + return Err("endpoint is not set".to_string()); + } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } + if settings.service.trim().is_empty() { + return Err("service is not set".to_string()); + } + validate_endpoint(&settings.endpoint)?; + + let mut headers = std::collections::BTreeMap::new(); + if let Some(key) = &settings.api_key { + headers.insert("DD-API-KEY".to_string(), key.expose_secret().clone()); + } + + let profile = ExportProfile { + content: settings.content.into(), + ..ExportProfile::datadog() + }; + + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "datadog".to_string(), + endpoint: settings.endpoint.clone(), + headers, + timeout: Duration::from_secs(settings.timeout_secs), + service_name: settings.service.clone(), + service_version: release.to_string(), + environment: Some(config.environment.clone()), + profile, + })); + Ok(Arc::new(BatchSink::spawn(transport, batch_config(config)))) +} + +/// Build every enabled backend. +/// +/// A backend that cannot be built is reported in `skipped` and the others still +/// start: one misconfigured exporter must not take instrumentation down. +/// +/// Must be called from within a Tokio runtime — each backend spawns a +/// background export task. +#[must_use] +// With no exporter compiled in there is nothing to build: every enabled backend +// is reported as skipped, so the accumulators are never written and the release +// string is never read. +#[cfg_attr( + not(any(feature = "langfuse", feature = "otlp")), + allow(unused_mut, unused_variables) +)] +pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { + let mut skipped = Vec::new(); + + if !config.enabled { + return BuildOutcome { + built: None, + skipped, + }; + } + + if let Err(reason) = validate_batch_config(config) { + let skipped = [ + ("langfuse", config.langfuse.enabled), + ("otlp", config.otlp.enabled), + ("datadog", config.datadog.enabled), + ] + .into_iter() + .filter(|(_, enabled)| *enabled) + .map(|(name, _)| SkippedBackend { + name: name.to_string(), + reason: reason.clone(), + }) + .collect(); + return BuildOutcome { + built: None, + skipped, + }; + } + + let mut sinks: Vec> = Vec::new(); + let mut backends = Vec::new(); + #[cfg(feature = "langfuse")] + let mut langfuse_client = None; + + if config.langfuse.enabled { + #[cfg(feature = "langfuse")] + match build_langfuse(&config.langfuse, config, release) { + Ok((langfuse_sinks, client)) => { + sinks.extend(langfuse_sinks); + langfuse_client = Some(client); + backends.push("langfuse".to_string()); + }, + Err(reason) => { + warn!(backend = "langfuse", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "langfuse".to_string(), + reason, + }); + }, + } + #[cfg(not(feature = "langfuse"))] + skipped.push(not_compiled_in("langfuse")); + } + + if config.otlp.enabled { + #[cfg(feature = "otlp")] + match build_otlp(&config.otlp, config, release) { + Ok(sink) => { + sinks.push(sink); + backends.push("otlp".to_string()); + }, + Err(reason) => { + warn!(backend = "otlp", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "otlp".to_string(), + reason, + }); + }, + } + #[cfg(not(feature = "otlp"))] + skipped.push(not_compiled_in("otlp")); + } + + if config.datadog.enabled { + #[cfg(feature = "otlp")] + match build_datadog(&config.datadog, config, release) { + Ok(sink) => { + sinks.push(sink); + backends.push("datadog".to_string()); + }, + Err(reason) => { + warn!(backend = "datadog", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "datadog".to_string(), + reason, + }); + }, + } + // Datadog is reached over the same OTLP wire format. + #[cfg(not(feature = "otlp"))] + skipped.push(not_compiled_in("datadog")); + } + + if sinks.is_empty() { + return BuildOutcome { + built: None, + skipped, + }; + } + + info!(backends = ?backends, "instrumentation enabled"); + + // Capture switches live on the Langfuse settings: it is the only backend + // that receives bodies at all, and the APM profiles gate content + // independently through their own `content` mode. + let recorder_settings = RecorderSettings { + redaction: RedactionPolicy::from_needles(&config.redact), + capture_input: config.langfuse.capture_input, + capture_output: config.langfuse.capture_output, + capture_tool_io: config.langfuse.capture_tool_io, + sample_rate: config.sample_rate.clamp(0.0, 1.0), + }; + + BuildOutcome { + built: Some(BuiltInstrumentation { + sink: Arc::new(SinkFanout::new(sinks)), + recorder_settings, + // Langfuse is the only backend with a score representation; the + // OTLP mapping drops score events on the floor. + scores: backends.iter().any(|name| name == "langfuse"), + #[cfg(feature = "langfuse")] + langfuse: langfuse_client, + backends, + }), + skipped, + } +} + +/// Report a backend the operator enabled but this binary cannot speak to. +/// +/// Reported as a skipped backend rather than logged, so a build without the +/// exporter feature explains itself in the settings UI instead of looking like +/// a misconfiguration. +#[cfg(not(all(feature = "langfuse", feature = "otlp")))] +fn not_compiled_in(name: &str) -> SkippedBackend { + warn!( + backend = name, + "instrumentation backend enabled in config but not compiled into this build" + ); + SkippedBackend { + name: name.to_string(), + reason: "not compiled into this build".to_string(), + } +} + +// These build real backends, so they need the exporters compiled in. `langfuse` +// implies `otlp`, so gating on it covers the default feature set; the reduced +// builds are covered by `reduced_build_tests` below. +#[cfg(all(test, feature = "langfuse"))] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + crate::{Event, ScoreRecord, ScoreValue, TraceId}, + secrecy::Secret, + }; + + use super::*; + + fn enabled_config() -> InstrumentationConfig { + InstrumentationConfig { + enabled: true, + ..Default::default() + } + } + + fn valid_langfuse() -> LangfuseSettings { + LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + } + } + + #[tokio::test] + async fn disabled_config_builds_nothing() { + let outcome = build(&InstrumentationConfig::default(), "test"); + assert!(outcome.built.is_none()); + assert!(outcome.skipped.is_empty()); + } + + #[tokio::test] + async fn master_switch_off_skips_enabled_backends_silently() { + let config = InstrumentationConfig { + enabled: false, + langfuse: valid_langfuse(), + ..Default::default() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + // Not "skipped": the operator turned the whole thing off deliberately. + assert!(outcome.skipped.is_empty()); + } + + #[tokio::test] + async fn langfuse_builds_with_valid_credentials() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + ..enabled_config() + }; + let outcome = build(&config, "20260726.01"); + let built = outcome.built.expect("langfuse should build"); + + assert_eq!(built.backends, vec!["langfuse"]); + assert!(built.langfuse.is_some(), "REST client needed for scores"); + } + + #[tokio::test] + async fn langfuse_without_credentials_is_skipped_with_a_reason() { + let config = InstrumentationConfig { + langfuse: LangfuseSettings { + enabled: true, + public_key: String::new(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert!(outcome.skipped[0].reason.contains("public_key")); + } + + #[tokio::test] + async fn one_broken_backend_does_not_disable_the_others() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + otlp: OtlpSettings { + enabled: true, + endpoint: String::new(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + let built = outcome.built.expect("langfuse should still build"); + + assert_eq!(built.backends, vec!["langfuse"]); + assert_eq!(outcome.skipped.len(), 1); + assert_eq!(outcome.skipped[0].name, "otlp"); + } + + #[tokio::test] + async fn multiple_backends_fan_out_together() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + otlp: OtlpSettings { + enabled: true, + endpoint: "http://localhost:4318/v1/traces".into(), + ..Default::default() + }, + datadog: DatadogSettings { + enabled: true, + ..Default::default() + }, + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert_eq!(built.backends, vec!["langfuse", "otlp", "datadog"]); + // Langfuse contributes two sinks: traces over OTLP and scores over the + // ingestion API. It still reports as one backend to the operator. + assert_eq!(built.sink.name(), "langfuse+langfuse-scores+otlp+datadog"); + assert_eq!( + built + .sink + .delivery_stats() + .into_iter() + .map(|stats| stats.name) + .collect::>(), + vec!["langfuse", "langfuse-scores", "otlp", "datadog"] + ); + } + + #[tokio::test] + async fn langfuse_gets_a_score_sink_alongside_the_trace_sink() { + // The OTLP mapping has no representation for a score and drops those + // events, so without a second sink every score would be recorded by the + // agent and then silently discarded on the way out. + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert_eq!(built.sink.name(), "langfuse+langfuse-scores"); + + built.sink.record(Event::Score(Box::new(ScoreRecord::new( + TraceId::generate(), + "quality", + ScoreValue::Numeric(1.0), + )))); + let delivery = built.sink.delivery_stats(); + assert_eq!(delivery[0].name, "langfuse"); + assert_eq!(delivery[0].accepted, 0); + assert_eq!(delivery[1].name, "langfuse-scores"); + assert_eq!(delivery[1].accepted, 1); + } + + #[tokio::test] + async fn plaintext_http_is_refused_for_remote_hosts() { + // Conversation content plus a credential header over the open network. + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "http://collector.example.com/v1/traces".into(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert!(outcome.skipped[0].reason.contains("plaintext http")); + } + + #[tokio::test] + async fn plaintext_http_is_allowed_for_loopback() { + // The common Datadog Agent and local-collector deployment. + for endpoint in [ + "http://localhost:4318/v1/traces", + "http://127.0.0.1:4318/v1/traces", + ] { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: endpoint.to_string(), + ..Default::default() + }, + ..enabled_config() + }; + assert!( + build(&config, "test").built.is_some(), + "{endpoint} should be accepted" + ); + } + } + + #[tokio::test] + async fn malformed_endpoints_are_rejected() { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "not a url".into(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + assert!(outcome.skipped[0].reason.contains("not a valid URL")); + } + + #[tokio::test] + async fn endpoint_credentials_are_rejected_without_echoing_them() { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "https://operator:super-secret@collector.example/v1/traces".into(), + ..Default::default() + }, + ..enabled_config() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert!( + outcome.skipped[0] + .reason + .contains("must not contain credentials") + ); + assert!(!outcome.skipped[0].reason.contains("super-secret")); + } + + #[tokio::test] + async fn zero_batch_settings_skip_backends_without_panicking() { + for (queue_capacity, flush_interval_ms, max_batch_bytes, field) in [ + (0, 1, 1, "queue_capacity"), + (1, 0, 1, "flush_interval_ms"), + (1, 1, 0, "max_batch_bytes"), + ] { + let config = InstrumentationConfig { + queue_capacity, + flush_interval_ms, + max_batch_bytes, + langfuse: valid_langfuse(), + ..enabled_config() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert!(outcome.skipped[0].reason.contains(field)); + } + } + + #[tokio::test] + async fn sample_rate_is_clamped_into_range() { + let config = InstrumentationConfig { + sample_rate: 7.5, + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + assert_eq!(built.recorder_settings.sample_rate, 1.0); + } + + #[tokio::test] + async fn capture_switches_flow_through_to_the_recorder() { + let config = InstrumentationConfig { + langfuse: LangfuseSettings { + capture_tool_io: false, + ..valid_langfuse() + }, + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert!(!built.recorder_settings.capture_tool_io); + assert!(built.recorder_settings.capture_input); + } + + #[tokio::test] + async fn configured_redaction_needles_extend_the_defaults() { + let config = InstrumentationConfig { + redact: vec!["customer_ref".into()], + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + let policy = &built.recorder_settings.redaction; + + assert!(policy.is_sensitive_key("customer_ref")); + assert!(policy.is_sensitive_key("password")); + } + + #[tokio::test] + async fn datadog_api_key_is_sent_as_a_header_not_a_query_param() { + let config = InstrumentationConfig { + datadog: DatadogSettings { + enabled: true, + endpoint: "https://otlp.datadoghq.com/v1/traces".into(), + api_key: Some(Secret::new("dd-key".to_string())), + ..Default::default() + }, + ..enabled_config() + }; + assert!(build(&config, "test").built.is_some()); + } + + #[test] + fn content_modes_convert_to_the_matching_profile_policy() { + assert_eq!( + ContentCapture::from(ContentCaptureMode::Full), + ContentCapture::Full + ); + assert_eq!( + ContentCapture::from(ContentCaptureMode::MetadataOnly), + ContentCapture::MetadataOnly + ); + assert_eq!( + ContentCapture::from(ContentCaptureMode::None), + ContentCapture::None + ); + } +} + +/// Coverage for builds that left an exporter out. +#[cfg(all(test, not(feature = "langfuse")))] +mod reduced_build_tests { + use {moltis_config::LangfuseSettings, secrecy::Secret}; + + use super::*; + + #[tokio::test] + async fn a_backend_this_build_lacks_is_skipped_with_a_reason() { + // Enabling a backend the binary cannot speak to must explain itself, + // otherwise it is indistinguishable from bad credentials. + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert_eq!(outcome.skipped[0].name, "langfuse"); + assert_eq!(outcome.skipped[0].reason, "not compiled into this build"); + } + + #[tokio::test] + async fn an_untouched_config_still_builds_nothing() { + let outcome = build(&InstrumentationConfig::default(), "test"); + + assert!(outcome.built.is_none()); + assert!(outcome.skipped.is_empty()); + } +} diff --git a/crates/observability/src/exporters/langfuse/client.rs b/crates/observability/src/exporters/langfuse/client.rs new file mode 100644 index 0000000000..a026e478c9 --- /dev/null +++ b/crates/observability/src/exporters/langfuse/client.rs @@ -0,0 +1,197 @@ +//! The Langfuse REST client and the request helpers its API modules share. +//! +//! Traces reach Langfuse over OTLP; everything OTLP cannot express — scores, +//! managed prompts, datasets, media, aggregated cost — goes through here. + +use reqwest::header::HeaderMap; + +use super::config::{HEALTH_PATH, INGESTION_PATH, LangfuseConfig}; + +/// Client for the Langfuse REST surfaces that OTLP cannot express. +pub struct LangfuseClient { + config: LangfuseConfig, + http: reqwest::Client, +} + +impl LangfuseClient { + /// Build a client over the workspace HTTP client, so the configured + /// upstream proxy applies. + #[must_use] + pub fn new(config: LangfuseConfig) -> Self { + Self { + config, + http: moltis_common::http_client::build_default_http_client(), + } + } + + /// The connection settings this client was built from. + #[must_use] + pub const fn config(&self) -> &LangfuseConfig { + &self.config + } + + /// Verify that the host is reachable and the credentials are accepted. + /// + /// Backs the settings UI's "Test connection" button. The health endpoint is + /// unauthenticated, so credentials are checked with an empty ingestion + /// batch: it is the cheapest authenticated call that creates no data. + pub async fn test_connection(&self) -> anyhow::Result<()> { + let health = self + .http + .get(self.config.url(HEALTH_PATH)) + .timeout(self.config.timeout) + .send() + .await + .map_err(|e| anyhow::anyhow!("cannot reach Langfuse at {}: {e}", self.config.host))?; + + if !health.status().is_success() { + return Err(anyhow::anyhow!( + "Langfuse health check failed with HTTP {}", + health.status() + )); + } + + let auth = self + .post(INGESTION_PATH) + .json(&serde_json::json!({ "batch": [] })) + .send() + .await + .map_err(|e| anyhow::anyhow!("Langfuse credential check failed: {e}"))?; + + let status = auth.status(); + match status { + s if s.is_success() => Ok(()), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN => Err( + anyhow::anyhow!("Langfuse rejected the credentials (HTTP {status})"), + ), + other => Err(anyhow::anyhow!( + "unexpected response from Langfuse: HTTP {other}" + )), + } + } + + /// Authenticated POST against `path`. + pub(super) fn post(&self, path: &str) -> reqwest::RequestBuilder { + self.http + .post(self.config.url(path)) + .headers(self.header_map()) + .timeout(self.config.timeout) + } + + /// Authenticated DELETE against `path`. + pub(super) fn delete(&self, path: &str) -> reqwest::RequestBuilder { + self.http + .delete(self.config.url(path)) + .headers(self.header_map()) + .timeout(self.config.timeout) + } + + /// Authenticated header map. + /// + /// Header values are marked sensitive so they stay out of any middleware + /// that logs request headers. + fn header_map(&self) -> HeaderMap { + let mut map = HeaderMap::new(); + for (key, value) in self.config.auth_headers() { + let Ok(name) = key.parse::() else { + continue; + }; + let Ok(mut val) = reqwest::header::HeaderValue::from_str(&value) else { + continue; + }; + val.set_sensitive(true); + map.insert(name, val); + } + map + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + std::time::Duration, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }, + }; + + use {super::*, secrecy::SecretString}; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + #[tokio::test] + async fn test_connection_accepts_a_healthy_authenticated_host() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .and(header( + "authorization", + "Basic cGstbGYtdGVzdDpzay1sZi1zZWNyZXQ=", + )) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], "errors": [] + }))) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect("healthy host should pass"); + } + + #[tokio::test] + async fn test_connection_reports_bad_credentials_distinctly() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect_err("401 should fail"); + + // The operator needs to know it is the keys, not the host. + assert!( + error.to_string().contains("rejected the credentials"), + "unhelpful message: {error}" + ); + } + + #[tokio::test] + async fn test_connection_reports_an_unreachable_host_distinctly() { + let error = LangfuseClient::new(config("http://127.0.0.1:1".into())) + .test_connection() + .await + .expect_err("connection refused should fail"); + + assert!( + error.to_string().contains("cannot reach Langfuse"), + "unhelpful message: {error}" + ); + } +} diff --git a/crates/observability/src/exporters/langfuse/config.rs b/crates/observability/src/exporters/langfuse/config.rs new file mode 100644 index 0000000000..2ae3c97431 --- /dev/null +++ b/crates/observability/src/exporters/langfuse/config.rs @@ -0,0 +1,225 @@ +//! Langfuse connection settings, endpoint paths and credential handling. + +use std::{collections::BTreeMap, time::Duration}; + +use { + base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}, + secrecy::{ExposeSecret, SecretString}, +}; + +use crate::{ + exporters::otlp::{OtlpConfig, OtlpTransport}, + profile::ExportProfile, +}; + +/// Path Langfuse serves OTLP traces on, appended to the configured host. +pub const OTEL_TRACES_PATH: &str = "/api/public/otel/v1/traces"; +/// Batch ingestion path, used for scores and dataset run items. +pub const INGESTION_PATH: &str = "/api/public/ingestion"; +/// Unauthenticated liveness probe. +pub const HEALTH_PATH: &str = "/api/public/health"; +/// Individual score resource, `{id}` appended, used for retraction. +pub const SCORES_PATH: &str = "/api/public/scores"; +/// Managed-prompt retrieval, `{name}` appended. +pub const PROMPTS_PATH: &str = "/api/public/v2/prompts"; +/// Dataset collection. +pub const DATASETS_PATH: &str = "/api/public/v2/datasets"; +/// Dataset item collection. +pub const DATASET_ITEMS_PATH: &str = "/api/public/dataset-items"; +/// Dataset run item collection, linking a trace to a dataset item. +pub const DATASET_RUN_ITEMS_PATH: &str = "/api/public/dataset-run-items"; +/// Media upload negotiation. +pub const MEDIA_PATH: &str = "/api/public/media"; +/// Aggregated daily usage and cost. +pub const DAILY_METRICS_PATH: &str = "/api/public/metrics/daily"; + +/// Langfuse connection settings. +#[derive(Clone)] +pub struct LangfuseConfig { + /// Base host, e.g. `https://cloud.langfuse.com` or a self-hosted URL. + pub host: String, + /// Project public key (Basic auth username). + pub public_key: String, + /// Project secret key (Basic auth password). + pub secret_key: SecretString, + /// Deployment environment. + pub environment: Option, + /// Build release identifier. + pub release: Option, + /// Per-request timeout. + pub timeout: Duration, +} + +// Manual impl: the derived one would print the secret key. +impl std::fmt::Debug for LangfuseConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LangfuseConfig") + .field("host", &self.host) + .field("public_key", &self.public_key) + .field("secret_key", &"[REDACTED]") + .field("environment", &self.environment) + .field("release", &self.release) + .field("timeout", &self.timeout) + .finish() + } +} + +impl LangfuseConfig { + /// Join `path` onto the configured host, tolerating a trailing slash. + #[must_use] + pub fn url(&self, path: &str) -> String { + format!("{}{path}", self.host.trim_end_matches('/')) + } + + /// `Authorization` header value for Basic auth. + /// + /// Langfuse authenticates with the public key as username and the secret + /// key as password. + #[must_use] + pub fn basic_auth_header(&self) -> String { + let raw = format!("{}:{}", self.public_key, self.secret_key.expose_secret()); + format!("Basic {}", BASE64.encode(raw)) + } + + /// Headers every authenticated Langfuse request carries. + #[must_use] + pub fn auth_headers(&self) -> BTreeMap { + let mut headers = BTreeMap::new(); + headers.insert("Authorization".to_string(), self.basic_auth_header()); + // Langfuse surfaces these in its own diagnostics. + headers.insert("X-Langfuse-Sdk-Name".to_string(), "moltis".to_string()); + headers.insert( + "X-Langfuse-Sdk-Version".to_string(), + env!("CARGO_PKG_VERSION").to_string(), + ); + headers.insert("X-Langfuse-Public-Key".to_string(), self.public_key.clone()); + headers + } + + /// Auth headers required by Langfuse's v4 OTLP ingestion contract. + fn otlp_auth_headers(&self) -> BTreeMap { + let mut headers = self.auth_headers(); + headers.insert("X-Langfuse-Ingestion-Version".to_string(), "4".to_string()); + headers + } + + /// Build the OTLP transport that carries traces to Langfuse. + /// + /// The profile is fixed to [`ExportProfile::langfuse`]: content capture is + /// the entire point of sending to Langfuse, and letting it be configured + /// down to `MetadataOnly` would produce traces with no conversation in + /// them, which is worse than not exporting at all. + #[must_use] + pub fn build_transport(&self, service_version: String) -> OtlpTransport { + OtlpTransport::new(OtlpConfig { + name: "langfuse".to_string(), + endpoint: self.url(OTEL_TRACES_PATH), + headers: self.otlp_auth_headers(), + timeout: self.timeout, + service_name: "moltis".to_string(), + service_version, + environment: self.environment.clone(), + profile: ExportProfile::langfuse(), + }) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + super::*, + crate::{model::Event, runtime::Transport}, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }, + }; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + #[test] + fn basic_auth_encodes_public_and_secret_key() { + let header = config("https://example.com".into()).basic_auth_header(); + let encoded = header.strip_prefix("Basic ").expect("basic scheme"); + let decoded = + String::from_utf8(BASE64.decode(encoded).expect("valid base64")).expect("valid utf-8"); + + assert_eq!(decoded, "pk-lf-test:sk-lf-secret"); + } + + #[test] + fn debug_never_prints_the_secret_key() { + let rendered = format!("{:?}", config("https://example.com".into())); + + assert!( + !rendered.contains("sk-lf-secret"), + "secret leaked: {rendered}" + ); + assert!(rendered.contains("[REDACTED]")); + // The public key is not a secret and is useful in diagnostics. + assert!(rendered.contains("pk-lf-test")); + } + + #[test] + fn url_join_tolerates_a_trailing_slash_on_the_host() { + let with = config("https://example.com/".into()); + let without = config("https://example.com".into()); + + assert_eq!(with.url(OTEL_TRACES_PATH), without.url(OTEL_TRACES_PATH)); + assert_eq!( + without.url(OTEL_TRACES_PATH), + "https://example.com/api/public/otel/v1/traces" + ); + } + + #[test] + fn transport_targets_the_otel_endpoint_with_the_langfuse_profile() { + let transport = config("https://cloud.langfuse.com".into()).build_transport("1.0".into()); + + assert_eq!( + transport.endpoint(), + "https://cloud.langfuse.com/api/public/otel/v1/traces" + ); + } + + #[test] + fn otlp_headers_request_ingestion_version_four() { + let headers = config("https://cloud.langfuse.com".into()).otlp_auth_headers(); + assert_eq!( + headers + .get("X-Langfuse-Ingestion-Version") + .map(String::as_str), + Some("4") + ); + } + + #[tokio::test] + async fn ingestion_version_four_is_sent_on_the_wire() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(OTEL_TRACES_PATH)) + .and(header("x-langfuse-ingestion-version", "4")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + let transport = config(server.uri()).build_transport("test".into()); + let mut trace = crate::model::TraceRecord::new("turn"); + trace.finish(); + + transport + .send(&[Event::Trace(Box::new(trace))]) + .await + .expect("OTLP request accepted"); + } +} diff --git a/crates/observability/src/exporters/langfuse/mod.rs b/crates/observability/src/exporters/langfuse/mod.rs new file mode 100644 index 0000000000..c743b959ef --- /dev/null +++ b/crates/observability/src/exporters/langfuse/mod.rs @@ -0,0 +1,24 @@ +//! Langfuse backend. +//! +//! Traces reach Langfuse over OTLP — that is Langfuse's own modern ingest path, +//! the one its v3+ Python and v4+ JS SDKs use, and the only one that carries the +//! full observation taxonomy (`AGENT`, `TOOL`, `RETRIEVER`, ...). The native +//! `/api/public/ingestion` event `observation-create` is marked deprecated +//! upstream and its non-deprecated `span-create`/`generation-create` pair +//! cannot express the newer types at all. +//! +//! Everything OTLP *cannot* express lives in the REST modules here: scores, +//! managed prompts, datasets and experiment runs, media, and aggregated cost. + +pub mod client; +pub mod config; +pub mod scores; + +pub use { + client::LangfuseClient, + config::{ + DAILY_METRICS_PATH, DATASET_ITEMS_PATH, DATASET_RUN_ITEMS_PATH, DATASETS_PATH, HEALTH_PATH, + INGESTION_PATH, LangfuseConfig, MEDIA_PATH, OTEL_TRACES_PATH, PROMPTS_PATH, SCORES_PATH, + }, + scores::{ScoreSink, ScoreTransport}, +}; diff --git a/crates/observability/src/exporters/langfuse/scores.rs b/crates/observability/src/exporters/langfuse/scores.rs new file mode 100644 index 0000000000..35d0748d1f --- /dev/null +++ b/crates/observability/src/exporters/langfuse/scores.rs @@ -0,0 +1,571 @@ +//! Score ingestion. +//! +//! Scores are the one part of the model OTLP cannot carry, so they take a +//! separate path: a dedicated sink that filters the fanout down to +//! [`Event::Score`] and posts stable requests to the Scores API. +//! +//! Without this sink `TurnRecorder::score` would emit into a void — the OTLP +//! mapping drops score events by design, so the Langfuse trace transport alone +//! silently discards every score. + +use std::{sync::Arc, time::Duration}; + +use { + async_trait::async_trait, + serde::Serialize, + tracing::{debug, warn}, +}; + +use { + super::{client::LangfuseClient, config::SCORES_PATH}, + crate::{ + model::{Event, ScoreDeleteRecord, ScoreRecord, ScoreValue}, + runtime::{BatchConfig, BatchSink, Transport, TransportError}, + sink::ObservationSink, + }, +}; + +/// Value shape accepted by the dedicated Scores API. BOOLEAN uses 0/1 on this +/// API even though the recorder models it as a real boolean. +#[derive(Debug, Serialize)] +#[serde(untagged)] +enum ScoreApiValue<'a> { + Numeric(f64), + Text(&'a str), +} + +/// Langfuse create-score request. +#[derive(Debug, Serialize)] +struct ScoreBody<'a> { + id: &'a str, + #[serde(rename = "traceId")] + trace_id: String, + #[serde(rename = "observationId", skip_serializing_if = "Option::is_none")] + observation_id: Option, + name: &'a str, + value: ScoreApiValue<'a>, + #[serde(rename = "dataType")] + data_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + comment: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + environment: Option<&'a str>, +} + +impl<'a> ScoreBody<'a> { + fn new(score: &'a ScoreRecord, default_environment: Option<&'a str>) -> Self { + let (value, data_type) = match &score.value { + ScoreValue::Numeric(value) => (ScoreApiValue::Numeric(*value), "NUMERIC"), + ScoreValue::Categorical(value) => (ScoreApiValue::Text(value), "CATEGORICAL"), + ScoreValue::Boolean(value) => ( + ScoreApiValue::Numeric(if *value { + 1.0 + } else { + 0.0 + }), + "BOOLEAN", + ), + }; + Self { + id: &score.id, + trace_id: score.trace_id.as_otel_hex(), + observation_id: score + .observation_id + .as_ref() + .map(crate::model::ObservationId::as_otel_hex), + name: &score.name, + value, + data_type, + comment: score.comment.as_deref(), + environment: score.environment.as_deref().or(default_environment), + } + } +} + +impl LangfuseClient { + /// Submit scores. Langfuse upserts on score id, so re-submitting a score + /// with the same id overwrites rather than duplicating — which is what + /// makes a user changing their reaction idempotent. + pub async fn submit_scores(&self, scores: &[ScoreRecord]) -> anyhow::Result<()> { + if scores.is_empty() { + return Ok(()); + } + + self.submit_scores_for_transport(scores) + .await + .map_err(anyhow::Error::new)?; + + debug!(count = scores.len(), "submitted scores to Langfuse"); + Ok(()) + } + + async fn submit_scores_for_transport( + &self, + scores: &[ScoreRecord], + ) -> Result<(), TransportError> { + for score in scores { + let body = ScoreBody::new(score, self.config().environment.as_deref()); + let response = self + .post(SCORES_PATH) + .json(&body) + .send() + .await + .map_err(|error| TransportError::Retryable(error.to_string()))?; + + let status = response.status(); + if status.is_success() { + continue; + } + let message = format!("Langfuse rejected score {} with HTTP {status}", score.id); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status == reqwest::StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + return Err(TransportError::Retryable(message)); + } + return Err(TransportError::Fatal(message)); + } + Ok(()) + } + + /// Delete a score by id. + /// + /// Backs reaction removal: a user who takes their thumb back has retracted + /// the opinion, and leaving the score in place would keep counting a vote + /// they withdrew. A 404 is success — the score is already gone, which is + /// the state the caller wanted. + pub async fn delete_score(&self, score_id: &str) -> anyhow::Result<()> { + let response = self + .delete(&format!("{SCORES_PATH}/{score_id}")) + .send() + .await?; + + let status = response.status(); + if status.is_success() || status == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + Err(anyhow::anyhow!( + "Langfuse rejected the score deletion with HTTP {status}" + )) + } + + async fn delete_score_for_transport( + &self, + deletion: &ScoreDeleteRecord, + ) -> Result<(), TransportError> { + let response = self + .delete(&format!("{SCORES_PATH}/{}", deletion.score_id)) + .send() + .await + .map_err(|error| TransportError::Retryable(error.to_string()))?; + let status = response.status(); + if status.is_success() || status == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + let message = format!( + "Langfuse rejected score deletion {} with HTTP {status}", + deletion.score_id + ); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status == reqwest::StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + Err(TransportError::Retryable(message)) + } else { + Err(TransportError::Fatal(message)) + } + } +} + +/// Batch transport that posts scores to the dedicated Scores API. +pub struct ScoreTransport { + client: Arc, +} + +impl ScoreTransport { + /// Build a transport over an existing client. + #[must_use] + pub const fn new(client: Arc) -> Self { + Self { client } + } +} + +#[async_trait] +impl Transport for ScoreTransport { + fn name(&self) -> &str { + "langfuse-scores" + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + for event in batch { + match event { + Event::Score(score) => { + self.client + .submit_scores_for_transport(std::slice::from_ref(score.as_ref())) + .await?; + }, + Event::ScoreDelete(deletion) => { + self.client + .delete_score_for_transport(deletion.as_ref()) + .await?; + }, + _ => {}, + } + } + Ok(()) + } +} + +/// Sink that carries only scores. +/// +/// The fanout hands every event to every sink, so this wrapper drops +/// non-score events before they reach the queue — otherwise a busy agent +/// would fill the score queue with trace events and evict real scores. +pub struct ScoreSink { + inner: BatchSink, +} + +impl ScoreSink { + /// Spawn a score sink over `client`. + #[must_use] + pub fn spawn(client: Arc, mut config: BatchConfig) -> Self { + // Each score is a distinct HTTP mutation. Single-event batches keep + // retries and delivery statistics exact without sacrificing queue order. + config.max_batch_events = 1; + Self { + inner: BatchSink::spawn(Arc::new(ScoreTransport::new(client)), config), + } + } +} + +#[async_trait] +impl ObservationSink for ScoreSink { + fn name(&self) -> &str { + "langfuse-scores" + } + + fn record(&self, event: Event) { + if matches!(event, Event::Score(_) | Event::ScoreDelete(_)) { + self.inner.record(event); + } + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + if let Err(error) = self.inner.flush(timeout).await { + warn!(%error, "score sink flush failed"); + return Err(error); + } + Ok(()) + } + + fn delivery_stats(&self) -> Vec { + self.inner.delivery_stats() + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + std::time::Duration, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json, method, path}, + }, + }; + + use { + super::*, + crate::{ + exporters::langfuse::config::LangfuseConfig, + model::{ObservationId, TraceId, TraceRecord}, + }, + secrecy::SecretString, + }; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + fn score(name: &str) -> ScoreRecord { + ScoreRecord::new( + TraceId("01234567-89ab-cdef-0123-456789abcdef".into()), + name, + ScoreValue::Numeric(1.0), + ) + } + + #[tokio::test] + async fn scores_post_to_the_dedicated_api_with_wire_ids() { + let server = MockServer::start().await; + let expected = serde_json::json!({ + "id": "score-1", + "traceId": "0123456789abcdef0123456789abcdef", + "observationId": "fedcba9876543210", + "name": "user-feedback", + "value": 1.0, + "dataType": "NUMERIC", + "comment": "helpful", + "environment": "production" + }); + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .and(body_json(expected)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id": "score-1" + }))) + .expect(1) + .mount(&server) + .await; + + let mut score = score("user-feedback"); + score.id = "score-1".into(); + score.observation_id = Some(ObservationId("fedcba98-7654-3210-fedc-ba9876543210".into())); + score.comment = Some("helpful".into()); + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) + .await + .expect("score submission should succeed"); + } + + #[tokio::test] + async fn boolean_scores_use_the_boolean_data_type() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .and(body_json(serde_json::json!({ + "id": "boolean-score", + "traceId": "0123456789abcdef0123456789abcdef", + "name": "user-feedback", + "value": 0.0, + "dataType": "BOOLEAN", + "environment": "production" + }))) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + + let mut score = score("user-feedback"); + score.id = "boolean-score".into(); + score.value = ScoreValue::Boolean(false); + LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) + .await + .expect("boolean score should be accepted"); + } + + #[tokio::test] + async fn empty_score_batches_skip_the_request() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[]) + .await + .expect("empty batch is a no-op"); + } + + #[test] + fn categorical_and_numeric_scores_declare_their_data_type() { + let numeric = ScoreValue::Numeric(0.5); + let categorical = ScoreValue::Categorical("helpful".into()); + + assert_eq!( + serde_json::to_value(&numeric).expect("serializable"), + serde_json::json!(0.5) + ); + assert_eq!( + serde_json::to_value(&categorical).expect("serializable"), + serde_json::json!("helpful") + ); + assert_eq!( + serde_json::to_value(ScoreValue::Boolean(true)).expect("serializable"), + serde_json::json!(true) + ); + } + + #[tokio::test] + async fn the_transport_ignores_non_score_events() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Trace(Box::new(TraceRecord::new("agent-run")))]; + + transport.send(&batch).await.expect("no scores, no request"); + } + + #[tokio::test] + async fn the_sink_drops_trace_events_before_they_reach_the_queue() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let sink = ScoreSink::spawn(client, BatchConfig::default()); + + // A busy agent emits far more trace events than scores; if they were + // queued here they would evict the scores we actually care about. + for _ in 0..64 { + sink.record(Event::Trace(Box::new(TraceRecord::new("agent-run")))); + } + sink.record(Event::Score(Box::new(score("user-feedback")))); + + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + } + + #[tokio::test] + async fn retracting_a_score_deletes_it_by_id() { + let server = MockServer::start().await; + Mock::given(method("DELETE")) + .and(path(format!("{SCORES_PATH}/score-1"))) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .delete_score("score-1") + .await + .expect("deletion should succeed"); + } + + #[tokio::test] + async fn deleting_an_already_absent_score_succeeds() { + let server = MockServer::start().await; + // Reaction-removal events can arrive twice, or after the score was + // cleaned up. Already-gone is the state the caller asked for. + Mock::given(method("DELETE")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .delete_score("missing") + .await + .expect("404 is success for a deletion"); + } + + #[tokio::test] + async fn a_failed_deletion_is_surfaced() { + let server = MockServer::start().await; + Mock::given(method("DELETE")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .delete_score("score-1") + .await + .expect_err("500 should fail"); + + assert!(error.to_string().contains("500"), "{error}"); + } + + #[tokio::test] + async fn a_rejected_batch_is_retryable_rather_than_dropped() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Score(Box::new(score("user-feedback")))]; + + let error = transport.send(&batch).await.expect_err("503 should fail"); + assert!( + matches!(error, TransportError::Retryable(_)), + "a transient rejection must not discard user feedback, got {error:?}" + ); + } + + #[tokio::test] + async fn invalid_scores_are_fatal() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(400)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Score(Box::new(score("user-feedback")))]; + + let error = transport.send(&batch).await.expect_err("400 should fail"); + assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + } + + #[test] + fn retry_payload_is_stable() { + let mut score = score("user-feedback"); + score.id = "stable-score".into(); + score.value = ScoreValue::Boolean(true); + let first = + serde_json::to_vec(&ScoreBody::new(&score, Some("production"))).expect("serializable"); + let second = + serde_json::to_vec(&ScoreBody::new(&score, Some("production"))).expect("serializable"); + assert_eq!(first, second); + } + + #[tokio::test] + async fn create_and_delete_are_delivered_in_queue_order() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path(format!("{SCORES_PATH}/ordered-score"))) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let mut created = score("user-feedback"); + created.id = "ordered-score".into(); + let deleted = ScoreDeleteRecord::new(created.trace_id.clone(), created.id.clone()); + + transport + .send(&[ + Event::Score(Box::new(created)), + Event::ScoreDelete(Box::new(deleted)), + ]) + .await + .expect("ordered mutations should succeed"); + + let requests = server.received_requests().await.expect("request log"); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method.as_str(), "POST"); + assert_eq!(requests[1].method.as_str(), "DELETE"); + } +} diff --git a/crates/observability/src/exporters/mod.rs b/crates/observability/src/exporters/mod.rs new file mode 100644 index 0000000000..a34e48cf7b --- /dev/null +++ b/crates/observability/src/exporters/mod.rs @@ -0,0 +1,7 @@ +//! Backend exporters. + +#[cfg(feature = "langfuse")] +pub mod langfuse; + +#[cfg(feature = "otlp")] +pub mod otlp; diff --git a/crates/observability/src/exporters/otlp/attributes.rs b/crates/observability/src/exporters/otlp/attributes.rs new file mode 100644 index 0000000000..b929824642 --- /dev/null +++ b/crates/observability/src/exporters/otlp/attributes.rs @@ -0,0 +1,104 @@ +//! Attribute names emitted on every exported span. +//! +//! Two vocabularies are written side by side: +//! +//! * `gen_ai.*` — the OpenTelemetry GenAI semantic conventions, understood by +//! Grafana, Datadog, Honeycomb and anything else that speaks OTel. +//! * `langfuse.*` — Langfuse's own mapping, which carries the richer +//! observation taxonomy (AGENT/TOOL/RETRIEVER/...), usage and cost detail, +//! and managed-prompt linkage that the GenAI conventions have no slot for. +//! +//! Emitting both costs a few hundred bytes per span and means a single +//! instrumentation pass serves every backend. Unknown attributes are ignored by +//! conformant backends, so the extra keys are inert where unrecognised. +//! +//! The `langfuse.*` names are taken verbatim from +//! `packages/shared/src/server/otel/attributes.ts` in the Langfuse source, +//! which is ahead of the published documentation page. + +// ── Langfuse: trace scope ─────────────────────────────────────────────────── + +/// Trace display name. +pub const TRACE_NAME: &str = "langfuse.trace.name"; +/// End-user identity. Primary spelling; `langfuse.user.id` is the legacy alias. +pub const USER_ID: &str = "user.id"; +/// Session grouping. Primary spelling; `langfuse.session.id` is the legacy alias. +pub const SESSION_ID: &str = "session.id"; +/// Filterable trace labels. +pub const TRACE_TAGS: &str = "langfuse.trace.tags"; +/// Trace-level input payload. +pub const TRACE_INPUT: &str = "langfuse.trace.input"; +/// Trace-level output payload. +pub const TRACE_OUTPUT: &str = "langfuse.trace.output"; +/// Trace-level structured metadata. +pub const TRACE_METADATA: &str = "langfuse.trace.metadata"; +/// Whether the trace is publicly viewable. +pub const TRACE_PUBLIC: &str = "langfuse.trace.public"; + +// ── Langfuse: observation ─────────────────────────────────────────────────── + +/// Observation taxonomy slot (`GENERATION`, `TOOL`, `AGENT`, ...). +pub const OBSERVATION_TYPE: &str = "langfuse.observation.type"; +/// Observation severity. +pub const OBSERVATION_LEVEL: &str = "langfuse.observation.level"; +/// Error or warning detail. +pub const OBSERVATION_STATUS_MESSAGE: &str = "langfuse.observation.status_message"; +/// Observation input payload. +pub const OBSERVATION_INPUT: &str = "langfuse.observation.input"; +/// Observation output payload. +pub const OBSERVATION_OUTPUT: &str = "langfuse.observation.output"; +/// Observation structured metadata. +pub const OBSERVATION_METADATA: &str = "langfuse.observation.metadata"; +/// Time the first output token arrived. +pub const OBSERVATION_COMPLETION_START_TIME: &str = "langfuse.observation.completion_start_time"; +/// Model identifier. +pub const OBSERVATION_MODEL: &str = "langfuse.observation.model.name"; +/// Sampling parameters. +pub const OBSERVATION_MODEL_PARAMETERS: &str = "langfuse.observation.model.parameters"; +/// Token usage breakdown. +pub const OBSERVATION_USAGE_DETAILS: &str = "langfuse.observation.usage_details"; +/// Managed-prompt name this generation was rendered from. +pub const OBSERVATION_PROMPT_NAME: &str = "langfuse.observation.prompt.name"; +/// Managed-prompt version this generation was rendered from. +pub const OBSERVATION_PROMPT_VERSION: &str = "langfuse.observation.prompt.version"; + +// ── Langfuse: general ─────────────────────────────────────────────────────── + +/// Deployment environment. +pub const ENVIRONMENT: &str = "langfuse.environment"; +/// Build release identifier. +pub const RELEASE: &str = "langfuse.release"; +/// Application-level version. +pub const VERSION: &str = "langfuse.version"; + +// ── OpenTelemetry GenAI semantic conventions ──────────────────────────────── + +/// Provider name, e.g. `anthropic`. +pub const GEN_AI_SYSTEM: &str = "gen_ai.system"; +/// Operation being performed, e.g. `chat`. +pub const GEN_AI_OPERATION_NAME: &str = "gen_ai.operation.name"; +/// Model requested. +pub const GEN_AI_REQUEST_MODEL: &str = "gen_ai.request.model"; +/// Model that actually served the request. +pub const GEN_AI_RESPONSE_MODEL: &str = "gen_ai.response.model"; +/// Requested sampling temperature. +pub const GEN_AI_REQUEST_TEMPERATURE: &str = "gen_ai.request.temperature"; +/// Requested output token ceiling. +pub const GEN_AI_REQUEST_MAX_TOKENS: &str = "gen_ai.request.max_tokens"; +/// Fresh input tokens consumed. +pub const GEN_AI_USAGE_INPUT_TOKENS: &str = "gen_ai.usage.input_tokens"; +/// Output tokens generated. +pub const GEN_AI_USAGE_OUTPUT_TOKENS: &str = "gen_ai.usage.output_tokens"; +/// Conversation grouping id. +pub const GEN_AI_CONVERSATION_ID: &str = "gen_ai.conversation.id"; +/// Tool name, for tool spans. +pub const GEN_AI_TOOL_NAME: &str = "gen_ai.tool.name"; + +// ── Resource attributes ───────────────────────────────────────────────────── + +/// Emitting service name. +pub const SERVICE_NAME: &str = "service.name"; +/// Emitting service version. +pub const SERVICE_VERSION: &str = "service.version"; +/// Deployment environment, OTel's own spelling. +pub const DEPLOYMENT_ENVIRONMENT: &str = "deployment.environment.name"; diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs new file mode 100644 index 0000000000..9a38e7ed34 --- /dev/null +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -0,0 +1,1200 @@ +//! Projection of the internal trace model onto OTLP spans. +//! +//! Every emission decision is gated on the sink's [`ExportProfile`], so the +//! same agent run yields a content-rich payload for Langfuse and a lean +//! operational span for Datadog or Grafana. See `crate::profile` for why. + +use std::collections::HashSet; + +use time::OffsetDateTime; + +use { + super::{attributes as attr, wire}, + crate::{ + model::{Event, Level, ObservationRecord, TraceRecord, TraceScope}, + profile::ExportProfile, + }, +}; + +/// Instrumentation scope name reported to backends. +pub const SCOPE_NAME: &str = "moltis"; + +/// Identity and policy applied to one export batch. +#[derive(Debug, Clone)] +pub struct ExportContext { + /// Value reported as `service.name`. + pub service_name: String, + /// Value reported as `service.version`. + pub service_version: String, + /// Deployment environment. + pub environment: Option, + /// What this backend is allowed to receive. + pub profile: ExportProfile, +} + +/// Convert an RFC3339 instant to nanoseconds since the Unix epoch. +fn unix_nanos(at: OffsetDateTime) -> String { + let nanos = at.unix_timestamp_nanos(); + // OTLP treats the field as unsigned; pre-epoch instants cannot occur here + // but clamping is cheaper than reasoning about a negative timestamp + // silently wrapping in a collector. + if nanos < 0 { + return "0".to_string(); + } + nanos.to_string() +} + +/// Truncate an attribute value to the profile's ceiling on a char boundary. +fn clamp(text: String, max: usize) -> String { + if max == 0 || text.len() <= max { + return text; + } + const SUFFIX: &str = "..."; + if max <= SUFFIX.len() { + return SUFFIX[..max].to_string(); + } + let mut end = max - SUFFIX.len(); + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + format!("{}{SUFFIX}", &text[..end]) +} + +/// Serialize a value for an attribute slot that expects JSON text. +fn json_attr(key: &str, value: &serde_json::Value, max: usize) -> Option { + match value { + serde_json::Value::Null => None, + serde_json::Value::String(s) => Some(wire::KeyValue::string(key, clamp(s.clone(), max))), + other => bounded_json(other, max).map(|s| wire::KeyValue::string(key, s)), + } +} + +/// Serialize structured JSON without ever truncating it into invalid syntax. +fn bounded_json(value: &serde_json::Value, max: usize) -> Option { + let serialized = serde_json::to_string(value).ok()?; + if max == 0 || serialized.len() <= max { + return Some(serialized); + } + + let marker = serde_json::json!({ + "moltis_truncated": true, + "original_bytes": serialized.len(), + }) + .to_string(); + if marker.len() <= max { + return Some(marker); + } + (max >= 4).then(|| "null".to_string()) +} + +fn push_structured_attr( + out: &mut Vec, + key: &str, + value: &serde_json::Value, + max: usize, +) { + if let Some(serialized) = bounded_json(value, max) { + out.push(wire::KeyValue::string(key, serialized)); + } +} + +/// Approximate size of a payload, for backends that get counts but not content. +fn payload_size(value: &serde_json::Value) -> i64 { + match value { + serde_json::Value::String(s) => s.len() as i64, + other => serde_json::to_string(other) + .map(|s| s.len() as i64) + .unwrap_or(0), + } +} + +/// Append attributes carried by every span in a trace. +fn push_scope_attrs(out: &mut Vec, scope: &TraceScope, profile: &ExportProfile) { + if let Some(session) = &scope.session_id + && profile.emit_session_id + { + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string(attr::SESSION_ID, session.clone())); + } + // GenAI semconv's own conversation grouping, understood everywhere. + out.push(wire::KeyValue::string( + attr::GEN_AI_CONVERSATION_ID, + session.clone(), + )); + } + if let Some(user) = &scope.user_id + && profile.emit_user_id + { + out.push(wire::KeyValue::string(attr::USER_ID, user.clone())); + } + if !scope.tags.is_empty() && profile.emit_tags && profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string_array(attr::TRACE_TAGS, &scope.tags)); + } + if let Some(env) = &scope.environment { + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string(attr::ENVIRONMENT, env.clone())); + } + out.push(wire::KeyValue::string( + attr::DEPLOYMENT_ENVIRONMENT, + env.clone(), + )); + } + if !profile.emits_langfuse_attrs() { + return; + } + if let Some(release) = &scope.release { + out.push(wire::KeyValue::string(attr::RELEASE, release.clone())); + } + if let Some(version) = &scope.version { + out.push(wire::KeyValue::string(attr::VERSION, version.clone())); + } +} + +/// Build the span representing a trace's root. +/// +/// Langfuse reconstructs the trace from its spans, so the completed root is a +/// real AGENT observation and also carries the trace-level compatibility attrs. +#[must_use] +pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { + let profile = &ctx.profile; + let mut attributes = Vec::new(); + let error_message = trace + .metadata + .get("error") + .and_then(serde_json::Value::as_str); + + if profile.emits_langfuse_attrs() { + attributes.push(wire::KeyValue::string(attr::TRACE_NAME, trace.name.clone())); + attributes.push(wire::KeyValue::string(attr::OBSERVATION_TYPE, "agent")); + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_LEVEL, + if error_message.is_some() { + Level::Error.as_str() + } else { + Level::Default.as_str() + }, + )); + if let Some(message) = error_message { + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_STATUS_MESSAGE, + clamp(message.to_string(), profile.max_attribute_bytes), + )); + } + } else { + attributes.push(wire::KeyValue::string(attr::GEN_AI_OPERATION_NAME, "agent")); + } + push_scope_attrs(&mut attributes, &trace.scope, profile); + + if profile.emits_bodies() { + if let Some(input) = &trace.input + && let Some(kv) = json_attr(attr::TRACE_INPUT, input, profile.max_attribute_bytes) + { + attributes.push(kv.clone()); + if let Some(observation) = + json_attr(attr::OBSERVATION_INPUT, input, profile.max_attribute_bytes) + { + attributes.push(observation); + } + } + if let Some(output) = &trace.output + && let Some(kv) = json_attr(attr::TRACE_OUTPUT, output, profile.max_attribute_bytes) + { + attributes.push(kv.clone()); + if let Some(observation) = json_attr( + attr::OBSERVATION_OUTPUT, + output, + profile.max_attribute_bytes, + ) { + attributes.push(observation); + } + } + } else if profile.emits_content_metadata() { + // Size without content: enough to spot a runaway prompt in an APM + // without copying the prompt itself into it. + if let Some(input) = &trace.input { + attributes.push(wire::KeyValue::int( + "moltis.input.bytes", + payload_size(input), + )); + } + if let Some(output) = &trace.output { + attributes.push(wire::KeyValue::int( + "moltis.output.bytes", + payload_size(output), + )); + } + } + + if profile.emits_langfuse_attrs() { + if !trace.metadata.is_empty() { + let metadata = serde_json::to_value(&trace.metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, + attr::TRACE_METADATA, + &metadata, + profile.max_attribute_bytes, + ); + push_structured_attr( + &mut attributes, + attr::OBSERVATION_METADATA, + &metadata, + profile.max_attribute_bytes, + ); + } + if trace.public { + attributes.push(wire::KeyValue::bool(attr::TRACE_PUBLIC, true)); + } + } + + let start = unix_nanos(trace.timestamp); + wire::Span { + trace_id: trace.id.as_otel_hex(), + // Deriving the root span id from the trace id keeps it stable across + // repeated trace updates, so the backend upserts instead of creating + // a fresh root span each time. + span_id: crate::model::ObservationId(trace.id.0.clone()).as_otel_hex(), + parent_span_id: None, + name: trace.name.clone(), + kind: wire::SPAN_KIND_INTERNAL, + end_time_unix_nano: unix_nanos(trace.end_time.unwrap_or(trace.timestamp)), + start_time_unix_nano: start, + attributes, + status: error_message.map_or_else(wire::Status::unset, |message| { + wire::Status::error(Some(message.to_string())) + }), + } +} + +/// Build the span representing an observation. +#[must_use] +pub fn observation_to_span(obs: &ObservationRecord, ctx: &ExportContext) -> wire::Span { + let profile = &ctx.profile; + let mut attributes = Vec::new(); + + if profile.emits_langfuse_attrs() { + if let Some(trace_name) = &obs.trace_name { + attributes.push(wire::KeyValue::string(attr::TRACE_NAME, trace_name.clone())); + } + if !obs.trace_metadata.is_empty() { + let metadata = serde_json::to_value(&obs.trace_metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, + attr::TRACE_METADATA, + &metadata, + profile.max_attribute_bytes, + ); + } + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_TYPE, + obs.kind.as_str(), + )); + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_LEVEL, + obs.level.as_str(), + )); + } else { + // Ops backends still need to group by step kind; a lowercase + // operation name is the GenAI-conventional way to express it. + attributes.push(wire::KeyValue::string( + attr::GEN_AI_OPERATION_NAME, + obs.kind.as_str(), + )); + } + + push_scope_attrs(&mut attributes, &obs.scope, profile); + + // Error detail is operational, not conversational: every backend gets it. + if let Some(message) = &obs.status_message { + let key = if profile.emits_langfuse_attrs() { + attr::OBSERVATION_STATUS_MESSAGE + } else { + "error.message" + }; + attributes.push(wire::KeyValue::string( + key, + clamp(message.clone(), profile.max_attribute_bytes), + )); + } + + push_payload_attrs(&mut attributes, obs, profile); + push_model_attrs(&mut attributes, obs, profile); + push_usage_attrs(&mut attributes, obs, profile); + + if profile.emits_langfuse_attrs() { + push_prompt_attrs(&mut attributes, obs); + if !obs.metadata.is_empty() { + let metadata = serde_json::to_value(&obs.metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, + attr::OBSERVATION_METADATA, + &metadata, + profile.max_attribute_bytes, + ); + } + } + + if matches!( + obs.kind, + crate::model::ObservationKind::Tool | crate::model::ObservationKind::Retriever + ) { + attributes.push(wire::KeyValue::string( + attr::GEN_AI_TOOL_NAME, + obs.name.clone(), + )); + } + + let status = if obs.level == Level::Error { + wire::Status::error(obs.status_message.clone()) + } else { + wire::Status::unset() + }; + + wire::Span { + trace_id: obs.trace_id.as_otel_hex(), + span_id: obs.id.as_otel_hex(), + parent_span_id: Some( + obs.parent_id + .as_ref() + .map_or_else( + || crate::model::ObservationId(obs.trace_id.0.clone()), + Clone::clone, + ) + .as_otel_hex(), + ), + name: obs.name.clone(), + kind: wire::SPAN_KIND_INTERNAL, + start_time_unix_nano: unix_nanos(obs.start_time), + // A still-running observation reports its start as its end; the update + // that arrives when it finishes carries the real end time. + end_time_unix_nano: unix_nanos(obs.end_time.unwrap_or(obs.start_time)), + attributes, + status, + } +} + +/// Input and output payloads, or their sizes. +fn push_payload_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + if profile.emits_bodies() { + if let Some(input) = &obs.input + && let Some(kv) = json_attr(attr::OBSERVATION_INPUT, input, profile.max_attribute_bytes) + { + out.push(kv); + } + if let Some(output) = &obs.output + && let Some(kv) = json_attr( + attr::OBSERVATION_OUTPUT, + output, + profile.max_attribute_bytes, + ) + { + out.push(kv); + } + return; + } + if !profile.emits_content_metadata() { + return; + } + if let Some(input) = &obs.input { + out.push(wire::KeyValue::int( + "moltis.input.bytes", + payload_size(input), + )); + } + if let Some(output) = &obs.output { + out.push(wire::KeyValue::int( + "moltis.output.bytes", + payload_size(output), + )); + } +} + +/// Model identity and sampling parameters. +fn push_model_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + let Some(model) = &obs.model else { + return; + }; + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string( + attr::OBSERVATION_MODEL, + model.clone(), + )); + } + // Model identity is low-cardinality and operationally essential, so it + // goes to every backend. + out.push(wire::KeyValue::string( + attr::GEN_AI_REQUEST_MODEL, + model.clone(), + )); + out.push(wire::KeyValue::string( + attr::GEN_AI_RESPONSE_MODEL, + model.clone(), + )); + + if obs.model_parameters.is_empty() { + return; + } + if profile.emits_langfuse_attrs() { + let parameters = serde_json::to_value(&obs.model_parameters).unwrap_or_default(); + push_structured_attr( + out, + attr::OBSERVATION_MODEL_PARAMETERS, + ¶meters, + profile.max_attribute_bytes, + ); + } + // Promote the two parameters GenAI consumers actually chart. + if let Some(temp) = obs + .model_parameters + .get("temperature") + .and_then(serde_json::Value::as_f64) + { + out.push(wire::KeyValue::double( + attr::GEN_AI_REQUEST_TEMPERATURE, + temp, + )); + } + if let Some(max) = obs + .model_parameters + .get("max_tokens") + .and_then(serde_json::Value::as_i64) + { + out.push(wire::KeyValue::int(attr::GEN_AI_REQUEST_MAX_TOKENS, max)); + } +} + +/// Token usage and cost. +fn push_usage_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + if !profile.emit_usage { + return; + } + + if let Some(completion_start) = obs.completion_start_time + && profile.emits_langfuse_attrs() + { + // Langfuse parses this as an ISO-8601 instant, not a nanosecond count. + if let Ok(text) = completion_start.format(&time::format_description::well_known::Rfc3339) { + out.push(wire::KeyValue::string( + attr::OBSERVATION_COMPLETION_START_TIME, + text, + )); + } + } + + if let Some(usage) = obs.usage.filter(|u| !u.is_empty()) { + if profile.emits_langfuse_attrs() { + // Only Langfuse prices the cache buckets; elsewhere this is an + // opaque JSON blob nothing can chart. + let details = usage.to_usage_details(); + let details = serde_json::to_value(details).unwrap_or_default(); + push_structured_attr( + out, + attr::OBSERVATION_USAGE_DETAILS, + &details, + profile.max_attribute_bytes, + ); + } + out.push(wire::KeyValue::int( + attr::GEN_AI_USAGE_INPUT_TOKENS, + i64::from(usage.input), + )); + out.push(wire::KeyValue::int( + attr::GEN_AI_USAGE_OUTPUT_TOKENS, + i64::from(usage.output), + )); + } + + // No cost attribute: Langfuse maintains versioned model definitions and + // prices spend from the model plus these counts, and no other backend has + // a price table to send one to. +} + +/// Managed-prompt linkage. Langfuse-only: no other backend models it. +fn push_prompt_attrs(out: &mut Vec, obs: &ObservationRecord) { + if let Some(name) = &obs.prompt_name { + out.push(wire::KeyValue::string( + attr::OBSERVATION_PROMPT_NAME, + name.clone(), + )); + } + if let Some(version) = obs.prompt_version { + out.push(wire::KeyValue::int( + attr::OBSERVATION_PROMPT_VERSION, + i64::from(version), + )); + } +} + +/// Resource attributes describing this Moltis instance. +#[must_use] +pub fn resource_attributes(ctx: &ExportContext) -> Vec { + let mut attrs = vec![ + wire::KeyValue::string(attr::SERVICE_NAME, ctx.service_name.clone()), + wire::KeyValue::string(attr::SERVICE_VERSION, ctx.service_version.clone()), + ]; + if let Some(env) = &ctx.environment { + attrs.push(wire::KeyValue::string( + attr::DEPLOYMENT_ENVIRONMENT, + env.clone(), + )); + if ctx.profile.emits_langfuse_attrs() { + attrs.push(wire::KeyValue::string(attr::ENVIRONMENT, env.clone())); + } + } + attrs +} + +/// Convert a batch of events into an OTLP export request. +/// +/// Score events carry no span representation and are skipped here; they are +/// delivered through the backend's scoring API instead. +#[must_use] +pub fn batch_to_request(events: &[Event], ctx: &ExportContext) -> wire::ExportTraceServiceRequest { + let mut exported_ids = HashSet::new(); + let spans: Vec = events + .iter() + .filter_map(|event| match event { + Event::Trace(trace) if trace.end_time.is_some() => Some(trace_to_span(trace, ctx)), + Event::ObservationEnd(obs) if obs.end_time.is_some() => { + Some(observation_to_span(obs, ctx)) + }, + Event::Trace(_) | Event::ObservationEnd(_) => None, + Event::Score(_) | Event::ScoreDelete(_) => None, + }) + .filter(|span| exported_ids.insert((span.trace_id.clone(), span.span_id.clone()))) + .collect(); + + wire::ExportTraceServiceRequest { + resource_spans: vec![wire::ResourceSpans { + resource: wire::Resource { + attributes: resource_attributes(ctx), + }, + scope_spans: vec![wire::ScopeSpans { + scope: wire::InstrumentationScope { + name: SCOPE_NAME.to_string(), + version: ctx.service_version.clone(), + }, + spans, + }], + }], + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use serde_json::json; + + use { + super::*, + crate::model::{ + ObservationId, ObservationKind, ScoreRecord, ScoreValue, TokenUsage, TraceId, + }, + }; + + fn ctx(profile: ExportProfile) -> ExportContext { + ExportContext { + service_name: "moltis".into(), + service_version: "20260726.01".into(), + environment: Some("production".into()), + profile, + } + } + + fn langfuse_ctx() -> ExportContext { + ctx(ExportProfile::langfuse()) + } + + fn otel_ctx() -> ExportContext { + ctx(ExportProfile::otel_generic()) + } + + fn attr_value(span: &wire::Span, key: &str) -> Option { + let json = serde_json::to_value(span).ok()?; + json["attributes"] + .as_array()? + .iter() + .find(|kv| kv["key"] == key) + .map(|kv| kv["value"].clone()) + } + + fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:main".into()), + user_id: Some("telegram:42".into()), + tags: vec!["telegram".into(), "agent:main".into()], + environment: Some("production".into()), + release: Some("20260726.01".into()), + version: Some("preset-v3".into()), + } + } + + fn generation() -> ObservationRecord { + let mut obs = ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "anthropic/claude", + ) + .with_scope(scope()); + obs.model = Some("claude-opus-4".into()); + obs.usage = Some(TokenUsage::from_provider_totals(100, 50, 900, 20)); + obs.input = Some(json!({ "messages": [{ "role": "user", "content": "secret plan" }] })); + obs.output = Some(json!("here is the answer")); + obs.finish(); + obs + } + + // ── Profile separation: the core of the Langfuse/APM split ────────── + + #[test] + fn langfuse_receives_conversation_bodies() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_some()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_some()); + } + + #[test] + fn generic_otel_never_receives_conversation_bodies() { + // The guarantee that makes it safe to point this at a shared APM. + let span = observation_to_span(&generation(), &otel_ctx()); + let rendered = serde_json::to_string(&span).expect("serializable"); + + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_none()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_none()); + assert!( + !rendered.contains("secret plan"), + "prompt content leaked into an APM payload" + ); + assert!(!rendered.contains("here is the answer")); + } + + #[test] + fn generic_otel_receives_payload_sizes_instead_of_payloads() { + // Enough to alert on a runaway prompt without copying the prompt. + let span = observation_to_span(&generation(), &otel_ctx()); + assert!(matches!( + attr_value(&span, "moltis.input.bytes"), + Some(v) if v["intValue"].is_string() + )); + } + + #[test] + fn content_none_emits_neither_bodies_nor_sizes() { + let profile = ExportProfile { + content: crate::profile::ContentCapture::None, + ..ExportProfile::otel_generic() + }; + let span = observation_to_span(&generation(), &ctx(profile)); + + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_none()); + assert!(attr_value(&span, "moltis.input.bytes").is_none()); + } + + #[test] + fn generic_otel_omits_langfuse_vocabulary() { + let span = observation_to_span(&generation(), &otel_ctx()); + let rendered = serde_json::to_string(&span).expect("serializable"); + assert!( + !rendered.contains("langfuse."), + "langfuse-specific attributes are noise in an APM" + ); + } + + #[test] + fn generic_otel_omits_user_and_session_ids() { + let span = observation_to_span(&generation(), &otel_ctx()); + + // Per-user cardinality is a billing and compliance problem in an APM. + assert!(attr_value(&span, attr::USER_ID).is_none()); + // Channel session keys contain account and peer identifiers and are + // therefore no safer than the explicit user id. + assert!(attr_value(&span, attr::GEN_AI_CONVERSATION_ID).is_none()); + } + + #[test] + fn datadog_profile_drops_tags() { + let span = observation_to_span(&generation(), &ctx(ExportProfile::datadog())); + assert!(attr_value(&span, attr::TRACE_TAGS).is_none()); + } + + #[test] + fn generic_otel_expresses_step_kind_as_gen_ai_operation() { + let mut obs = generation(); + obs.kind = ObservationKind::Retriever; + let span = observation_to_span(&obs, &otel_ctx()); + + assert_eq!( + attr_value(&span, attr::GEN_AI_OPERATION_NAME), + Some(json!({ "stringValue": "retriever" })) + ); + assert!(attr_value(&span, attr::OBSERVATION_TYPE).is_none()); + } + + #[test] + fn model_and_error_reach_every_backend() { + // Operationally essential and low-cardinality, so never gated. + let mut obs = generation(); + obs.fail("provider returned 500"); + + for context in [langfuse_ctx(), otel_ctx(), ctx(ExportProfile::datadog())] { + let span = observation_to_span(&obs, &context); + assert!(attr_value(&span, attr::GEN_AI_REQUEST_MODEL).is_some()); + assert_eq!(span.status.code, wire::STATUS_ERROR); + } + } + + #[test] + fn token_counts_reach_every_backend_but_usage_details_do_not() { + let obs = generation(); + + let otel = observation_to_span(&obs, &otel_ctx()); + assert!(attr_value(&otel, attr::GEN_AI_USAGE_INPUT_TOKENS).is_some()); + // The cache-aware breakdown is a Langfuse concept; an APM gets counts. + assert!(attr_value(&otel, attr::OBSERVATION_USAGE_DETAILS).is_none()); + + let lf = observation_to_span(&obs, &langfuse_ctx()); + assert!(attr_value(&lf, attr::GEN_AI_USAGE_INPUT_TOKENS).is_some()); + assert!(attr_value(&lf, attr::OBSERVATION_USAGE_DETAILS).is_some()); + } + + #[test] + fn usage_can_be_disabled_entirely() { + let profile = ExportProfile { + emit_usage: false, + ..ExportProfile::otel_generic() + }; + let span = observation_to_span(&generation(), &ctx(profile)); + assert!(attr_value(&span, attr::GEN_AI_USAGE_INPUT_TOKENS).is_none()); + } + + #[test] + fn prompt_linkage_is_langfuse_only() { + let mut obs = generation(); + obs.prompt_name = Some("system-prompt".into()); + obs.prompt_version = Some(7); + + assert!( + attr_value( + &observation_to_span(&obs, &langfuse_ctx()), + attr::OBSERVATION_PROMPT_NAME + ) + .is_some() + ); + assert!( + attr_value( + &observation_to_span(&obs, &otel_ctx()), + attr::OBSERVATION_PROMPT_NAME + ) + .is_none() + ); + } + + #[test] + fn trace_bodies_are_withheld_from_an_apm() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!("private question")); + trace.output = Some(json!("private answer")); + + let rendered = + serde_json::to_string(&trace_to_span(&trace, &otel_ctx())).expect("serializable"); + assert!(!rendered.contains("private question")); + assert!(!rendered.contains("private answer")); + } + + #[test] + fn attribute_values_are_clamped_to_the_profile_ceiling() { + let mut obs = generation(); + obs.output = Some(json!("x".repeat(100_000))); + let profile = ExportProfile { + max_attribute_bytes: 100, + ..ExportProfile::langfuse() + }; + let span = observation_to_span(&obs, &ctx(profile)); + let value = attr_value(&span, attr::OBSERVATION_OUTPUT) + .and_then(|v| v["stringValue"].as_str().map(str::to_string)) + .expect("output present"); + + assert!( + value.len() <= 110, + "clamped value was {} bytes", + value.len() + ); + } + + #[test] + fn clamping_does_not_split_multibyte_characters() { + // Slicing blindly at a byte offset would panic on this input. + let clamped = clamp("ααααααααα".to_string(), 5); + assert!(clamped.ends_with("...")); + assert!(clamped.len() <= 5); + } + + // ── Shared structural behaviour ───────────────────────────────────── + + #[test] + fn observation_type_uses_langfuse_taxonomy() { + let mut obs = generation(); + obs.kind = ObservationKind::Tool; + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::OBSERVATION_TYPE), + Some(json!({ "stringValue": "tool" })) + ); + } + + #[test] + fn tool_spans_carry_gen_ai_tool_name_on_every_backend() { + let mut obs = generation(); + obs.kind = ObservationKind::Tool; + obs.name = "exec".into(); + + for context in [langfuse_ctx(), otel_ctx()] { + assert_eq!( + attr_value(&observation_to_span(&obs, &context), attr::GEN_AI_TOOL_NAME), + Some(json!({ "stringValue": "exec" })) + ); + } + } + + #[test] + fn scope_attributes_ride_on_every_span() { + // OTLP has no trace-level attribute concept, so a backend filtering by + // session sees only what each span itself carries. + let span = observation_to_span(&generation(), &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::SESSION_ID), + Some(json!({ "stringValue": "agent:main:main" })) + ); + assert_eq!( + attr_value(&span, attr::USER_ID), + Some(json!({ "stringValue": "telegram:42" })) + ); + assert_eq!( + attr_value(&span, attr::RELEASE), + Some(json!({ "stringValue": "20260726.01" })) + ); + } + + #[test] + fn usage_details_carry_cache_buckets_separately() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + let raw = attr_value(&span, attr::OBSERVATION_USAGE_DETAILS) + .and_then(|v| v["stringValue"].as_str().map(str::to_string)) + .expect("usage details present"); + let parsed: serde_json::Value = serde_json::from_str(&raw).expect("valid json"); + + assert_eq!(parsed["input"], 100); + assert_eq!(parsed["input_cache_read"], 900); + assert_eq!(parsed["input_cache_write"], 20); + } + + #[test] + fn empty_usage_is_omitted_rather_than_reported_as_zero() { + let mut obs = generation(); + obs.usage = Some(TokenUsage::default()); + let span = observation_to_span(&obs, &langfuse_ctx()); + + // A zero-token generation distorts the backend's cost charts. + assert!(attr_value(&span, attr::OBSERVATION_USAGE_DETAILS).is_none()); + assert!(attr_value(&span, attr::GEN_AI_USAGE_INPUT_TOKENS).is_none()); + } + + #[test] + fn error_level_sets_otel_status_code_two() { + let mut obs = generation(); + obs.fail("provider returned 500"); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!(span.status.code, wire::STATUS_ERROR); + assert_eq!( + span.status.message.as_deref(), + Some("provider returned 500") + ); + } + + #[test] + fn apm_gets_error_message_under_the_conventional_key() { + let mut obs = generation(); + obs.fail("provider returned 500"); + let span = observation_to_span(&obs, &otel_ctx()); + + assert_eq!( + attr_value(&span, "error.message"), + Some(json!({ "stringValue": "provider returned 500" })) + ); + } + + #[test] + fn non_error_levels_leave_status_unset() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + assert_eq!(span.status.code, wire::STATUS_UNSET); + } + + #[test] + fn orphan_observations_are_parented_to_the_trace_root() { + // Otherwise a top-level observation becomes a second root span and the + // backend renders two disconnected traces. + let obs = generation(); + let span = observation_to_span(&obs, &langfuse_ctx()); + let expected = ObservationId(obs.trace_id.0.clone()).as_otel_hex(); + + assert_eq!(span.parent_span_id, Some(expected)); + } + + #[test] + fn nested_observations_point_at_their_parent() { + let parent = generation(); + let child = + ObservationRecord::start(parent.trace_id.clone(), ObservationKind::Tool, "exec") + .with_parent(Some(parent.id.clone())); + + let span = observation_to_span(&child, &langfuse_ctx()); + assert_eq!(span.parent_span_id, Some(parent.id.as_otel_hex())); + } + + #[test] + fn trace_root_span_id_is_derived_from_the_trace_id() { + // A fresh random root id per update would create duplicate roots. + let trace = TraceRecord::new("turn"); + let first = trace_to_span(&trace, &langfuse_ctx()); + let second = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!(first.span_id, second.span_id); + assert!(first.parent_span_id.is_none()); + } + + #[test] + fn completed_trace_is_a_duration_bearing_agent_observation() { + let mut trace = TraceRecord::new("turn"); + trace.timestamp = OffsetDateTime::UNIX_EPOCH; + trace.end_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)); + trace.input = Some(json!({ "role": "user", "content": "hello" })); + trace.output = Some(json!({ "role": "assistant", "content": "hi" })); + let span = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!(span.start_time_unix_nano, "0"); + assert_eq!(span.end_time_unix_nano, "2000000000"); + assert_eq!( + attr_value(&span, attr::OBSERVATION_TYPE), + Some(json!({ "stringValue": "agent" })) + ); + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_some()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_some()); + // Trace attrs are retained because Langfuse builds its trace view from + // them independently of the root observation view. + assert!(attr_value(&span, attr::TRACE_INPUT).is_some()); + assert!(attr_value(&span, attr::TRACE_OUTPUT).is_some()); + } + + #[test] + fn child_spans_carry_filterable_trace_context() { + let mut obs = generation(); + obs.trace_name = Some("support-agent".into()); + obs.trace_metadata.insert("tenant".into(), json!("acme")); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::TRACE_NAME), + Some(json!({ "stringValue": "support-agent" })) + ); + let metadata_attr = attr_value(&span, attr::TRACE_METADATA).expect("trace metadata"); + let metadata = metadata_attr["stringValue"] + .as_str() + .map(|value| serde_json::from_str::(value).expect("valid json")) + .expect("trace metadata string"); + assert_eq!(metadata["tenant"], "acme"); + } + + #[test] + fn string_inputs_are_not_double_encoded() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!("what is the weather")); + let span = trace_to_span(&trace, &langfuse_ctx()); + + // Wrapping a plain string in JSON quotes renders as `"\"text\""` in the + // backend UI, on every single trace. + assert_eq!( + attr_value(&span, attr::TRACE_INPUT), + Some(json!({ "stringValue": "what is the weather" })) + ); + } + + #[test] + fn structured_inputs_are_json_encoded() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!({ "role": "user" })); + let span = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::TRACE_INPUT), + Some(json!({ "stringValue": "{\"role\":\"user\"}" })) + ); + } + + #[test] + fn null_payloads_are_dropped() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(serde_json::Value::Null); + let span = trace_to_span(&trace, &langfuse_ctx()); + assert!(attr_value(&span, attr::TRACE_INPUT).is_none()); + } + + #[test] + fn completion_start_time_is_rfc3339_not_nanos() { + let mut obs = generation(); + obs.completion_start_time = Some(OffsetDateTime::UNIX_EPOCH); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::OBSERVATION_COMPLETION_START_TIME), + Some(json!({ "stringValue": "1970-01-01T00:00:00Z" })) + ); + } + + #[test] + fn running_observations_report_end_equal_to_start() { + let mut obs = generation(); + obs.end_time = None; + let span = observation_to_span(&obs, &langfuse_ctx()); + assert_eq!(span.start_time_unix_nano, span.end_time_unix_nano); + } + + #[test] + fn model_parameters_promote_temperature_and_max_tokens() { + let mut obs = generation(); + obs.model_parameters + .insert("temperature".into(), json!(0.7)); + obs.model_parameters + .insert("max_tokens".into(), json!(4096)); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::GEN_AI_REQUEST_TEMPERATURE), + Some(json!({ "doubleValue": 0.7 })) + ); + assert_eq!( + attr_value(&span, attr::GEN_AI_REQUEST_MAX_TOKENS), + Some(json!({ "intValue": "4096" })) + ); + } + + #[test] + fn in_progress_spans_are_dropped_for_all_immutable_span_backends() { + let mut obs = generation(); + obs.end_time = None; + let events = vec![Event::ObservationEnd(Box::new(obs))]; + + let lf = batch_to_request(&events, &langfuse_ctx()); + assert!(lf.resource_spans[0].scope_spans[0].spans.is_empty()); + + // Tempo and Datadog would render this and the later completion as two + // separate spans with the same id. + let otel = batch_to_request(&events, &otel_ctx()); + assert!(otel.resource_spans[0].scope_spans[0].spans.is_empty()); + } + + #[test] + fn completed_spans_reach_every_backend() { + let events = vec![Event::ObservationEnd(Box::new(generation()))]; + for context in [langfuse_ctx(), otel_ctx()] { + let request = batch_to_request(&events, &context); + assert_eq!(request.resource_spans[0].scope_spans[0].spans.len(), 1); + } + } + + #[test] + fn scores_produce_no_spans() { + let events = vec![Event::Score(Box::new(ScoreRecord::new( + TraceId::generate(), + "user-feedback", + ScoreValue::Numeric(1.0), + )))]; + let request = batch_to_request(&events, &langfuse_ctx()); + + assert!(request.resource_spans[0].scope_spans[0].spans.is_empty()); + } + + #[test] + fn batch_carries_resource_and_scope_identity() { + let mut trace = TraceRecord::new("turn"); + trace.finish(); + let events = vec![Event::Trace(Box::new(trace))]; + let request = batch_to_request(&events, &langfuse_ctx()); + let json = serde_json::to_value(&request).expect("serializable"); + + assert_eq!( + json["resourceSpans"][0]["scopeSpans"][0]["scope"]["name"], + SCOPE_NAME + ); + let names: Vec<&str> = json["resourceSpans"][0]["resource"]["attributes"] + .as_array() + .map(|a| a.iter().filter_map(|kv| kv["key"].as_str()).collect()) + .unwrap_or_default(); + assert!(names.contains(&attr::SERVICE_NAME)); + assert!(names.contains(&attr::DEPLOYMENT_ENVIRONMENT)); + } + + #[test] + fn pre_epoch_timestamps_clamp_to_zero() { + let mut trace = TraceRecord::new("turn"); + trace.timestamp = OffsetDateTime::UNIX_EPOCH - time::Duration::days(1); + let span = trace_to_span(&trace, &langfuse_ctx()); + + // OTLP treats the field as unsigned; a negative value wraps to a + // far-future timestamp in collectors. + assert_eq!(span.start_time_unix_nano, "0"); + } + + #[test] + fn incomplete_roots_and_duplicate_completed_ids_are_filtered() { + let incomplete = TraceRecord::new("turn"); + let mut complete = incomplete.clone(); + complete.finish(); + let observation = generation(); + let events = vec![ + Event::Trace(Box::new(incomplete)), + Event::Trace(Box::new(complete.clone())), + Event::Trace(Box::new(complete)), + Event::ObservationEnd(Box::new(observation.clone())), + Event::ObservationEnd(Box::new(observation)), + ]; + let request = batch_to_request(&events, &langfuse_ctx()); + let spans = &request.resource_spans[0].scope_spans[0].spans; + let ids: HashSet<_> = spans.iter().map(|span| span.span_id.as_str()).collect(); + + assert_eq!(spans.len(), 2); + assert_eq!(ids.len(), spans.len(), "each wire id must occur once"); + } + + #[test] + fn oversized_structured_attributes_remain_valid_json_within_the_limit() { + let mut obs = generation(); + obs.output = Some(json!({ "items": ["x".repeat(500), "y".repeat(500)] })); + obs.metadata.insert("large".into(), json!("z".repeat(500))); + let profile = ExportProfile { + max_attribute_bytes: 80, + ..ExportProfile::langfuse() + }; + let span = observation_to_span(&obs, &ctx(profile)); + + for key in [attr::OBSERVATION_OUTPUT, attr::OBSERVATION_METADATA] { + let attribute = attr_value(&span, key).expect("attribute present"); + let value = attribute["stringValue"].as_str().expect("string value"); + assert!(value.len() <= 80); + serde_json::from_str::(value).expect("valid bounded json"); + } + } +} diff --git a/crates/observability/src/exporters/otlp/mod.rs b/crates/observability/src/exporters/otlp/mod.rs new file mode 100644 index 0000000000..30d58c5668 --- /dev/null +++ b/crates/observability/src/exporters/otlp/mod.rs @@ -0,0 +1,422 @@ +//! OTLP/HTTP JSON exporter. +//! +//! One transport serves every backend we target: Langfuse (which accepts OTLP +//! on `/api/public/otel/v1/traces`), Grafana Tempo/Alloy, Datadog's OTLP +//! intake, Honeycomb, and any other OTLP-compatible collector. The difference +//! between them is the endpoint and the auth header, not the payload. + +pub mod attributes; +pub mod mapping; +pub mod wire; + +use std::{collections::BTreeMap, time::Duration}; + +use { + async_trait::async_trait, + reqwest::{ + StatusCode, + header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue}, + }, + tracing::{debug, warn}, +}; + +use crate::{ + model::Event, + profile::ExportProfile, + runtime::{Transport, TransportError}, +}; + +/// Configuration for an OTLP exporter. +#[derive(Clone)] +pub struct OtlpConfig { + /// Display name for logs and status reporting. + pub name: String, + /// Full traces endpoint, e.g. `https://cloud.langfuse.com/api/public/otel/v1/traces`. + pub endpoint: String, + /// Additional headers, typically carrying authentication. + pub headers: BTreeMap, + /// Per-request timeout. + pub timeout: Duration, + /// Value reported as `service.name`. + pub service_name: String, + /// Value reported as `service.version`. + pub service_version: String, + /// Deployment environment, reported on the resource and every span. + pub environment: Option, + /// What this backend is allowed to receive. Defaults to the conservative + /// profile so a misconfigured backend cannot silently receive prompts. + pub profile: ExportProfile, +} + +impl std::fmt::Debug for OtlpConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OtlpConfig") + .field("name", &self.name) + .field("endpoint", &self.endpoint) + .field("header_names", &self.headers.keys().collect::>()) + .field("timeout", &self.timeout) + .field("service_name", &self.service_name) + .field("service_version", &self.service_version) + .field("environment", &self.environment) + .field("profile", &self.profile) + .finish() + } +} + +impl Default for OtlpConfig { + fn default() -> Self { + Self { + name: "otlp".to_string(), + endpoint: String::new(), + headers: BTreeMap::new(), + timeout: Duration::from_secs(10), + service_name: "moltis".to_string(), + service_version: env!("CARGO_PKG_VERSION").to_string(), + environment: None, + profile: ExportProfile::default(), + } + } +} + +/// OTLP/HTTP JSON transport. +pub struct OtlpTransport { + config: OtlpConfig, + client: reqwest::Client, + headers: HeaderMap, +} + +impl OtlpTransport { + /// Build a transport, reusing the workspace HTTP client so the configured + /// upstream proxy and default headers apply. + /// + /// Header values that cannot be encoded are dropped with a warning rather + /// than failing construction: one malformed custom header should not + /// disable instrumentation entirely. + #[must_use] + pub fn new(config: OtlpConfig) -> Self { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + + for (key, value) in &config.headers { + let Ok(name) = key.parse::() else { + warn!(header = %key, "ignoring invalid OTLP header name"); + continue; + }; + let Ok(mut val) = HeaderValue::from_str(value) else { + warn!(header = %key, "ignoring OTLP header with unencodable value"); + continue; + }; + // Credentials must never surface in a debug dump of the headers. + val.set_sensitive(true); + headers.insert(name, val); + } + + Self { + client: moltis_common::http_client::build_default_http_client(), + headers, + config, + } + } + + /// The configured endpoint. + #[must_use] + pub fn endpoint(&self) -> &str { + &self.config.endpoint + } + + /// Identity and policy applied to each exported batch. + fn export_context(&self) -> mapping::ExportContext { + mapping::ExportContext { + service_name: self.config.service_name.clone(), + service_version: self.config.service_version.clone(), + environment: self.config.environment.clone(), + profile: self.config.profile.clone(), + } + } + + /// Classify an HTTP status into a retry decision. + /// + /// 4xx other than 408/429 are fatal: retrying a rejected payload or bad + /// credentials cannot succeed and only burns rate-limit budget. + fn classify(status: StatusCode) -> TransportError { + let message = format!("collector rejected OTLP batch with HTTP {status}"); + if status == StatusCode::TOO_MANY_REQUESTS + || status == StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + TransportError::Retryable(message) + } else { + TransportError::Fatal(message) + } + } +} + +#[async_trait] +impl Transport for OtlpTransport { + fn name(&self) -> &str { + &self.config.name + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + let request = mapping::batch_to_request(batch, &self.export_context()); + + // A batch of only score events produces no spans; sending an empty + // resourceSpans payload is a wasted round trip. + if request + .resource_spans + .iter() + .all(|rs| rs.scope_spans.iter().all(|ss| ss.spans.is_empty())) + { + return Ok(()); + } + + let body = serde_json::to_vec(&request) + .map_err(|e| TransportError::Fatal(format!("failed to encode OTLP payload: {e}")))?; + + let response = self + .client + .post(&self.config.endpoint) + .headers(self.headers.clone()) + .timeout(self.config.timeout) + .body(body) + .send() + .await + .map_err(|e| { + // Connection-level failures are always worth retrying: the + // collector may simply be restarting. + TransportError::Retryable(format!("collector request failed: {e}")) + })?; + + let status = response.status(); + if status.is_success() { + debug!( + sink = %self.config.name, + events = batch.len(), + "OTLP batch accepted" + ); + return Ok(()); + } + + Err(Self::classify(status)) + } + + fn accepts(&self, event: &Event) -> bool { + !matches!(event, Event::Score(_) | Event::ScoreDelete(_)) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId}, + }; + + fn config(endpoint: String) -> OtlpConfig { + let mut headers = BTreeMap::new(); + headers.insert("Authorization".to_string(), "Basic dGVzdA==".to_string()); + OtlpConfig { + name: "test".into(), + endpoint, + headers, + timeout: Duration::from_secs(5), + service_name: "moltis".into(), + service_version: "test".into(), + environment: Some("staging".into()), + profile: ExportProfile::langfuse(), + } + } + + fn events() -> Vec { + let mut obs = + ObservationRecord::start(TraceId::generate(), ObservationKind::Generation, "llm-call"); + obs.finish(); + vec![Event::ObservationEnd(Box::new(obs))] + } + + #[tokio::test] + async fn posts_json_with_configured_headers() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/traces")) + .and(header("content-type", "application/json")) + .and(header("authorization", "Basic dGVzdA==")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + transport + .send(&events()) + .await + .expect("send should succeed"); + } + + #[tokio::test] + async fn empty_span_batches_skip_the_request() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + // Scores map to no spans, so this batch has nothing to send. + let score = crate::model::ScoreRecord::new( + TraceId::generate(), + "feedback", + crate::model::ScoreValue::Numeric(1.0), + ); + transport + .send(&[Event::Score(Box::new(score))]) + .await + .expect("empty batch should be a no-op"); + } + + #[tokio::test] + async fn server_errors_are_retryable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("503 should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[tokio::test] + async fn rate_limiting_is_retryable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(429)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("429 should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[tokio::test] + async fn unauthorized_is_fatal_not_retried() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("401 should fail"); + + // Retrying bad credentials cannot succeed and wastes rate-limit budget. + assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + } + + #[tokio::test] + async fn bad_request_is_fatal() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(400) + .set_body_string("echoed prompt and Authorization: Basic super-secret"), + ) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("400 should fail"); + + let TransportError::Fatal(message) = error else { + panic!("expected fatal error"); + }; + assert!(message.contains("HTTP 400")); + assert!(!message.contains("echoed prompt")); + assert!(!message.contains("super-secret")); + } + + #[tokio::test] + async fn connection_failure_is_retryable() { + // Port 1 is reserved and refuses connections immediately. + let transport = OtlpTransport::new(config("http://127.0.0.1:1/v1/traces".into())); + let error = transport + .send(&events()) + .await + .expect_err("connection refused should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[test] + fn invalid_headers_are_skipped_without_disabling_the_exporter() { + let mut headers = BTreeMap::new(); + headers.insert("valid-header".to_string(), "ok".to_string()); + // Newlines cannot be encoded in a header value. + headers.insert("bad".to_string(), "line\nbreak".to_string()); + // Spaces are not legal in a header name. + headers.insert("in valid".to_string(), "x".to_string()); + + let transport = OtlpTransport::new(OtlpConfig { + headers, + ..Default::default() + }); + + assert!(transport.headers.contains_key("valid-header")); + assert!(!transport.headers.contains_key("bad")); + assert_eq!(transport.headers.len(), 2, "content-type plus valid-header"); + } + + #[test] + fn credential_headers_are_marked_sensitive() { + let mut headers = BTreeMap::new(); + headers.insert("authorization".to_string(), "Basic secret".to_string()); + let transport = OtlpTransport::new(OtlpConfig { + headers, + ..Default::default() + }); + + let value = transport + .headers + .get("authorization") + .expect("header present"); + // Sensitive headers are redacted by reqwest's own Debug output. + assert!(value.is_sensitive()); + assert!(!format!("{:?}", transport.headers).contains("Basic secret")); + } + + #[test] + fn config_debug_omits_header_values() { + let mut config = config("https://collector.example/v1/traces".into()); + config + .headers + .insert("Authorization".into(), "Bearer super-secret".into()); + + let rendered = format!("{config:?}"); + + assert!(rendered.contains("Authorization")); + assert!(!rendered.contains("super-secret")); + } +} diff --git a/crates/observability/src/exporters/otlp/wire.rs b/crates/observability/src/exporters/otlp/wire.rs new file mode 100644 index 0000000000..f7e9e19f48 --- /dev/null +++ b/crates/observability/src/exporters/otlp/wire.rs @@ -0,0 +1,312 @@ +//! OTLP/HTTP JSON wire types. +//! +//! Hand-written rather than generated from protobuf: the JSON encoding is a +//! small, stable subset and hand-rolling it avoids pulling `prost`, `tonic` and +//! a build-time protoc dependency into the workspace for one request shape. +//! +//! The one sharp edge in OTLP/JSON is that 64-bit integers are encoded as +//! **decimal strings**, not JSON numbers, because JSON numbers cannot carry the +//! full int64 range. Timestamps and `intValue` therefore serialize as strings. + +use serde::Serialize; + +/// Top-level OTLP trace export request. +#[derive(Debug, Clone, Serialize)] +pub struct ExportTraceServiceRequest { + /// Spans grouped by emitting resource. + #[serde(rename = "resourceSpans")] + pub resource_spans: Vec, +} + +/// Spans from one resource (one service instance). +#[derive(Debug, Clone, Serialize)] +pub struct ResourceSpans { + /// Attributes describing the emitting service. + pub resource: Resource, + /// Spans grouped by instrumentation scope. + #[serde(rename = "scopeSpans")] + pub scope_spans: Vec, +} + +/// The emitting service. +#[derive(Debug, Clone, Serialize)] +pub struct Resource { + /// Resource-level attributes. + pub attributes: Vec, +} + +/// Spans from one instrumentation scope. +#[derive(Debug, Clone, Serialize)] +pub struct ScopeSpans { + /// The instrumentation library. + pub scope: InstrumentationScope, + /// The spans themselves. + pub spans: Vec, +} + +/// Instrumentation library identity. +#[derive(Debug, Clone, Serialize)] +pub struct InstrumentationScope { + /// Library name. + pub name: String, + /// Library version. + pub version: String, +} + +/// A single span. +#[derive(Debug, Clone, Serialize)] +pub struct Span { + /// 32-character lowercase hex trace id. + #[serde(rename = "traceId")] + pub trace_id: String, + /// 16-character lowercase hex span id. + #[serde(rename = "spanId")] + pub span_id: String, + /// Parent span id, omitted for a root span. + #[serde(rename = "parentSpanId", skip_serializing_if = "Option::is_none")] + pub parent_span_id: Option, + /// Span name. + pub name: String, + /// Span kind. `1` is INTERNAL, which is correct for in-process work. + pub kind: i32, + /// Start time in nanoseconds since the Unix epoch, as a decimal string. + #[serde(rename = "startTimeUnixNano")] + pub start_time_unix_nano: String, + /// End time in nanoseconds since the Unix epoch, as a decimal string. + #[serde(rename = "endTimeUnixNano")] + pub end_time_unix_nano: String, + /// Span attributes. + pub attributes: Vec, + /// Span status. + pub status: Status, +} + +/// Span kind: in-process work. +pub const SPAN_KIND_INTERNAL: i32 = 1; + +/// Span completion status. +#[derive(Debug, Clone, Serialize)] +pub struct Status { + /// `0` unset, `1` OK, `2` ERROR. + pub code: i32, + /// Human-readable detail, present only on error. + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +/// Status code: no explicit status. +pub const STATUS_UNSET: i32 = 0; +/// Status code: failed. +pub const STATUS_ERROR: i32 = 2; + +impl Status { + /// A status carrying no signal. + #[must_use] + pub const fn unset() -> Self { + Self { + code: STATUS_UNSET, + message: None, + } + } + + /// A failure status with detail. + #[must_use] + pub fn error(message: Option) -> Self { + Self { + code: STATUS_ERROR, + message, + } + } +} + +/// One attribute. +#[derive(Debug, Clone, Serialize)] +pub struct KeyValue { + /// Attribute name. + pub key: String, + /// Attribute value. + pub value: AnyValue, +} + +impl KeyValue { + /// String-valued attribute. + #[must_use] + pub fn string(key: impl Into, value: impl Into) -> Self { + Self { + key: key.into(), + value: AnyValue::String { + string_value: value.into(), + }, + } + } + + /// Integer-valued attribute. + #[must_use] + pub fn int(key: impl Into, value: i64) -> Self { + Self { + key: key.into(), + value: AnyValue::Int { + int_value: value.to_string(), + }, + } + } + + /// Float-valued attribute. + #[must_use] + pub fn double(key: impl Into, value: f64) -> Self { + Self { + key: key.into(), + value: AnyValue::Double { + double_value: value, + }, + } + } + + /// Boolean-valued attribute. + #[must_use] + pub fn bool(key: impl Into, value: bool) -> Self { + Self { + key: key.into(), + value: AnyValue::Bool { bool_value: value }, + } + } + + /// String-array-valued attribute. + #[must_use] + pub fn string_array(key: impl Into, values: &[String]) -> Self { + Self { + key: key.into(), + value: AnyValue::Array { + array_value: ArrayValue { + values: values + .iter() + .map(|v| AnyValue::String { + string_value: v.clone(), + }) + .collect(), + }, + }, + } + } +} + +/// OTLP `AnyValue`. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum AnyValue { + /// UTF-8 string. + String { + /// The string. + #[serde(rename = "stringValue")] + string_value: String, + }, + /// 64-bit integer, encoded as a decimal string per OTLP/JSON. + Int { + /// The integer, decimal-encoded. + #[serde(rename = "intValue")] + int_value: String, + }, + /// Double-precision float. + Double { + /// The float. + #[serde(rename = "doubleValue")] + double_value: f64, + }, + /// Boolean. + Bool { + /// The boolean. + #[serde(rename = "boolValue")] + bool_value: bool, + }, + /// Homogeneous array. + Array { + /// The array. + #[serde(rename = "arrayValue")] + array_value: ArrayValue, + }, +} + +/// OTLP array value wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct ArrayValue { + /// Array elements. + pub values: Vec, +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn int_values_serialize_as_decimal_strings() { + // OTLP/JSON requires int64 as a string; emitting a JSON number here + // makes collectors reject or silently truncate large values. + let kv = KeyValue::int("gen_ai.usage.input_tokens", 9_007_199_254_740_993); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["value"]["intValue"], "9007199254740993"); + assert!(json["value"]["intValue"].is_string()); + } + + #[test] + fn string_values_serialize_under_string_value() { + let kv = KeyValue::string("service.name", "moltis"); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["key"], "service.name"); + assert_eq!(json["value"]["stringValue"], "moltis"); + } + + #[test] + fn bool_and_double_use_their_own_slots() { + let b = serde_json::to_value(KeyValue::bool("k", true)).expect("serializable"); + let d = serde_json::to_value(KeyValue::double("k", 0.5)).expect("serializable"); + + assert_eq!(b["value"]["boolValue"], true); + assert_eq!(d["value"]["doubleValue"], 0.5); + } + + #[test] + fn string_arrays_nest_under_array_value() { + let kv = KeyValue::string_array("langfuse.trace.tags", &["a".into(), "b".into()]); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["value"]["arrayValue"]["values"][0]["stringValue"], "a"); + assert_eq!(json["value"]["arrayValue"]["values"][1]["stringValue"], "b"); + } + + #[test] + fn unset_status_omits_the_message_field() { + let json = serde_json::to_value(Status::unset()).expect("serializable"); + assert_eq!(json["code"], 0); + assert!(json.get("message").is_none()); + } + + #[test] + fn error_status_carries_code_two_and_detail() { + let json = serde_json::to_value(Status::error(Some("boom".into()))).expect("serializable"); + assert_eq!(json["code"], 2); + assert_eq!(json["message"], "boom"); + } + + #[test] + fn root_span_omits_parent_span_id() { + let span = Span { + trace_id: "0".repeat(32), + span_id: "0".repeat(16), + parent_span_id: None, + name: "root".into(), + kind: SPAN_KIND_INTERNAL, + start_time_unix_nano: "1".into(), + end_time_unix_nano: "2".into(), + attributes: Vec::new(), + status: Status::unset(), + }; + let json = serde_json::to_value(&span).expect("serializable"); + + // A present-but-null parentSpanId makes some collectors treat the span + // as a child of a missing parent, orphaning the trace. + assert!(json.get("parentSpanId").is_none()); + } +} diff --git a/crates/observability/src/feedback.rs b/crates/observability/src/feedback.rs new file mode 100644 index 0000000000..c8c04a99b9 --- /dev/null +++ b/crates/observability/src/feedback.rs @@ -0,0 +1,371 @@ +//! End-user feedback and its mapping onto scores. +//! +//! A reaction on a chat message is the cheapest quality signal there is: no +//! form, no rating widget, just a thumb. This module turns one into a +//! [`ScoreRecord`] without knowing anything about the channel it came from — +//! Telegram sends a raw emoji, Slack sends a shortcode, Discord sends either, +//! and all three normalize to the same signal here. + +use std::collections::BTreeSet; + +use crate::model::{ScoreRecord, ScoreValue, TraceId}; + +/// Score name used for end-user reaction feedback. +/// +/// Fixed rather than configurable: it is the join key for every feedback +/// dashboard, and letting it drift per deployment makes those dashboards +/// silently stop matching. +pub const USER_FEEDBACK_SCORE: &str = "user-feedback"; + +/// Namespace for deriving deterministic feedback score ids. +const FEEDBACK_NAMESPACE: uuid::Uuid = uuid::uuid!("6f9f4d3a-1b2c-4e5f-8a90-0b1c2d3e4f50"); + +/// A feedback signal expressed by an end user. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FeedbackSignal { + /// The answer was good. + Positive, + /// The answer was bad. + Negative, +} + +impl FeedbackSignal { + /// Boolean value carried to the backend. + #[must_use] + pub const fn score_value(self) -> bool { + match self { + Self::Positive => true, + Self::Negative => false, + } + } +} + +/// Reaction tokens that count as feedback. +/// +/// Configurable because reaction conventions are team-specific: some treat 🎉 +/// as praise, others use it for releases only. The defaults stay deliberately +/// narrow — a broad default vocabulary turns every celebratory emoji into a +/// quality score and quietly poisons the metric. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FeedbackVocabulary { + positive: BTreeSet, + negative: BTreeSet, +} + +impl Default for FeedbackVocabulary { + fn default() -> Self { + Self::new(DEFAULT_POSITIVE, DEFAULT_NEGATIVE) + } +} + +/// Unambiguous approval, in both raw-emoji and shortcode form. +const DEFAULT_POSITIVE: &[&str] = &[ + "\u{1f44d}", // thumbs up + "+1", + "thumbsup", + "\u{2764}", // red heart + "heart", + "\u{1f4af}", // hundred points + "100", + "\u{2705}", // check mark button + "white_check_mark", +]; + +/// Unambiguous disapproval. +const DEFAULT_NEGATIVE: &[&str] = &[ + "\u{1f44e}", // thumbs down + "-1", + "thumbsdown", + "\u{274c}", // cross mark + "x", + "\u{1f621}", // enraged face + "rage", + "\u{1f4a9}", // pile of poo + "poop", +]; + +impl FeedbackVocabulary { + /// Build a vocabulary from explicit token lists. + #[must_use] + pub fn new>(positive: &[S], negative: &[S]) -> Self { + Self { + positive: positive.iter().map(|s| normalize(s.as_ref())).collect(), + negative: negative.iter().map(|s| normalize(s.as_ref())).collect(), + } + } + + /// Build from configured lists, falling back to the defaults when a list is + /// empty. + /// + /// Per-list rather than all-or-nothing so an operator can extend the + /// positive side without having to restate the negative one. + #[must_use] + pub fn from_config(positive: &[String], negative: &[String]) -> Self { + let default = Self::default(); + Self { + positive: if positive.is_empty() { + default.positive + } else { + positive.iter().map(|s| normalize(s)).collect() + }, + negative: if negative.is_empty() { + default.negative + } else { + negative.iter().map(|s| normalize(s)).collect() + }, + } + } + + /// Classify a raw reaction token, or `None` when it carries no signal. + #[must_use] + pub fn classify(&self, raw: &str) -> Option { + let token = normalize(raw); + if token.is_empty() { + return None; + } + if self.positive.contains(&token) { + return Some(FeedbackSignal::Positive); + } + if self.negative.contains(&token) { + return Some(FeedbackSignal::Negative); + } + None + } +} + +/// Reduce a reaction token to a comparable form. +/// +/// Handles the three shapes the channels actually send: a bare emoji +/// (Telegram, Discord), a shortcode (Slack), and a shortcode wrapped in colons +/// or carrying a skin-tone suffix. Skin tone and presentation selectors are +/// stripped so `👍🏾` and `👍` are the same signal — a preference about how an +/// emoji renders is not a different opinion about the answer. +fn normalize(raw: &str) -> String { + let trimmed = raw.trim().trim_matches(':'); + // Slack encodes skin tone as a `::skin-tone-4` suffix on the shortcode. + let base = trimmed.split("::").next().unwrap_or(trimmed); + + base.chars() + .filter(|c| !is_modifier(*c)) + .flat_map(char::to_lowercase) + .collect() +} + +/// Whether `c` only affects rendering rather than meaning. +const fn is_modifier(c: char) -> bool { + matches!( + c, + // Variation selectors: text vs emoji presentation. + '\u{fe0e}' | '\u{fe0f}' + // Fitzpatrick skin tone modifiers. + | '\u{1f3fb}'..='\u{1f3ff}' + ) +} + +/// Build a score for a feedback signal against a trace. +/// +/// The score id is derived from the trace, the score name and the reacting +/// user, so it is stable across submissions. Langfuse upserts on score id, +/// which makes this idempotent in exactly the way reactions need: a user who +/// switches from 👍 to 👎 overwrites their own score instead of adding a +/// second one, and a duplicate webhook delivery is a no-op rather than +/// double-counting. +#[must_use] +pub fn feedback_score( + trace_id: &TraceId, + signal: FeedbackSignal, + user_id: Option<&str>, + comment: Option, + environment: Option, +) -> ScoreRecord { + ScoreRecord { + id: feedback_score_id(trace_id, user_id), + trace_id: trace_id.clone(), + observation_id: None, + name: USER_FEEDBACK_SCORE.to_string(), + value: ScoreValue::Boolean(signal.score_value()), + comment, + environment, + } +} + +/// Deterministic score id for one user's feedback on one trace. +#[must_use] +pub fn feedback_score_id(trace_id: &TraceId, user_id: Option<&str>) -> String { + let key = format!( + "{}|{USER_FEEDBACK_SCORE}|{}", + trace_id.0, + user_id.unwrap_or("anonymous") + ); + uuid::Uuid::new_v5(&FEEDBACK_NAMESPACE, key.as_bytes()).to_string() +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + fn vocab() -> FeedbackVocabulary { + FeedbackVocabulary::default() + } + + #[test] + fn a_bare_thumb_classifies_in_both_directions() { + assert_eq!( + vocab().classify("\u{1f44d}"), + Some(FeedbackSignal::Positive) + ); + assert_eq!( + vocab().classify("\u{1f44e}"), + Some(FeedbackSignal::Negative) + ); + } + + #[test] + fn slack_shortcodes_classify_the_same_as_the_emoji() { + // Slack sends `+1`, Telegram sends the emoji; both are one thumb up. + assert_eq!(vocab().classify("+1"), Some(FeedbackSignal::Positive)); + assert_eq!(vocab().classify("thumbsup"), Some(FeedbackSignal::Positive)); + assert_eq!(vocab().classify("-1"), Some(FeedbackSignal::Negative)); + assert_eq!( + vocab().classify("thumbsdown"), + Some(FeedbackSignal::Negative) + ); + } + + #[test] + fn shortcodes_wrapped_in_colons_are_accepted() { + assert_eq!(vocab().classify(":+1:"), Some(FeedbackSignal::Positive)); + } + + #[test] + fn skin_tone_is_not_a_different_opinion() { + // A rendering preference must not read as a different signal, or as no + // signal at all. + assert_eq!( + vocab().classify("\u{1f44d}\u{1f3ff}"), + Some(FeedbackSignal::Positive) + ); + assert_eq!( + vocab().classify("+1::skin-tone-4"), + Some(FeedbackSignal::Positive) + ); + } + + #[test] + fn variation_selectors_do_not_defeat_matching() { + // Telegram sends the heart with an emoji presentation selector. + assert_eq!( + vocab().classify("\u{2764}\u{fe0f}"), + Some(FeedbackSignal::Positive) + ); + } + + #[test] + fn unrelated_reactions_carry_no_signal() { + // A default vocabulary that swallowed every emoji would turn party + // reactions into quality data. + assert_eq!(vocab().classify("\u{1f389}"), None); // tada + assert_eq!(vocab().classify("eyes"), None); + assert_eq!(vocab().classify(""), None); + assert_eq!(vocab().classify(" "), None); + } + + #[test] + fn case_is_ignored_for_shortcodes() { + assert_eq!(vocab().classify("ThumbsUp"), Some(FeedbackSignal::Positive)); + } + + #[test] + fn configured_lists_replace_only_the_side_they_set() { + let custom = FeedbackVocabulary::from_config(&["\u{1f389}".to_string()], &[]); + + assert_eq!(custom.classify("\u{1f389}"), Some(FeedbackSignal::Positive)); + // The positive override must not silently wipe the negative defaults. + assert_eq!(custom.classify("\u{1f44e}"), Some(FeedbackSignal::Negative)); + // ...but it does replace the positive defaults it overrode. + assert_eq!(custom.classify("\u{1f44d}"), None); + } + + #[test] + fn empty_config_falls_back_to_the_defaults() { + let empty = FeedbackVocabulary::from_config(&[], &[]); + assert_eq!(empty, FeedbackVocabulary::default()); + } + + #[test] + fn signals_map_to_booleans() { + assert!(FeedbackSignal::Positive.score_value()); + assert!(!FeedbackSignal::Negative.score_value()); + } + + #[test] + fn the_score_id_is_stable_for_the_same_user_and_trace() { + let trace = TraceId("trace-1".into()); + let first = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("telegram:42"), + None, + None, + ); + let second = feedback_score( + &trace, + FeedbackSignal::Negative, + Some("telegram:42"), + None, + None, + ); + + // Langfuse upserts on id, so a user changing their mind must overwrite + // rather than add a second vote. + assert_eq!(first.id, second.id); + } + + #[test] + fn different_users_score_the_same_trace_independently() { + let trace = TraceId("trace-1".into()); + let alice = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("slack:alice"), + None, + None, + ); + let bob = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("slack:bob"), + None, + None, + ); + + assert_ne!(alice.id, bob.id, "one user's vote overwrote another's"); + } + + #[test] + fn the_same_user_scores_different_traces_independently() { + let user = Some("telegram:42"); + let first = feedback_score_id(&TraceId("trace-1".into()), user); + let second = feedback_score_id(&TraceId("trace-2".into()), user); + + assert_ne!(first, second); + } + + #[test] + fn a_score_carries_the_canonical_name_and_environment() { + let score = feedback_score( + &TraceId("trace-1".into()), + FeedbackSignal::Positive, + Some("web:1"), + Some("great answer".into()), + Some("production".into()), + ); + + assert_eq!(score.name, USER_FEEDBACK_SCORE); + assert_eq!(score.value, ScoreValue::Boolean(true)); + assert_eq!(score.comment.as_deref(), Some("great answer")); + assert_eq!(score.environment.as_deref(), Some("production")); + } +} diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs new file mode 100644 index 0000000000..1e2494654a --- /dev/null +++ b/crates/observability/src/lib.rs @@ -0,0 +1,47 @@ +//! Agent instrumentation with pluggable backends. +//! +//! One instrumentation pass in the agent runtime feeds any number of backends +//! through [`sink::ObservationSink`]. What each backend actually receives is +//! governed by its [`profile::ExportProfile`], because an LLM observability +//! product and an infrastructure APM want very different things: +//! +//! * **Langfuse** gets the full conversation — prompts, completions, tool +//! arguments and results — plus the observation taxonomy, cache-aware token +//! usage, and reaction feedback. Langfuse infers cost from the model and +//! usage details. +//! * **Grafana, Datadog, Honeycomb** get operational shape only: latency, +//! errors, model, token counts. No conversation content, no per-user +//! cardinality. +//! +//! See `docs/src/instrumentation.md` for the operator-facing guide. + +pub mod builder; +pub mod exporters; +pub mod feedback; +pub mod model; +pub mod profile; +pub mod recent; +pub mod recorder; +pub mod redact; +pub mod runtime; +pub mod sink; + +pub use { + builder::{BuildOutcome, BuiltInstrumentation, SkippedBackend, build}, + feedback::{ + FeedbackSignal, FeedbackVocabulary, USER_FEEDBACK_SCORE, feedback_score, feedback_score_id, + }, + model::{ + Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreDeleteRecord, + ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, + }, + profile::{ContentCapture, ExportProfile, Vocabulary}, + recent::{recent_trace, remember_trace}, + recorder::{RecorderSettings, StepGuard, TurnRecorder, wait_for_active_turns}, + redact::RedactionPolicy, + runtime::{BatchConfig, BatchSink, SinkStatsSnapshot, Transport, TransportError}, + sink::{ + ObservationSink, SinkFanout, clear_global_sink, global_sink, is_enabled, record, + set_global_sink, + }, +}; diff --git a/crates/observability/src/model.rs b/crates/observability/src/model.rs new file mode 100644 index 0000000000..1ed53db62f --- /dev/null +++ b/crates/observability/src/model.rs @@ -0,0 +1,635 @@ +//! Backend-agnostic trace model. +//! +//! The shape follows Langfuse's observation taxonomy because it is the richest +//! of the backends we target: OTLP exporters can always project a richer model +//! down onto spans, but the reverse loses tool/generation/retriever semantics. + +use std::collections::BTreeMap; + +use { + serde::{Deserialize, Serialize}, + time::OffsetDateTime, + uuid::Uuid, +}; + +// ── Identifiers ───────────────────────────────────────────────────────────── + +/// Trace identifier. One trace per agent run (one user turn). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct TraceId(pub String); + +/// Observation identifier, unique within a trace. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ObservationId(pub String); + +impl TraceId { + /// Generate a fresh random trace id. + #[must_use] + pub fn generate() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Lowercase 32-char hex form required by the OTLP wire format. + #[must_use] + pub fn as_otel_hex(&self) -> String { + otel_hex(&self.0, 32) + } +} + +impl ObservationId { + /// Generate a fresh random observation id. + #[must_use] + pub fn generate() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Lowercase 16-char hex form required by the OTLP wire format. + #[must_use] + pub fn as_otel_hex(&self) -> String { + otel_hex(&self.0, 16) + } +} + +/// Project an arbitrary id string onto a fixed-width lowercase hex string. +/// +/// UUIDs already carry 32 hex digits once dashes are stripped, so the common +/// case is a pure filter. Non-UUID ids (or ids shorter than `width`) are padded +/// deterministically so the same input always yields the same OTLP id. +fn otel_hex(raw: &str, width: usize) -> String { + let mut hex: String = raw + .chars() + .filter(char::is_ascii_hexdigit) + .map(|c| c.to_ascii_lowercase()) + .take(width) + .collect(); + if hex.len() < width { + // Deterministic padding: repeat the digest of the raw id. + let filler = format!("{:016x}", fnv1a64(raw.as_bytes())); + while hex.len() < width { + let need = width - hex.len(); + let take = need.min(filler.len()); + hex.push_str(&filler[..take]); + } + } + hex +} + +/// FNV-1a 64-bit. Used only to derive deterministic padding for OTLP ids, never +/// for anything security-relevant. +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + hash ^= u64::from(*b); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +// ── Taxonomy ──────────────────────────────────────────────────────────────── + +/// Observation kind. Mirrors Langfuse's observation taxonomy so the exporter +/// never has to guess at a mapping. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum ObservationKind { + /// Generic unit of work. + Span, + /// Point-in-time occurrence with no duration. + Event, + /// An LLM completion call. + Generation, + /// An agent run, possibly containing nested agents. + Agent, + /// A tool invocation. + Tool, + /// A fixed multi-step sequence. + Chain, + /// A retrieval/search step (memory, code index). + Retriever, + /// A quality-evaluation step. + Evaluator, + /// An embedding computation. + Embedding, + /// A policy check that may block execution. + Guardrail, +} + +impl ObservationKind { + /// Langfuse OTLP wire representation. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Span => "span", + Self::Event => "event", + Self::Generation => "generation", + Self::Agent => "agent", + Self::Tool => "tool", + Self::Chain => "chain", + Self::Retriever => "retriever", + Self::Evaluator => "evaluator", + Self::Embedding => "embedding", + Self::Guardrail => "guardrail", + } + } + + /// Whether Langfuse treats this kind as generation-like, i.e. carrying + /// model, usage and cost fields. + #[must_use] + pub const fn is_generation_like(self) -> bool { + matches!( + self, + Self::Generation + | Self::Agent + | Self::Tool + | Self::Chain + | Self::Retriever + | Self::Evaluator + | Self::Embedding + | Self::Guardrail + ) + } +} + +/// Severity attached to an observation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Level { + /// Verbose diagnostic detail. + Debug, + /// Normal operation. + #[default] + Default, + /// Recovered from something unexpected. + Warning, + /// Failed. + Error, +} + +impl Level { + /// Langfuse wire representation. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Debug => "DEBUG", + Self::Default => "DEFAULT", + Self::Warning => "WARNING", + Self::Error => "ERROR", + } + } +} + +// ── Usage and cost ────────────────────────────────────────────────────────── + +/// Token usage for a generation. +/// +/// `input` is the count of *fresh* (non-cached) input tokens. Callers that +/// receive a provider total which already includes cache reads must subtract +/// before constructing this type — see [`TokenUsage::from_provider_totals`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenUsage { + /// Fresh input tokens. + pub input: u32, + /// Generated output tokens. + pub output: u32, + /// Tokens served from the provider's prompt cache. + pub cache_read: u32, + /// Tokens written into the provider's prompt cache. + pub cache_write: u32, + /// Reasoning tokens, when the provider reports them separately. + pub reasoning: u32, +} + +impl TokenUsage { + /// Build usage from provider counters where `input` is already exclusive of + /// cached tokens (Anthropic's `input_tokens` behaves this way). + #[must_use] + pub const fn from_provider_totals( + input: u32, + output: u32, + cache_read: u32, + cache_write: u32, + ) -> Self { + Self { + input, + output, + cache_read, + cache_write, + reasoning: 0, + } + } + + /// Total tokens across every bucket. + #[must_use] + pub const fn total(&self) -> u32 { + self.input + .saturating_add(self.output) + .saturating_add(self.cache_read) + .saturating_add(self.cache_write) + } + + /// Whether every counter is zero, in which case the field is omitted from + /// the payload rather than reporting a misleading zero-token generation. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.input == 0 + && self.output == 0 + && self.cache_read == 0 + && self.cache_write == 0 + && self.reasoning == 0 + } + + /// Langfuse `usageDetails` map. + /// + /// Key names match what Langfuse's ingestion recognises for cache-aware + /// cost attribution. + #[must_use] + pub fn to_usage_details(self) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert("input".to_string(), self.input); + map.insert("output".to_string(), self.output); + if self.cache_read > 0 { + map.insert("input_cache_read".to_string(), self.cache_read); + } + if self.cache_write > 0 { + map.insert("input_cache_write".to_string(), self.cache_write); + } + if self.reasoning > 0 { + map.insert("output_reasoning_tokens".to_string(), self.reasoning); + } + map.insert("total".to_string(), self.total()); + map + } +} + +// ── Records ───────────────────────────────────────────────────────────────── + +/// Free-form metadata attached to traces and observations. +/// +/// `BTreeMap` keeps key order stable, which makes payload assertions in tests +/// deterministic. +pub type Metadata = BTreeMap; + +/// Trace-scoped attributes. +/// +/// These are duplicated onto every observation rather than held only on the +/// trace. OTLP has no trace-level attribute concept — backends reconstruct +/// trace metadata from the spans — so an exporter that kept this only on the +/// trace record would lose user and session attribution whenever a trace event +/// was dropped, batched separately, or evicted from a cache. Duplication costs +/// a few bytes per span and makes exporters stateless. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TraceScope { + /// Groups related traces in the backend's session view. + pub session_id: Option, + /// End-user identity, when known. + pub user_id: Option, + /// Filterable labels. + pub tags: Vec, + /// Deployment environment (`production`, `staging`, ...). + pub environment: Option, + /// Build release identifier. + pub release: Option, + /// Application-level version, e.g. an agent preset version. + pub version: Option, +} + +/// A trace: one agent run, corresponding to one user turn. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TraceRecord { + /// Unique trace id. + pub id: TraceId, + /// Human-readable trace name. + pub name: String, + /// Wall-clock start. + #[serde(with = "time::serde::rfc3339")] + pub timestamp: OffsetDateTime, + /// Wall-clock end; `None` until the turn is complete. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, + /// Trace-scoped attributes. + pub scope: TraceScope, + /// Structured context. + pub metadata: Metadata, + /// Turn input, subject to redaction and capture settings. + pub input: Option, + /// Turn output, subject to redaction and capture settings. + pub output: Option, + /// Whether the backend should expose this trace publicly. + pub public: bool, +} + +impl TraceRecord { + /// Start a new trace with the given name at the current instant. + #[must_use] + pub fn new(name: impl Into) -> Self { + Self { + id: TraceId::generate(), + name: name.into(), + timestamp: OffsetDateTime::now_utc(), + end_time: None, + scope: TraceScope::default(), + metadata: Metadata::new(), + input: None, + output: None, + public: false, + } + } + + /// Stamp the trace end time as now. + pub fn finish(&mut self) { + self.end_time = Some(OffsetDateTime::now_utc()); + } +} + +/// A single observation within a trace. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObservationRecord { + /// Unique observation id. + pub id: ObservationId, + /// Owning trace. + pub trace_id: TraceId, + /// Enclosing observation, when nested. + pub parent_id: Option, + /// What kind of work this represents. + pub kind: ObservationKind, + /// Human-readable name. + pub name: String, + /// Owning trace's display name, copied for stateless OTLP export. + pub trace_name: Option, + /// Owning trace's filterable metadata, copied for stateless OTLP export. + pub trace_metadata: Metadata, + /// Trace-scoped attributes, copied from the owning trace. + pub scope: TraceScope, + /// Wall-clock start. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Wall-clock end; `None` while still running. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, + /// Time the first output token arrived, for streaming generations. + #[serde(with = "time::serde::rfc3339::option")] + pub completion_start_time: Option, + /// Severity. + pub level: Level, + /// Error or warning detail. + pub status_message: Option, + /// Step input, subject to redaction and capture settings. + pub input: Option, + /// Step output, subject to redaction and capture settings. + pub output: Option, + /// Structured context. + pub metadata: Metadata, + /// Model identifier, for generation-like observations. + pub model: Option, + /// Sampling parameters, for generation-like observations. + pub model_parameters: Metadata, + /// Token usage, for generation-like observations. + pub usage: Option, + /// Managed-prompt name this generation was rendered from. + pub prompt_name: Option, + /// Managed-prompt version this generation was rendered from. + pub prompt_version: Option, +} + +impl ObservationRecord { + /// Start an observation of `kind` in `trace_id` at the current instant. + #[must_use] + pub fn start(trace_id: TraceId, kind: ObservationKind, name: impl Into) -> Self { + Self { + id: ObservationId::generate(), + trace_id, + parent_id: None, + kind, + name: name.into(), + trace_name: None, + trace_metadata: Metadata::new(), + scope: TraceScope::default(), + start_time: OffsetDateTime::now_utc(), + end_time: None, + completion_start_time: None, + level: Level::Default, + status_message: None, + input: None, + output: None, + metadata: Metadata::new(), + model: None, + model_parameters: Metadata::new(), + usage: None, + prompt_name: None, + prompt_version: None, + } + } + + /// Set the parent observation, producing a nested observation. + #[must_use] + pub fn with_parent(mut self, parent: Option) -> Self { + self.parent_id = parent; + self + } + + /// Copy trace-scoped attributes onto this observation. + #[must_use] + pub fn with_scope(mut self, scope: TraceScope) -> Self { + self.scope = scope; + self + } + + /// Copy trace identity and filterable metadata onto this observation. + #[must_use] + pub fn with_trace_context(mut self, name: String, metadata: Metadata) -> Self { + self.trace_name = Some(name); + self.trace_metadata = metadata; + self + } + + /// Stamp the end time as now. + pub fn finish(&mut self) { + self.end_time = Some(OffsetDateTime::now_utc()); + } + + /// Mark this observation as failed with the given message. + pub fn fail(&mut self, message: impl Into) { + self.level = Level::Error; + self.status_message = Some(message.into()); + } +} + +/// Kinds of score value a backend accepts. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ScoreValue { + /// Numeric score. + Numeric(f64), + /// Categorical score, e.g. "helpful". + Categorical(String), + /// Boolean score, e.g. end-user approval. + Boolean(bool), +} + +/// A quality score attached to a trace or observation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScoreRecord { + /// Unique score id. + pub id: String, + /// Trace the score applies to. + pub trace_id: TraceId, + /// Specific observation, when the score is narrower than the whole trace. + pub observation_id: Option, + /// Score name, e.g. `user-feedback`. + pub name: String, + /// The score itself. + pub value: ScoreValue, + /// Optional rationale. + pub comment: Option, + /// Deployment environment. + pub environment: Option, +} + +/// Delete a previously recorded score through the same ordered queue used for +/// score creation. Keeping both mutations on one transport prevents an older +/// queued create from racing a direct deletion and recreating the score. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScoreDeleteRecord { + /// Trace retained for sink routing and diagnostics. + pub trace_id: TraceId, + /// Stable Langfuse score id to delete. + pub score_id: String, +} + +impl ScoreDeleteRecord { + #[must_use] + pub fn new(trace_id: TraceId, score_id: impl Into) -> Self { + Self { + trace_id, + score_id: score_id.into(), + } + } +} + +impl ScoreRecord { + /// Build a score with a generated id. + #[must_use] + pub fn new(trace_id: TraceId, name: impl Into, value: ScoreValue) -> Self { + Self { + id: Uuid::new_v4().to_string(), + trace_id, + observation_id: None, + name: name.into(), + value, + comment: None, + environment: None, + } + } +} + +/// A single unit of work handed to a sink. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Event { + /// A trace completed. + Trace(Box), + /// An observation finished. + ObservationEnd(Box), + /// A score was produced. + Score(Box), + /// A score was retracted. + ScoreDelete(Box), +} + +impl Event { + /// The trace this event belongs to. + #[must_use] + pub fn trace_id(&self) -> &TraceId { + match self { + Self::Trace(t) => &t.id, + Self::ObservationEnd(o) => &o.trace_id, + Self::Score(s) => &s.trace_id, + Self::ScoreDelete(s) => &s.trace_id, + } + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn usage_details_split_cache_buckets() { + let usage = TokenUsage::from_provider_totals(100, 50, 900, 20); + let details = usage.to_usage_details(); + + // `input` must stay exclusive of cache buckets: Langfuse prices + // cache reads and writes at different rates than fresh input, so + // folding them together silently inflates the reported cost. + assert_eq!(details.get("input"), Some(&100)); + assert_eq!(details.get("output"), Some(&50)); + assert_eq!(details.get("input_cache_read"), Some(&900)); + assert_eq!(details.get("input_cache_write"), Some(&20)); + assert_eq!(details.get("total"), Some(&1070)); + } + + #[test] + fn usage_details_omit_zero_cache_buckets() { + let usage = TokenUsage::from_provider_totals(10, 5, 0, 0); + let details = usage.to_usage_details(); + + assert!(!details.contains_key("input_cache_read")); + assert!(!details.contains_key("input_cache_write")); + assert_eq!(details.get("total"), Some(&15)); + } + + #[test] + fn usage_total_saturates_instead_of_overflowing() { + let usage = TokenUsage::from_provider_totals(u32::MAX, u32::MAX, 0, 0); + assert_eq!(usage.total(), u32::MAX); + } + + #[test] + fn trace_id_renders_as_32_hex_digits() { + let hex = TraceId::generate().as_otel_hex(); + assert_eq!(hex.len(), 32); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn observation_id_renders_as_16_hex_digits() { + let hex = ObservationId::generate().as_otel_hex(); + assert_eq!(hex.len(), 16); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn otel_hex_is_deterministic_for_non_uuid_ids() { + let id = ObservationId("not-a-uuid".to_string()); + assert_eq!(id.as_otel_hex(), id.as_otel_hex()); + assert_eq!(id.as_otel_hex().len(), 16); + } + + #[test] + fn otel_hex_pads_short_ids_to_full_width() { + // "ab" contributes two hex digits; the rest must be filled. + let id = TraceId("ab".to_string()); + let hex = id.as_otel_hex(); + assert_eq!(hex.len(), 32); + assert!(hex.starts_with("ab")); + } + + #[test] + fn generation_like_kinds_match_langfuse() { + assert!(ObservationKind::Generation.is_generation_like()); + assert!(ObservationKind::Tool.is_generation_like()); + assert!(ObservationKind::Agent.is_generation_like()); + assert!(!ObservationKind::Span.is_generation_like()); + assert!(!ObservationKind::Event.is_generation_like()); + } + + #[test] + fn observation_fail_sets_error_level_and_message() { + let mut obs = + ObservationRecord::start(TraceId::generate(), ObservationKind::Generation, "llm-call"); + obs.fail("rate limited"); + + assert_eq!(obs.level, Level::Error); + assert_eq!(obs.status_message.as_deref(), Some("rate limited")); + } +} diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs new file mode 100644 index 0000000000..47a47e4cbe --- /dev/null +++ b/crates/observability/src/profile.rs @@ -0,0 +1,218 @@ +//! Per-backend export profiles. +//! +//! Langfuse and an infrastructure APM want fundamentally different things from +//! the same agent run, so they must not receive the same payload: +//! +//! * **Langfuse** is an LLM-native product. It wants the whole conversation — +//! prompts, completions, tool arguments and results — plus the observation +//! taxonomy, token usage split by cache bucket, per-turn cost, managed-prompt +//! versions, and session/user attribution. Content *is* the product. +//! +//! * **Datadog, Grafana Tempo, Honeycomb** are operational tools. They want +//! latency, error rates and throughput. Shipping prompt bodies to them is +//! actively harmful: it explodes span size and cardinality, most vendors bill +//! per ingested span-byte or custom metric, and it copies user conversation +//! content into a system that was never scoped to hold it. For these backends +//! the useful signal is the *shape* of the run — how long, how many +//! iterations, which model, did it error — not what was said. +//! +//! A profile is therefore attached to each sink, and the mapping layer consults +//! it before emitting anything. + +use serde::{Deserialize, Serialize}; + +/// How much conversation content an exporter may carry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContentCapture { + /// Export inputs, outputs, tool arguments and tool results. + Full, + /// Export structural metadata only: sizes, counts, model, timings. + #[default] + MetadataOnly, + /// Export neither content nor content-derived metadata. + None, +} + +impl ContentCapture { + /// Whether payload bodies may be exported. + #[must_use] + pub const fn allows_bodies(self) -> bool { + matches!(self, Self::Full) + } +} + +/// Which attribute vocabularies an exporter emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Vocabulary { + /// Langfuse's own `langfuse.*` mapping, plus `gen_ai.*`. + Langfuse, + /// OpenTelemetry GenAI semantic conventions only. + GenAi, +} + +/// What a given backend receives. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ExportProfile { + /// Content policy. + pub content: ContentCapture, + /// Attribute vocabulary. + pub vocabulary: Vocabulary, + /// Whether to attach end-user identity. + /// + /// High-cardinality in an APM's index and often a compliance question; + /// essential in Langfuse, where per-user analysis is a core view. + pub emit_user_id: bool, + /// Whether to attach session identity. + pub emit_session_id: bool, + /// Whether to attach free-form tags. + pub emit_tags: bool, + /// Whether to attach token usage and cost. + /// + /// Langfuse prices these; an APM generally has no model price table, so + /// the same numbers are better served from the Prometheus endpoint. + pub emit_usage: bool, + /// Ceiling on any single exported attribute value, in bytes. + pub max_attribute_bytes: usize, +} + +impl ExportProfile { + /// Profile for Langfuse: everything, because the product is built on it. + #[must_use] + pub const fn langfuse() -> Self { + Self { + content: ContentCapture::Full, + vocabulary: Vocabulary::Langfuse, + emit_user_id: true, + emit_session_id: true, + emit_tags: true, + emit_usage: true, + max_attribute_bytes: 32_768, + } + } + + /// Profile for a generic OTel backend (Grafana Tempo, Honeycomb, an + /// OpenTelemetry Collector): operational shape only, no conversation + /// content, no per-user cardinality. + #[must_use] + pub const fn otel_generic() -> Self { + Self { + content: ContentCapture::MetadataOnly, + vocabulary: Vocabulary::GenAi, + emit_user_id: false, + emit_session_id: false, + emit_tags: true, + emit_usage: true, + max_attribute_bytes: 4_096, + } + } + + /// Profile for Datadog APM. As [`Self::otel_generic`], but tags are dropped + /// too: Datadog indexes span tags and bills on custom metric cardinality, + /// so an unbounded tag set is a billing surprise waiting to happen. + #[must_use] + pub const fn datadog() -> Self { + Self { + content: ContentCapture::MetadataOnly, + vocabulary: Vocabulary::GenAi, + emit_user_id: false, + emit_session_id: false, + emit_tags: false, + emit_usage: true, + max_attribute_bytes: 4_096, + } + } + + /// Whether `langfuse.*` attributes should be written. + #[must_use] + pub const fn emits_langfuse_attrs(&self) -> bool { + matches!(self.vocabulary, Vocabulary::Langfuse) + } + + /// Whether payload bodies may be written. + #[must_use] + pub const fn emits_bodies(&self) -> bool { + self.content.allows_bodies() + } + + /// Whether content-derived structural metadata (lengths, counts) may be + /// written. Suppressed only under [`ContentCapture::None`]. + #[must_use] + pub const fn emits_content_metadata(&self) -> bool { + !matches!(self.content, ContentCapture::None) + } +} + +impl Default for ExportProfile { + fn default() -> Self { + Self::otel_generic() + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn langfuse_profile_carries_conversation_content() { + let profile = ExportProfile::langfuse(); + assert!(profile.emits_bodies()); + assert!(profile.emits_langfuse_attrs()); + assert!(profile.emit_user_id); + assert!(profile.emit_usage); + } + + #[test] + fn generic_otel_profile_withholds_conversation_content() { + // The whole point of the split: prompt bodies must not reach an APM. + let profile = ExportProfile::otel_generic(); + assert!(!profile.emits_bodies()); + assert!(!profile.emits_langfuse_attrs()); + assert!( + !profile.emit_user_id, + "user id is high-cardinality in an APM" + ); + assert!( + !profile.emit_session_id, + "channel session keys contain account and peer identifiers" + ); + // Structural metadata is still useful for latency and error analysis. + assert!(profile.emits_content_metadata()); + } + + #[test] + fn datadog_profile_also_drops_tags() { + let profile = ExportProfile::datadog(); + assert!(!profile.emits_bodies()); + assert!(!profile.emit_tags, "Datadog bills on tag cardinality"); + assert!(!profile.emit_session_id); + } + + #[test] + fn content_none_suppresses_even_derived_metadata() { + let profile = ExportProfile { + content: ContentCapture::None, + ..ExportProfile::otel_generic() + }; + assert!(!profile.emits_bodies()); + assert!(!profile.emits_content_metadata()); + } + + #[test] + fn default_profile_is_the_conservative_one() { + // Defaulting to Full would leak conversation content to any newly + // configured backend that forgot to set a profile. + assert_eq!(ExportProfile::default(), ExportProfile::otel_generic()); + assert!(!ExportProfile::default().emits_bodies()); + } + + #[test] + fn langfuse_allows_larger_attributes_than_an_apm() { + assert!( + ExportProfile::langfuse().max_attribute_bytes + > ExportProfile::otel_generic().max_attribute_bytes + ); + } +} diff --git a/crates/observability/src/recent.rs b/crates/observability/src/recent.rs new file mode 100644 index 0000000000..82153b9c38 --- /dev/null +++ b/crates/observability/src/recent.rs @@ -0,0 +1,161 @@ +//! The most recent trace per session. +//! +//! A channel reply is delivered by code far from the agent loop that produced +//! it — the runner returns text, and a separate dispatcher sends it. Feedback +//! attribution needs the trace id at that send point, and threading it through +//! every outbound signature would touch every channel for the benefit of one +//! feature. +//! +//! Sessions process turns serially, so "the current trace for this session" is +//! unambiguous at the moment its reply goes out. That is what this records. +//! +//! Bounded, because an install with many channel conversations would otherwise +//! accumulate one entry per session forever. Eviction is by insertion order, +//! and losing an entry costs a feedback attribution, never a message. + +use std::{ + collections::HashMap, + sync::{OnceLock, RwLock}, +}; + +use crate::model::TraceId; + +/// How many sessions keep a remembered trace. +const CAPACITY: usize = 4_096; + +/// Insertion-ordered bounded map of session key to trace id. +#[derive(Default)] +struct RecentTraces { + by_session: HashMap, + order: Vec, +} + +impl RecentTraces { + fn remember(&mut self, session_key: String, trace_id: TraceId) { + if self + .by_session + .insert(session_key.clone(), trace_id) + .is_none() + { + self.order.push(session_key); + } + while self.order.len() > CAPACITY { + let oldest = self.order.remove(0); + self.by_session.remove(&oldest); + } + } + + fn get(&self, session_key: &str) -> Option { + self.by_session.get(session_key).cloned() + } +} + +fn registry() -> &'static RwLock { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| RwLock::new(RecentTraces::default())) +} + +/// Record `trace_id` as the session's current trace. +/// +/// No-op for an empty session key: those would all collapse onto one entry and +/// cross-attribute unrelated turns. +pub fn remember_trace(session_key: &str, trace_id: &TraceId) { + if session_key.is_empty() { + return; + } + let mut guard = registry() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.remember(session_key.to_string(), trace_id.clone()); +} + +/// The session's most recent trace, if one is still remembered. +#[must_use] +pub fn recent_trace(session_key: &str) -> Option { + let guard = registry() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.get(session_key) +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn a_remembered_trace_is_retrievable() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t1".into()))); + } + + #[test] + fn a_newer_turn_replaces_the_previous_trace_for_the_session() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + recent.remember("s1".into(), TraceId("t2".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t2".into()))); + } + + #[test] + fn sessions_do_not_share_traces() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + recent.remember("s2".into(), TraceId("t2".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t1".into()))); + assert_eq!(recent.get("s2"), Some(TraceId("t2".into()))); + } + + #[test] + fn an_unknown_session_has_no_trace() { + assert_eq!(RecentTraces::default().get("nope"), None); + } + + #[test] + fn the_map_stays_bounded_under_many_sessions() { + // A busy install has one session per chat; without a bound this grows + // for the life of the process. + let mut recent = RecentTraces::default(); + for i in 0..(CAPACITY + 100) { + recent.remember(format!("s{i}"), TraceId(format!("t{i}"))); + } + + assert_eq!(recent.by_session.len(), CAPACITY); + assert_eq!(recent.order.len(), CAPACITY); + // The oldest were evicted, the newest retained. + assert!(recent.get("s0").is_none()); + assert!(recent.get(&format!("s{}", CAPACITY + 99)).is_some()); + } + + #[test] + fn repeated_turns_on_one_session_do_not_grow_the_map() { + let mut recent = RecentTraces::default(); + for i in 0..1_000 { + recent.remember("s1".into(), TraceId(format!("t{i}"))); + } + + assert_eq!(recent.by_session.len(), 1); + assert_eq!(recent.order.len(), 1, "insertion order list leaked entries"); + } + + #[test] + fn empty_session_keys_are_not_remembered() { + // Otherwise every unattributed turn collapses onto one entry and + // feedback lands on whichever turn happened to be last. + remember_trace("", &TraceId("t1".into())); + assert_eq!(recent_trace(""), None); + } + + #[test] + fn the_global_registry_round_trips() { + remember_trace("global-session", &TraceId("global-trace".into())); + assert_eq!( + recent_trace("global-session"), + Some(TraceId("global-trace".into())) + ); + } +} diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs new file mode 100644 index 0000000000..c12cfe25d0 --- /dev/null +++ b/crates/observability/src/recorder.rs @@ -0,0 +1,1024 @@ +//! The instrumentation API used by the agent runtime. +//! +//! Callers hold a [`TurnRecorder`] for the duration of an agent run and open a +//! [`StepGuard`] around each LLM call, tool invocation or retrieval. The +//! recorder owns id generation, parent/child nesting, trace-scope propagation +//! and redaction, so instrumentation at the call site stays to a few lines. +//! +//! Every entry point is a no-op when no sink is installed, and the constructor +//! returns `None` in that case so callers pay nothing beyond one atomic read. + +use std::sync::{ + Arc, LazyLock, Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +use {time::OffsetDateTime, tokio::sync::Notify}; + +use crate::{ + model::{ + Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, + TokenUsage, TraceId, TraceRecord, TraceScope, + }, + redact::{REDACTED, RedactionPolicy}, + sink::{self, ObservationSink}, +}; + +/// Settings that shape what a recorder emits. +#[derive(Debug, Clone)] +pub struct RecorderSettings { + /// Redaction applied to every payload before it is handed to a sink. + pub redaction: RedactionPolicy, + /// Whether to attach turn and step inputs. + pub capture_input: bool, + /// Whether to attach turn and step outputs. + pub capture_output: bool, + /// Whether to attach tool arguments and results. + pub capture_tool_io: bool, + /// Fraction of turns to trace, in `0.0..=1.0`. + pub sample_rate: f64, +} + +impl Default for RecorderSettings { + fn default() -> Self { + Self { + redaction: RedactionPolicy::default(), + capture_input: true, + capture_output: true, + capture_tool_io: true, + sample_rate: 1.0, + } + } +} + +impl RecorderSettings { + /// Whether this turn should be traced, given the sample rate. + #[must_use] + fn sampled(&self) -> bool { + if self.sample_rate >= 1.0 { + return true; + } + if self.sample_rate <= 0.0 { + return false; + } + rand::random::() < self.sample_rate + } +} + +/// Records one agent run. +pub struct TurnRecorder { + sink: Arc, + settings: RecorderSettings, + trace_id: TraceId, + scope: TraceScope, + /// The run's root observation, parent of every step. + root_id: ObservationId, + /// Trace record, retained so the closing update carries the final output. + trace: Arc>, + /// Present only for turns opened through the process-wide sink. + _active_turn: Option, +} + +static ACTIVE_TURNS: AtomicUsize = AtomicUsize::new(0); +static ACTIVE_TURNS_CHANGED: LazyLock = LazyLock::new(Notify::new); + +struct ActiveTurnGuard; + +impl ActiveTurnGuard { + fn new() -> Self { + ACTIVE_TURNS.fetch_add(1, Ordering::AcqRel); + Self + } +} + +impl Drop for ActiveTurnGuard { + fn drop(&mut self) { + ACTIVE_TURNS.fetch_sub(1, Ordering::AcqRel); + ACTIVE_TURNS_CHANGED.notify_waiters(); + } +} + +/// Wait until all turns opened through the process-wide sink have closed. +/// +/// Returns `false` when the deadline expires first. +pub async fn wait_for_active_turns(timeout: std::time::Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let changed = ACTIVE_TURNS_CHANGED.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); + if ACTIVE_TURNS.load(Ordering::Acquire) == 0 { + return true; + } + if tokio::time::timeout_at(deadline, changed).await.is_err() { + return false; + } + } +} + +impl TurnRecorder { + /// Begin recording a turn, or return `None` when instrumentation is off or + /// the turn was not sampled. + #[must_use] + pub fn begin( + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + ) -> Option { + if !settings.sampled() { + return None; + } + let (sink, active_turn) = + sink::with_global_sink(|sink| (Arc::clone(sink), ActiveTurnGuard::new()))?; + Some(Self::begin_inner( + sink, + name, + scope, + settings, + Some(active_turn), + )) + } + + /// Begin recording a turn against an explicit sink. + /// + /// The global sink is process-wide, which makes it unusable for anything + /// that needs its own destination: parallel tests, and the experiment + /// runner, which routes a dataset run's traces separately from live + /// traffic. Both would otherwise race with whatever was installed last. + #[must_use] + pub fn begin_with_sink( + sink: Arc, + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + ) -> Option { + settings + .sampled() + .then(|| Self::begin_inner(sink, name, scope, settings, None)) + } + + fn begin_inner( + sink: Arc, + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + active_turn: Option, + ) -> Self { + let mut trace = TraceRecord::new(name); + trace.scope = scope.clone(); + let trace_id = trace.id.clone(); + + // The root observation shares the trace id so exporters can parent + // orphan steps onto it without extra bookkeeping. + let root_id = ObservationId(trace_id.0.clone()); + + Self { + sink, + settings, + trace_id: trace_id.clone(), + scope, + root_id, + trace: Arc::new(Mutex::new(trace)), + _active_turn: active_turn, + } + } + + /// The trace being recorded, for correlating scores later. + #[must_use] + pub fn trace_id(&self) -> &TraceId { + &self.trace_id + } + + /// Attach the turn's input. + pub fn set_input(&self, input: serde_json::Value) { + if !self.settings.capture_input { + return; + } + let redacted = self.settings.redaction.redact(&input); + self.with_trace(|trace| trace.input = Some(redacted)); + } + + /// Attach the turn's output. + pub fn set_output(&self, output: serde_json::Value) { + if !self.settings.capture_output { + return; + } + let redacted = self.settings.redaction.redact(&output); + self.with_trace(|trace| trace.output = Some(redacted)); + } + + /// Attach a metadata entry to the trace. + pub fn set_metadata(&self, key: impl Into, value: serde_json::Value) { + let key = key.into(); + let value = self.redact_metadata_value(&key, value); + self.with_trace(|trace| { + trace.metadata.insert(key, value); + }); + } + + /// Open a step nested under the run's root. + #[must_use] + pub fn step(&self, kind: ObservationKind, name: impl Into) -> StepGuard { + self.step_under(kind, name, Some(self.root_id.clone())) + } + + /// Open a step nested under an explicit parent, for sub-agents and tools + /// invoked from within another step. + #[must_use] + pub fn step_under( + &self, + kind: ObservationKind, + name: impl Into, + parent: Option, + ) -> StepGuard { + let record = ObservationRecord::start(self.trace_id.clone(), kind, name) + .with_parent(parent.or_else(|| Some(self.root_id.clone()))) + .with_scope(self.scope.clone()); + + StepGuard { + sink: Arc::clone(&self.sink), + settings: self.settings.clone(), + trace: Arc::clone(&self.trace), + record: Some(record), + } + } + + /// Record a score against this turn. + pub fn score(&self, name: impl Into, value: ScoreValue, comment: Option) { + let mut score = ScoreRecord::new(self.trace_id.clone(), name, value); + score.comment = comment.map(|value| self.settings.redaction.redact_str(&value)); + score.environment = self.scope.environment.clone(); + self.sink.record(Event::Score(Box::new(score))); + } + + /// Close the turn, emitting the final trace state. + /// + /// Takes `&self` rather than `self` so the recorder can be shared across + /// the concurrently-executing tool futures in the agent loop. + pub fn finish(&self) { + let trace = { + let mut guard = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if guard.end_time.is_some() { + return; + } + guard.end_time = Some(OffsetDateTime::now_utc()); + guard.clone() + }; + self.sink.record(Event::Trace(Box::new(trace))); + } + + /// Close the turn as failed. + pub fn finish_with_error(&self, message: impl Into) { + self.set_metadata("error", serde_json::Value::String(message.into())); + self.finish(); + } + + fn with_trace(&self, f: impl FnOnce(&mut TraceRecord)) { + let mut guard = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&mut guard); + } + + fn redact_metadata_value(&self, key: &str, value: serde_json::Value) -> serde_json::Value { + if self.settings.redaction.is_sensitive_key(key) { + serde_json::Value::String(REDACTED.to_string()) + } else { + self.settings.redaction.redact(&value) + } + } +} + +/// An open observation. Emits its completion on drop, so an early `?` return +/// still produces a closed span rather than one that hangs open forever. +pub struct StepGuard { + sink: Arc, + settings: RecorderSettings, + trace: Arc>, + /// `None` once the guard has emitted, so drop does not emit twice. + record: Option, +} + +impl StepGuard { + /// This step's id, for nesting children beneath it. + #[must_use] + pub fn id(&self) -> Option { + self.record.as_ref().map(|r| r.id.clone()) + } + + /// Attach the step's input. + pub fn set_input(&mut self, input: serde_json::Value) { + if !self.capture_input_allowed() { + return; + } + let redacted = self.settings.redaction.redact(&input); + if let Some(record) = self.record.as_mut() { + record.input = Some(redacted); + } + } + + /// Attach the step's output. + pub fn set_output(&mut self, output: serde_json::Value) { + if !self.capture_output_allowed() { + return; + } + let redacted = self.settings.redaction.redact(&output); + if let Some(record) = self.record.as_mut() { + record.output = Some(redacted); + } + } + + /// Name the model this step used. + pub fn set_model(&mut self, model: impl Into) { + if let Some(record) = self.record.as_mut() { + record.model = Some(model.into()); + } + } + + /// Set a sampling parameter. + pub fn set_model_parameter(&mut self, key: impl Into, value: serde_json::Value) { + if let Some(record) = self.record.as_mut() { + record.model_parameters.insert(key.into(), value); + } + } + + /// Attach token usage. + /// + /// Cost is deliberately not derived from it: Langfuse maintains versioned + /// model prices and infers spend from the model plus these counts, and no + /// other backend has a price table at all. + pub fn set_usage(&mut self, usage: TokenUsage) { + if let Some(record) = self.record.as_mut() { + record.usage = Some(usage); + } + } + + /// Mark the instant the first output token arrived. + /// + /// Ignored if called twice: only the first token defines time-to-first-token. + pub fn mark_first_token(&mut self) { + if let Some(record) = self.record.as_mut() + && record.completion_start_time.is_none() + { + record.completion_start_time = Some(OffsetDateTime::now_utc()); + } + } + + /// Link this generation to a managed prompt version. + pub fn set_prompt(&mut self, name: impl Into, version: i32) { + if let Some(record) = self.record.as_mut() { + record.prompt_name = Some(name.into()); + record.prompt_version = Some(version); + } + } + + /// Attach a metadata entry. + pub fn set_metadata(&mut self, key: impl Into, value: serde_json::Value) { + let key = key.into(); + let value = if self.settings.redaction.is_sensitive_key(&key) { + serde_json::Value::String(REDACTED.to_string()) + } else { + self.settings.redaction.redact(&value) + }; + if let Some(record) = self.record.as_mut() { + record.metadata.insert(key, value); + } + } + + /// Attach output-derived metadata while honoring the output capture switch. + pub fn set_output_metadata(&mut self, key: impl Into, value: serde_json::Value) { + if self.capture_output_allowed() { + self.set_metadata(key, value); + } + } + + /// Raise the severity of this step. + pub fn set_level(&mut self, level: Level, message: Option) { + if let Some(record) = self.record.as_mut() { + record.level = level; + record.status_message = message.map(|value| self.settings.redaction.redact_str(&value)); + } + } + + /// Mark the step failed. The span is still emitted on drop. + pub fn fail(&mut self, message: impl Into) { + if let Some(record) = self.record.as_mut() { + record.fail(self.settings.redaction.redact_str(&message.into())); + } + } + + /// Emit the completed observation now rather than at drop. + pub fn finish(mut self) { + self.emit(); + } + + fn emit(&mut self) { + let Some(mut record) = self.record.take() else { + return; + }; + { + let trace = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + record.trace_name = Some(trace.name.clone()); + record.trace_metadata = trace.metadata.clone(); + } + record.finish(); + self.sink.record(Event::ObservationEnd(Box::new(record))); + } + + /// Tool arguments are gated by their own switch as well as the input + /// switch, since they are the most likely place for credentials to appear. + fn capture_input_allowed(&self) -> bool { + match self.record.as_ref().map(|r| r.kind) { + Some(ObservationKind::Tool | ObservationKind::Retriever) => { + self.settings.capture_tool_io + }, + _ => self.settings.capture_input, + } + } + + fn capture_output_allowed(&self) -> bool { + match self.record.as_ref().map(|r| r.kind) { + Some(ObservationKind::Tool | ObservationKind::Retriever) => { + self.settings.capture_tool_io + }, + _ => self.settings.capture_output, + } + } +} + +impl Drop for StepGuard { + fn drop(&mut self) { + if let Some(record) = self.record.as_mut() { + record.fail("observation cancelled before completion"); + } + self.emit(); + } +} + +impl Drop for TurnRecorder { + fn drop(&mut self) { + self.finish_with_error("agent run cancelled before completion"); + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use std::sync::Mutex as StdMutex; + + use {async_trait::async_trait, serde_json::json}; + + use super::*; + + struct CollectingSink { + events: StdMutex>, + } + + impl CollectingSink { + fn new() -> Arc { + Arc::new(Self { + events: StdMutex::new(Vec::new()), + }) + } + + fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + fn observations(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|e| match e { + Event::ObservationEnd(o) => Some(*o), + _ => None, + }) + .collect() + } + + fn traces(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|e| match e { + Event::Trace(t) => Some(*t), + _ => None, + }) + .collect() + } + } + + #[async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + "collecting" + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: std::time::Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + fn with_sink(f: impl FnOnce(Arc) -> R) -> R { + let _guard = sink::GLOBAL_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let collected = CollectingSink::new(); + sink::set_global_sink(collected.clone()); + let out = f(collected); + sink::clear_global_sink(); + out + } + + fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:main".into()), + user_id: Some("telegram:42".into()), + tags: vec!["telegram".into()], + environment: Some("production".into()), + release: None, + version: None, + } + } + + #[test] + fn returns_none_when_no_sink_is_installed() { + let _guard = sink::GLOBAL_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sink::clear_global_sink(); + + // Instrumentation must cost nothing when switched off. + assert!(TurnRecorder::begin("turn", scope(), RecorderSettings::default()).is_none()); + } + + #[test] + fn zero_sample_rate_skips_the_turn_entirely() { + with_sink(|collected| { + let settings = RecorderSettings { + sample_rate: 0.0, + ..Default::default() + }; + assert!(TurnRecorder::begin("turn", scope(), settings).is_none()); + assert!(collected.events().is_empty()); + }); + } + + #[test] + fn steps_are_parented_to_the_run_root() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let root = ObservationId(recorder.trace_id().0.clone()); + + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + + let observations = collected.observations(); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].parent_id, Some(root)); + }); + } + + #[test] + fn nested_steps_are_parented_to_their_enclosing_step() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let outer = recorder.step(ObservationKind::Agent, "sub-agent"); + let outer_id = outer.id().expect("open step has an id"); + recorder + .step_under(ObservationKind::Tool, "exec", Some(outer_id.clone())) + .finish(); + outer.finish(); + recorder.finish(); + + let tool = collected + .observations() + .into_iter() + .find(|o| o.kind == ObservationKind::Tool) + .expect("tool observation emitted"); + assert_eq!(tool.parent_id, Some(outer_id)); + }); + } + + #[test] + fn steps_carry_the_trace_scope() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert_eq!( + observed.scope.session_id.as_deref(), + Some("agent:main:main") + ); + assert_eq!(observed.scope.user_id.as_deref(), Some("telegram:42")); + }); + } + + #[test] + fn no_observation_event_is_emitted_before_the_step_completes() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let step = recorder.step(ObservationKind::Generation, "llm"); + + assert!(collected.events().is_empty()); + + step.finish(); + recorder.finish(); + }); + } + + #[test] + fn dropping_a_step_still_closes_it() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + // Simulates an early `?` return out of the agent loop. + drop(recorder.step(ObservationKind::Tool, "exec")); + recorder.finish(); + + let observations = collected.observations(); + assert_eq!(observations.len(), 1, "dropped step must still be emitted"); + assert!(observations[0].end_time.is_some()); + assert_eq!(observations[0].level, Level::Error); + assert_eq!( + observations[0].status_message.as_deref(), + Some("observation cancelled before completion") + ); + }); + } + + #[test] + fn dropping_an_unfinished_turn_emits_an_error_root() { + let collected = CollectingSink::new(); + let recorder = TurnRecorder::begin_with_sink( + collected.clone(), + "turn", + scope(), + RecorderSettings::default(), + ) + .expect("sampled"); + + drop(recorder); + + let traces = collected.traces(); + assert_eq!(traces.len(), 1); + assert_eq!( + traces[0].metadata["error"], + json!("agent run cancelled before completion") + ); + assert!(traces[0].end_time.is_some()); + } + + #[test] + fn finishing_a_step_does_not_emit_it_twice() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.step(ObservationKind::Tool, "exec").finish(); + recorder.finish(); + + assert_eq!(collected.observations().len(), 1); + }); + } + + #[test] + fn payloads_are_redacted_before_reaching_the_sink() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_input(json!({ "api_key": "abc123", "cmd": "ls" })); + step.finish(); + recorder.set_input(json!({ "password": "hunter2" })); + recorder.finish(); + + let tool_input = collected.observations()[0] + .input + .clone() + .expect("input captured"); + assert_eq!(tool_input["api_key"], json!("[REDACTED]")); + assert_eq!(tool_input["cmd"], json!("ls")); + + let trace = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!( + trace.input.expect("trace input")["password"], + json!("[REDACTED]") + ); + }); + } + + #[test] + fn capture_switches_suppress_payloads() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_input: false, + capture_output: false, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_input(json!({ "cmd": "ls" })); + step.set_output(json!("file list")); + step.finish(); + recorder.set_input(json!("hello")); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert!(observed.input.is_none()); + assert!(observed.output.is_none()); + assert!(collected.traces().last().expect("trace").input.is_none()); + }); + } + + #[test] + fn output_capture_switch_suppresses_output_derived_metadata() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_output: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.set_output_metadata("reasoning", json!("private reasoning")); + step.finish(); + recorder.finish(); + + assert!( + !collected.observations()[0] + .metadata + .contains_key("reasoning") + ); + }); + } + + #[test] + fn tool_io_switch_is_independent_of_the_generation_switches() { + with_sink(|collected| { + // Tool arguments are the likeliest place for credentials, so they + // must be suppressible without losing LLM inputs. + let settings = RecorderSettings { + capture_input: true, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + + let mut tool = recorder.step(ObservationKind::Tool, "exec"); + tool.set_input(json!({ "cmd": "ls" })); + tool.finish(); + + let mut generation = recorder.step(ObservationKind::Generation, "llm"); + generation.set_input(json!("hello")); + generation.finish(); + recorder.finish(); + + let observations = collected.observations(); + let tool = observations + .iter() + .find(|o| o.kind == ObservationKind::Tool) + .expect("tool emitted"); + let generation = observations + .iter() + .find(|o| o.kind == ObservationKind::Generation) + .expect("generation emitted"); + + assert!(tool.input.is_none()); + assert!(generation.input.is_some()); + }); + } + + #[test] + fn retriever_io_uses_the_tool_capture_switch() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_input: true, + capture_output: true, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + let mut retriever = recorder.step(ObservationKind::Retriever, "memory-search"); + retriever.set_input(json!({ "query": "secret" })); + retriever.set_output(json!({ "matches": ["private"] })); + retriever.finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert!(observed.input.is_none()); + assert!(observed.output.is_none()); + }); + } + + #[test] + fn first_token_marker_records_only_the_first_call() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.mark_first_token(); + let first = step + .record + .as_ref() + .and_then(|r| r.completion_start_time) + .expect("first token recorded"); + step.mark_first_token(); + let second = step + .record + .as_ref() + .and_then(|r| r.completion_start_time) + .expect("still recorded"); + step.finish(); + recorder.finish(); + + assert_eq!(first, second, "time-to-first-token must not drift"); + assert!(collected.observations()[0].completion_start_time.is_some()); + }); + } + + #[test] + fn failed_steps_carry_error_level_and_message() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.fail("provider returned 500"); + step.finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert_eq!(observed.level, Level::Error); + assert_eq!( + observed.status_message.as_deref(), + Some("provider returned 500") + ); + }); + } + + #[test] + fn metadata_status_and_score_comments_are_redacted() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_metadata("api_key", json!("raw-trace-secret")); + recorder.set_metadata("context", json!({ "password": "raw-password" })); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_metadata("access_token", json!("raw-step-secret")); + step.set_level( + Level::Warning, + Some("Bearer abcdefghijklmnopqrstuvwxyz".into()), + ); + step.finish(); + recorder.score( + "review", + ScoreValue::Boolean(true), + Some("sk-live-abcdefghijkl".into()), + ); + recorder.finish(); + + let rendered = serde_json::to_string(&collected.events()).expect("serializable"); + assert!(!rendered.contains("raw-trace-secret")); + assert!(!rendered.contains("raw-password")); + assert!(!rendered.contains("raw-step-secret")); + assert!(!rendered.contains("Bearer abcdefghijklmnopqrstuvwxyz")); + assert!(!rendered.contains("sk-live-abcdefghijkl")); + assert!(rendered.contains(REDACTED)); + }); + } + + #[test] + fn model_and_usage_are_recorded_in_either_order() { + // Callers set whichever they learn first: a streaming provider names + // the model up front and reports usage at the end, a non-streaming one + // returns both together. + with_sink(|collected| { + let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) + .expect("recorder starts"); + + let mut model_first = recorder.step(ObservationKind::Generation, "model-first"); + model_first.set_model("claude-opus-4"); + model_first.set_usage(TokenUsage::from_provider_totals(1_000, 500, 0, 0)); + model_first.finish(); + + let mut usage_first = recorder.step(ObservationKind::Generation, "usage-first"); + usage_first.set_usage(TokenUsage::from_provider_totals(1_000, 500, 0, 0)); + usage_first.set_model("claude-opus-4"); + usage_first.finish(); + + let observations = collected.observations(); + assert_eq!(observations.len(), 2); + for record in observations { + assert_eq!(record.model.as_deref(), Some("claude-opus-4")); + assert_eq!(record.usage.map(|u| u.input), Some(1_000)); + assert_eq!(record.usage.map(|u| u.output), Some(500)); + } + }); + } + + #[test] + fn scores_are_emitted_against_the_turn() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let trace_id = recorder.trace_id().clone(); + + recorder.score( + "user-feedback", + ScoreValue::Numeric(1.0), + Some("helpful".into()), + ); + recorder.finish(); + + let score = collected + .events() + .into_iter() + .find_map(|e| match e { + Event::Score(s) => Some(*s), + _ => None, + }) + .expect("score emitted"); + + assert_eq!(score.trace_id, trace_id); + assert_eq!(score.environment.as_deref(), Some("production")); + }); + } + + #[test] + fn the_closing_trace_carries_output_set_during_the_turn() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_output(json!("final answer")); + recorder.finish(); + + let last = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!(last.output, Some(json!("final answer"))); + }); + } + + #[test] + fn trace_and_observation_are_emitted_once_after_completion() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_metadata("tenant", json!("acme")); + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + recorder.finish(); + + let traces = collected.traces(); + assert_eq!(traces.len(), 1); + assert!(traces[0].end_time.is_some()); + let observations = collected.observations(); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].trace_name.as_deref(), Some("turn")); + assert_eq!( + observations[0].trace_metadata.get("tenant"), + Some(&json!("acme")) + ); + }); + } + + #[test] + fn finish_with_error_records_the_failure_on_the_trace() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.finish_with_error("context window exceeded"); + + let last = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!( + last.metadata.get("error"), + Some(&json!("context window exceeded")) + ); + }); + } +} diff --git a/crates/observability/src/redact.rs b/crates/observability/src/redact.rs new file mode 100644 index 0000000000..76497c89c2 --- /dev/null +++ b/crates/observability/src/redact.rs @@ -0,0 +1,340 @@ +//! Redaction of sensitive values before export. +//! +//! Instrumentation ships conversation content, tool arguments and tool results +//! to a third party, so redaction is a security control, not a nicety. Two +//! independent passes run over every payload: +//! +//! 1. **Key-based**: any object key containing a configured needle (`password`, +//! `token`, ...) has its value replaced wholesale. +//! 2. **Shape-based**: string values that look like credentials (provider key +//! prefixes, bearer tokens, PEM private keys) are replaced even when the key +//! name is innocuous, because tool results are unstructured. +//! +//! Both passes are applied to keys and values at every depth. Redaction failing +//! open would be a silent data leak, so unknown structures are traversed rather +//! than skipped. + +use serde_json::Value; + +/// Replacement text substituted for redacted values. +pub const REDACTED: &str = "[REDACTED]"; + +/// Marker appended to truncated strings. +const TRUNCATION_SUFFIX: &str = "…[truncated]"; + +/// Default key needles. Matched case-insensitively as substrings. +pub const DEFAULT_KEY_NEEDLES: &[&str] = &[ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "credential", + "private_key", + "session_key", + "access_key", +]; + +/// Credential-looking string prefixes, matched case-sensitively because real +/// key formats are case-sensitive and lowering would cause false positives. +const SECRET_PREFIXES: &[&str] = &[ + "sk-", + "sk_live_", + "sk_test_", + "pk_live_", + "rk_live_", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "github_pat_", + "xoxb-", + "xoxp-", + "xoxa-", + "xapp-", + "AIza", + "ya29.", + "AKIA", + "ASIA", + "hf_", + "lf_", + "Bearer ", + "bearer ", + "-----BEGIN", +]; + +/// Policy controlling what leaves the process. +#[derive(Debug, Clone)] +pub struct RedactionPolicy { + /// Object keys whose values are always replaced. + key_needles: Vec, + /// Whether to also scan string values for credential shapes. + scan_values: bool, + /// Maximum length of any single string before truncation. `0` disables. + max_string_len: usize, +} + +impl Default for RedactionPolicy { + fn default() -> Self { + Self { + key_needles: DEFAULT_KEY_NEEDLES + .iter() + .map(|s| (*s).to_string()) + .collect(), + scan_values: true, + max_string_len: 32_768, + } + } +} + +impl RedactionPolicy { + /// Build a policy from configured needles, merging in the defaults so an + /// operator cannot accidentally disable baseline protection by supplying a + /// narrower list. + #[must_use] + pub fn from_needles(needles: &[String]) -> Self { + let mut key_needles: Vec = DEFAULT_KEY_NEEDLES + .iter() + .map(|s| (*s).to_string()) + .collect(); + for needle in needles { + let lowered = needle.to_lowercase(); + if !lowered.is_empty() && !key_needles.contains(&lowered) { + key_needles.push(lowered); + } + } + Self { + key_needles, + ..Self::default() + } + } + + /// Set the per-string truncation limit. `0` disables truncation. + #[must_use] + pub const fn with_max_string_len(mut self, max: usize) -> Self { + self.max_string_len = max; + self + } + + /// Enable or disable credential-shape scanning of string values. + #[must_use] + pub const fn with_value_scanning(mut self, scan: bool) -> Self { + self.scan_values = scan; + self + } + + /// Whether `key` names something that must never be exported. + #[must_use] + pub fn is_sensitive_key(&self, key: &str) -> bool { + let lowered = key.to_lowercase(); + self.key_needles + .iter() + .any(|n| lowered.contains(n.as_str())) + } + + /// Whether `value` looks like a credential regardless of its key. + #[must_use] + pub fn looks_like_secret(&self, value: &str) -> bool { + if !self.scan_values { + return false; + } + let trimmed = value.trim(); + // Very short strings cannot be meaningful credentials and matching them + // produces false positives on ordinary prose. + if trimmed.len() < 8 { + return false; + } + SECRET_PREFIXES.iter().any(|prefix| { + trimmed.match_indices(prefix).any(|(index, _)| { + index == 0 + || trimmed[..index].chars().next_back().is_some_and(|c| { + c.is_whitespace() || matches!(c, '=' | ':' | '\'' | '"' | '`') + }) + }) + }) + } + + /// Redact a JSON value in place-equivalent fashion, returning a clean copy. + #[must_use] + pub fn redact(&self, value: &Value) -> Value { + match value { + Value::Object(map) => { + let cleaned = map + .iter() + .map(|(key, val)| { + if self.is_sensitive_key(key) { + (key.clone(), Value::String(REDACTED.to_string())) + } else { + (key.clone(), self.redact(val)) + } + }) + .collect(); + Value::Object(cleaned) + }, + Value::Array(items) => { + Value::Array(items.iter().map(|item| self.redact(item)).collect()) + }, + Value::String(text) => Value::String(self.redact_str(text)), + other => other.clone(), + } + } + + /// Redact and truncate a single string. + #[must_use] + pub fn redact_str(&self, text: &str) -> String { + if self.looks_like_secret(text) { + return REDACTED.to_string(); + } + self.truncate(text) + } + + /// Truncate `text` to the configured limit on a character boundary. + fn truncate(&self, text: &str) -> String { + if self.max_string_len == 0 || text.len() <= self.max_string_len { + return text.to_string(); + } + // `floor_char_boundary` is unstable, so walk back to a boundary by hand + // rather than slicing blindly and panicking on multi-byte input. + let mut end = self.max_string_len; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + format!("{}{TRUNCATION_SUFFIX}", &text[..end]) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use {super::*, serde_json::json}; + + #[test] + fn redacts_sensitive_keys_at_any_depth() { + let policy = RedactionPolicy::default(); + let input = json!({ + "safe": "keep me", + "nested": { + "api_key": "abc123", + "deeper": [{ "password": "hunter2" }] + } + }); + + let out = policy.redact(&input); + + assert_eq!(out["safe"], json!("keep me")); + assert_eq!(out["nested"]["api_key"], json!(REDACTED)); + assert_eq!(out["nested"]["deeper"][0]["password"], json!(REDACTED)); + } + + #[test] + fn key_matching_is_case_insensitive_and_substring_based() { + let policy = RedactionPolicy::default(); + let input = json!({ + "ANTHROPIC_API_KEY": "x", + "userToken": "y", + "Authorization": "z", + }); + + let out = policy.redact(&input); + + assert_eq!(out["ANTHROPIC_API_KEY"], json!(REDACTED)); + assert_eq!(out["userToken"], json!(REDACTED)); + assert_eq!(out["Authorization"], json!(REDACTED)); + } + + #[test] + fn redacts_credential_shaped_values_under_innocuous_keys() { + let policy = RedactionPolicy::default(); + // Tool results are unstructured: a leaked key often lands under a key + // name that no needle list would ever catch. + let input = json!({ + "stdout": "sk-ant-api03-abcdefghijklmnop", + "header": "Bearer eyJhbGciOiJIUzI1NiJ9", + "note": "the deploy finished successfully", + }); + + let out = policy.redact(&input); + + assert_eq!(out["stdout"], json!(REDACTED)); + assert_eq!(out["header"], json!(REDACTED)); + assert_eq!(out["note"], json!("the deploy finished successfully")); + } + + #[test] + fn redacts_credentials_embedded_in_shell_output_and_commands() { + let policy = RedactionPolicy::default(); + for value in [ + "OPENAI_API_KEY=sk-abcdefghijklmnop", + "curl -H 'Authorization: Bearer abcdefghijklmnop' https://example.com", + "request failed: xoxb-example", + ] { + assert_eq!(policy.redact(&json!(value)), json!(REDACTED), "{value}"); + } + assert!(!policy.looks_like_secret("ordinary task-sk-result text")); + } + + #[test] + fn redacts_pem_private_key_blocks() { + let policy = RedactionPolicy::default(); + let input = json!("-----BEGIN RSA PRIVATE KEY-----\nMIIEow==\n"); + assert_eq!(policy.redact(&input), json!(REDACTED)); + } + + #[test] + fn short_strings_are_not_treated_as_secrets() { + let policy = RedactionPolicy::default(); + // "sk-1" is too short to be a real key; matching it would mangle prose. + assert!(!policy.looks_like_secret("sk-1")); + assert!(policy.looks_like_secret("sk-ant-api03-abcdefgh")); + } + + #[test] + fn value_scanning_can_be_disabled() { + let policy = RedactionPolicy::default().with_value_scanning(false); + assert!(!policy.looks_like_secret("sk-ant-api03-abcdefghijkl")); + } + + #[test] + fn custom_needles_extend_rather_than_replace_defaults() { + let policy = RedactionPolicy::from_needles(&["internal_id".to_string()]); + + assert!(policy.is_sensitive_key("internal_id")); + // The baseline must survive a narrower operator-supplied list. + assert!(policy.is_sensitive_key("password")); + assert!(policy.is_sensitive_key("api_key")); + } + + #[test] + fn empty_needles_are_ignored() { + let policy = RedactionPolicy::from_needles(&[String::new()]); + // An empty needle is a substring of every key and would redact + // everything, so it must be dropped. + assert!(!policy.is_sensitive_key("harmless")); + } + + #[test] + fn truncates_long_strings_on_a_char_boundary() { + let policy = RedactionPolicy::default().with_max_string_len(8); + // Multi-byte characters straddling the limit must not panic. + let out = policy.redact_str("ααααααααααα"); + + assert!(out.ends_with(TRUNCATION_SUFFIX)); + assert!(out.len() < 40); + } + + #[test] + fn truncation_can_be_disabled() { + let policy = RedactionPolicy::default().with_max_string_len(0); + let long = "a".repeat(100_000); + assert_eq!(policy.redact_str(&long).len(), 100_000); + } + + #[test] + fn non_string_scalars_pass_through_unchanged() { + let policy = RedactionPolicy::default(); + let input = json!({ "n": 42, "b": true, "nil": null, "f": 1.5 }); + assert_eq!(policy.redact(&input), input); + } +} diff --git a/crates/observability/src/runtime.rs b/crates/observability/src/runtime.rs new file mode 100644 index 0000000000..9469a8bc9a --- /dev/null +++ b/crates/observability/src/runtime.rs @@ -0,0 +1,712 @@ +//! Batching export runtime shared by every backend. +//! +//! A backend supplies a [`Transport`]; this module supplies everything around +//! it — a bounded queue, size- and time-based batching, retry with exponential +//! backoff and jitter, and drop accounting. +//! +//! The central invariant: **the agent loop is never blocked by telemetry**. +//! [`BatchSink::record`] uses `try_send` and drops on a full queue. Losing +//! traces is an acceptable failure; stalling a user's turn is not. + +use std::{ + sync::{ + Arc, RwLock, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use { + async_trait::async_trait, + tokio::sync::{mpsc, oneshot}, + tracing::{debug, trace, warn}, +}; + +use crate::{model::Event, sink::ObservationSink}; + +/// Why a transport send failed. +#[derive(Debug, thiserror::Error)] +pub enum TransportError { + /// Transient: worth retrying (5xx, 429, connection reset). + #[error("retryable transport failure: {0}")] + Retryable(String), + /// Permanent: retrying cannot help (401, 400, malformed payload). + #[error("fatal transport failure: {0}")] + Fatal(String), +} + +/// Backend-specific delivery mechanism. +#[async_trait] +pub trait Transport: Send + Sync + 'static { + /// Transport name, used in logs and status reporting. + fn name(&self) -> &str; + + /// Deliver a batch. Called from the background task only. + async fn send(&self, batch: &[Event]) -> Result<(), TransportError>; + + /// Whether this transport can represent `event`. This runs on the recording + /// hot path and must be fast and non-blocking. + fn accepts(&self, _event: &Event) -> bool { + true + } + + /// Approximate serialized size of `event`, used for batch sizing. + fn estimate_bytes(&self, event: &Event) -> usize { + serde_json::to_vec(event).map(|v| v.len()).unwrap_or(1024) + } +} + +/// Batching and retry parameters. +#[derive(Debug, Clone)] +pub struct BatchConfig { + /// Maximum events held before a forced flush. + pub max_batch_events: usize, + /// Maximum estimated batch size in bytes before a forced flush. + pub max_batch_bytes: usize, + /// Maximum time an event waits before being flushed. + pub flush_interval: Duration, + /// Bounded queue depth. Events are dropped once this fills. + pub queue_capacity: usize, + /// Retry attempts for retryable failures. + pub max_retries: u32, + /// Base delay for exponential backoff. + pub initial_backoff: Duration, + /// Ceiling for exponential backoff. + pub max_backoff: Duration, +} + +impl Default for BatchConfig { + fn default() -> Self { + Self { + max_batch_events: 128, + // Comfortably under Langfuse's per-request ceiling, leaving room + // for the estimate to be wrong without the request being rejected. + max_batch_bytes: 3_000_000, + flush_interval: Duration::from_secs(5), + queue_capacity: 10_000, + max_retries: 3, + initial_backoff: Duration::from_millis(500), + max_backoff: Duration::from_secs(30), + } + } +} + +/// Counters describing sink health, surfaced in the settings UI. +#[derive(Debug, Default)] +pub struct SinkStats { + name: String, + /// Events accepted into the queue. + pub accepted: AtomicU64, + /// Events dropped because the queue was full. + pub dropped_queue_full: AtomicU64, + /// Events dropped after exhausting retries or hitting a fatal error. + pub dropped_failed: AtomicU64, + /// Batches delivered successfully. + pub batches_sent: AtomicU64, + /// Batches that failed permanently. + pub batches_failed: AtomicU64, + /// Events delivered successfully. + pub delivered: AtomicU64, + /// Retry attempts made after retryable failures. + pub retries: AtomicU64, + last_success_at: RwLock>, + last_error: RwLock>, + last_error_at: RwLock>, +} + +/// Point-in-time snapshot of [`SinkStats`]. +#[derive(Debug, Clone, serde::Serialize)] +pub struct SinkStatsSnapshot { + /// Sink whose counters are represented by this snapshot. + pub name: String, + /// Events accepted into the queue. + pub accepted: u64, + /// Events dropped because the queue was full. + pub dropped_queue_full: u64, + /// Events dropped after exhausting retries or hitting a fatal error. + pub dropped_failed: u64, + /// Batches delivered successfully. + pub batches_sent: u64, + /// Batches that failed permanently. + pub batches_failed: u64, + /// Events delivered successfully. + pub delivered: u64, + /// Retry attempts made after retryable failures. + pub retries: u64, + /// RFC 3339 timestamp of the latest successful delivery. + pub last_success_at: Option, + /// Most recent delivery error. + pub last_error: Option, + /// RFC 3339 timestamp of the most recent delivery error. + pub last_error_at: Option, +} + +impl SinkStats { + fn new(name: String) -> Self { + Self { + name, + ..Default::default() + } + } + + /// Read every counter. + #[must_use] + pub fn snapshot(&self) -> SinkStatsSnapshot { + SinkStatsSnapshot { + name: self.name.clone(), + accepted: self.accepted.load(Ordering::Relaxed), + dropped_queue_full: self.dropped_queue_full.load(Ordering::Relaxed), + dropped_failed: self.dropped_failed.load(Ordering::Relaxed), + batches_sent: self.batches_sent.load(Ordering::Relaxed), + batches_failed: self.batches_failed.load(Ordering::Relaxed), + delivered: self.delivered.load(Ordering::Relaxed), + retries: self.retries.load(Ordering::Relaxed), + last_success_at: read_state(&self.last_success_at), + last_error: read_state(&self.last_error), + last_error_at: read_state(&self.last_error_at), + } + } + + fn record_success(&self, count: u64) { + self.delivered.fetch_add(count, Ordering::Relaxed); + write_state(&self.last_success_at, now_rfc3339()); + } + + fn record_error(&self, reason: String) { + write_state(&self.last_error, Some(reason)); + write_state(&self.last_error_at, now_rfc3339()); + } +} + +fn read_state(state: &RwLock>) -> Option { + state + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() +} + +fn write_state(state: &RwLock>, value: Option) { + match state.write() { + Ok(mut guard) => *guard = value, + Err(poisoned) => *poisoned.into_inner() = value, + } +} + +fn now_rfc3339() -> Option { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .ok() +} + +/// Queue message: either an event or a flush barrier. +enum Message { + Event(Box), + Flush(oneshot::Sender<()>), +} + +/// An [`ObservationSink`] that batches into a [`Transport`]. +pub struct BatchSink { + name: String, + transport: Arc, + tx: mpsc::Sender, + stats: Arc, +} + +impl BatchSink { + /// Spawn the background export task and return a sink feeding it. + #[must_use] + pub fn spawn(transport: Arc, config: BatchConfig) -> Self { + let (tx, rx) = mpsc::channel(config.queue_capacity); + let name = transport.name().to_string(); + let stats = Arc::new(SinkStats::new(name.clone())); + + tokio::spawn(export_loop( + rx, + Arc::clone(&transport), + config, + Arc::clone(&stats), + )); + + Self { + name, + transport, + tx, + stats, + } + } + + /// Health counters for this sink. + #[must_use] + pub fn stats(&self) -> SinkStatsSnapshot { + self.stats.snapshot() + } +} + +#[async_trait] +impl ObservationSink for BatchSink { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + if !self.transport.accepts(&event) { + return; + } + + // `try_send` never awaits: a saturated queue drops the event rather + // than applying backpressure to the caller's turn. + match self.tx.try_send(Message::Event(Box::new(event))) { + Ok(()) => { + self.stats.accepted.fetch_add(1, Ordering::Relaxed); + }, + Err(mpsc::error::TrySendError::Full(_)) => { + let dropped = self + .stats + .dropped_queue_full + .fetch_add(1, Ordering::Relaxed) + + 1; + // Log sparsely: a saturated queue would otherwise generate more + // log volume than the telemetry it is failing to send. + if dropped.is_power_of_two() { + warn!( + sink = %self.name, + dropped, + "observability queue full; dropping events" + ); + } + }, + Err(mpsc::error::TrySendError::Closed(_)) => { + self.stats.dropped_failed.fetch_add(1, Ordering::Relaxed); + }, + } + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + "moltis_observability_events_total", + "sink" => self.name.clone() + ) + .increment(1); + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + let flush = async { + let (ack_tx, ack_rx) = oneshot::channel(); + if self.tx.send(Message::Flush(ack_tx)).await.is_err() { + return Err(anyhow::anyhow!("export task for {} has stopped", self.name)); + } + ack_rx.await.map_err(|_| { + anyhow::anyhow!("export task for {} dropped the flush barrier", self.name) + }) + }; + match tokio::time::timeout(timeout, flush).await { + Ok(result) => result, + Err(_) => Err(anyhow::anyhow!("flush of {} timed out", self.name)), + } + } + + fn delivery_stats(&self) -> Vec { + vec![self.stats()] + } +} + +/// Drain the queue, batching by count, size and time. +async fn export_loop( + mut rx: mpsc::Receiver, + transport: Arc, + config: BatchConfig, + stats: Arc, +) { + let mut batch: Vec = Vec::with_capacity(config.max_batch_events.min(1024)); + let mut batch_bytes = 0usize; + let mut ticker = tokio::time::interval(config.flush_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + tokio::select! { + message = rx.recv() => { + match message { + Some(Message::Event(event)) => { + batch_bytes += transport.estimate_bytes(&event); + batch.push(*event); + + if batch.len() >= config.max_batch_events + || batch_bytes >= config.max_batch_bytes + { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + } + }, + Some(Message::Flush(ack)) => { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + // The receiver may have timed out; that is not an error. + let _ = ack.send(()); + }, + None => { + // Sink dropped: make a final delivery attempt so a + // clean shutdown does not discard buffered spans. + deliver(&transport, &mut batch, &config, &stats).await; + return; + }, + } + }, + _ = ticker.tick() => { + if !batch.is_empty() { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + } + }, + } + } +} + +/// Send `batch` with retries, then clear it regardless of outcome. +async fn deliver( + transport: &Arc, + batch: &mut Vec, + config: &BatchConfig, + stats: &Arc, +) { + if batch.is_empty() { + return; + } + + let count = batch.len() as u64; + let mut attempt = 0u32; + + loop { + match transport.send(batch).await { + Ok(()) => { + stats.batches_sent.fetch_add(1, Ordering::Relaxed); + stats.record_success(count); + trace!(sink = transport.name(), events = count, "batch delivered"); + break; + }, + Err(TransportError::Fatal(reason)) => { + // Retrying a 401 or a malformed payload only burns quota. + warn!( + sink = transport.name(), + events = count, + %reason, + "dropping batch after fatal export failure" + ); + stats.batches_failed.fetch_add(1, Ordering::Relaxed); + stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + stats.record_error(reason); + break; + }, + Err(TransportError::Retryable(reason)) if attempt < config.max_retries => { + let delay = backoff_delay(config, attempt); + debug!( + sink = transport.name(), + attempt, + delay_ms = delay.as_millis(), + %reason, + "retrying observability export" + ); + stats.retries.fetch_add(1, Ordering::Relaxed); + stats.record_error(reason); + tokio::time::sleep(delay).await; + attempt += 1; + }, + Err(TransportError::Retryable(reason)) => { + warn!( + sink = transport.name(), + events = count, + attempts = attempt + 1, + %reason, + "dropping batch after exhausting export retries" + ); + stats.batches_failed.fetch_add(1, Ordering::Relaxed); + stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + stats.record_error(reason); + break; + }, + } + } + + batch.clear(); +} + +/// Exponential backoff with full jitter, clamped to `max_backoff`. +/// +/// Jitter matters here: without it, every Moltis instance that hits the same +/// rate limit retries in lockstep and re-creates the overload. +fn backoff_delay(config: &BatchConfig, attempt: u32) -> Duration { + let exponent = attempt.min(16); + let base = config + .initial_backoff + .saturating_mul(2u32.saturating_pow(exponent)); + let capped = base.min(config.max_backoff); + let millis = capped.as_millis().min(u128::from(u64::MAX)) as u64; + if millis == 0 { + return Duration::ZERO; + } + let jittered = rand::random_range(0..=millis); + Duration::from_millis(jittered) +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use std::sync::Mutex; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId}, + }; + + #[derive(Default)] + struct RecordingTransport { + batches: Mutex>, + /// Number of leading calls that fail retryably. + fail_retryable: AtomicU64, + /// Whether every call fails fatally. + fail_fatal: bool, + } + + #[async_trait] + impl Transport for RecordingTransport { + fn name(&self) -> &str { + "recording" + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + if self.fail_fatal { + return Err(TransportError::Fatal("unauthorized".into())); + } + if self.fail_retryable.load(Ordering::Relaxed) > 0 { + self.fail_retryable.fetch_sub(1, Ordering::Relaxed); + return Err(TransportError::Retryable("503".into())); + } + self.batches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(batch.len()); + Ok(()) + } + } + + impl RecordingTransport { + fn batches(&self) -> Vec { + self.batches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + } + + fn event() -> Event { + Event::ObservationEnd(Box::new(ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "llm-call", + ))) + } + + fn fast_config() -> BatchConfig { + BatchConfig { + flush_interval: Duration::from_millis(50), + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(5), + ..Default::default() + } + } + + #[tokio::test] + async fn flush_delivers_pending_events() { + let transport = Arc::new(RecordingTransport::default()); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches(), vec![2]); + assert_eq!(sink.stats().accepted, 2); + assert_eq!(sink.stats().delivered, 2); + assert!(sink.stats().last_success_at.is_some()); + } + + #[tokio::test] + async fn batch_is_forced_once_event_count_reached() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 3, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + for _ in 0..3 { + sink.record(event()); + } + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + // The size trigger must fire on its own, before the flush barrier. + assert_eq!(transport.batches().first(), Some(&3)); + } + + #[tokio::test] + async fn batch_is_forced_once_byte_budget_reached() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 10_000, + max_batch_bytes: 1, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches().first(), Some(&1)); + } + + #[tokio::test] + async fn retryable_failures_are_retried_then_succeed() { + let transport = Arc::new(RecordingTransport::default()); + transport.fail_retryable.store(2, Ordering::Relaxed); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches(), vec![1]); + assert_eq!(sink.stats().batches_sent, 1); + assert_eq!(sink.stats().dropped_failed, 0); + assert_eq!(sink.stats().delivered, 1); + assert_eq!(sink.stats().retries, 2); + assert!(sink.stats().last_error.is_some()); + assert!(sink.stats().last_success_at.is_some()); + } + + #[tokio::test] + async fn retries_are_bounded_and_batch_is_dropped() { + let transport = Arc::new(RecordingTransport::default()); + transport.fail_retryable.store(100, Ordering::Relaxed); + let config = BatchConfig { + max_retries: 2, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should still complete"); + + assert!(transport.batches().is_empty()); + assert_eq!(sink.stats().dropped_failed, 1); + assert_eq!(sink.stats().batches_failed, 1); + assert_eq!(sink.stats().retries, 2); + assert_eq!(sink.stats().delivered, 0); + assert!(sink.stats().last_error_at.is_some()); + } + + #[tokio::test] + async fn fatal_failures_are_not_retried() { + let transport = Arc::new(RecordingTransport { + fail_fatal: true, + ..Default::default() + }); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should still complete"); + + assert_eq!(sink.stats().batches_failed, 1); + assert_eq!(sink.stats().dropped_failed, 1); + assert_eq!(sink.stats().retries, 0); + assert!( + sink.stats() + .last_error + .is_some_and(|error| error.contains("unauthorized")) + ); + } + + #[tokio::test] + async fn full_queue_drops_events_without_blocking_caller() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + queue_capacity: 4, + max_batch_events: 100_000, + // Long interval so the drain never runs during the test. + flush_interval: Duration::from_secs(3600), + ..fast_config() + }; + let sink = BatchSink::spawn(transport, config); + + // Far more events than the queue can hold. `record` is synchronous, so + // if it ever blocked this test would hang rather than fail. + for _ in 0..500 { + sink.record(event()); + } + + let stats = sink.stats(); + assert!( + stats.dropped_queue_full > 0, + "expected drops on a full queue" + ); + assert!( + stats.accepted < 500, + "queue must not have absorbed every event" + ); + assert_eq!(stats.accepted + stats.dropped_queue_full, 500); + } + + #[tokio::test] + async fn time_based_flush_delivers_without_an_explicit_barrier() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 10_000, + flush_interval: Duration::from_millis(20), + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + // Deliberately no flush() call: the ticker must do the work. + for _ in 0..50 { + if !transport.batches().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + + assert_eq!(transport.batches(), vec![1]); + } + + #[test] + fn backoff_is_capped_and_jittered_within_bounds() { + let config = BatchConfig { + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_millis(1000), + ..Default::default() + }; + + for attempt in 0..8 { + let delay = backoff_delay(&config, attempt); + assert!( + delay <= config.max_backoff, + "attempt {attempt} exceeded the cap" + ); + } + } + + #[test] + fn backoff_does_not_overflow_on_large_attempt_counts() { + let config = BatchConfig::default(); + // A pathological attempt count must clamp, not panic on overflow. + assert!(backoff_delay(&config, u32::MAX) <= config.max_backoff); + } +} diff --git a/crates/observability/src/sink.rs b/crates/observability/src/sink.rs new file mode 100644 index 0000000000..0876b2abb3 --- /dev/null +++ b/crates/observability/src/sink.rs @@ -0,0 +1,343 @@ +//! The [`ObservationSink`] boundary plus the process-wide registry. +//! +//! `record` is deliberately synchronous and non-blocking: instrumentation sits +//! on the agent's hot path, so a slow or unreachable backend must never stall a +//! turn. Implementations enqueue and return; dropping data is always preferable +//! to applying backpressure to the agent loop. + +use std::{ + sync::{Arc, RwLock}, + time::Duration, +}; + +use async_trait::async_trait; + +use crate::{model::Event, runtime::SinkStatsSnapshot}; + +/// Destination for observability events. +#[async_trait] +pub trait ObservationSink: Send + Sync { + /// Sink name, used in logs and status reporting. + fn name(&self) -> &str; + + /// Enqueue an event. Must not block and must not fail loudly. + fn record(&self, event: Event); + + /// Flush pending events, giving up after `timeout`. + async fn flush(&self, timeout: Duration) -> anyhow::Result<()>; + + /// Point-in-time delivery health for this sink and any sinks below it. + fn delivery_stats(&self) -> Vec { + Vec::new() + } +} + +/// Fans one event out to several sinks, so Langfuse and an OTLP collector can +/// run side by side from a single instrumentation pass. +pub struct SinkFanout { + sinks: Vec>, + name: String, +} + +impl SinkFanout { + /// Build a fanout over `sinks`. + #[must_use] + pub fn new(sinks: Vec>) -> Self { + let name = sinks + .iter() + .map(|s| s.name().to_string()) + .collect::>() + .join("+"); + Self { sinks, name } + } + + /// Whether any sink is attached. + #[must_use] + pub fn is_empty(&self) -> bool { + self.sinks.is_empty() + } + + /// Number of attached sinks. + #[must_use] + pub fn len(&self) -> usize { + self.sinks.len() + } + + /// Names of the attached sinks. + #[must_use] + pub fn sink_names(&self) -> Vec { + self.sinks.iter().map(|s| s.name().to_string()).collect() + } +} + +#[async_trait] +impl ObservationSink for SinkFanout { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + // Clone per sink except the last, which takes ownership. + let mut iter = self.sinks.iter().peekable(); + while let Some(sink) = iter.next() { + if iter.peek().is_some() { + sink.record(event.clone()); + } else { + sink.record(event); + return; + } + } + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + // Flush concurrently: a slow backend must not serialise the others. + let futures = self.sinks.iter().map(|s| s.flush(timeout)); + let results = futures::future::join_all(futures).await; + + let errors: Vec = results + .into_iter() + .filter_map(|r| r.err().map(|e| e.to_string())) + .collect(); + + if errors.is_empty() { + return Ok(()); + } + Err(anyhow::anyhow!("sink flush failed: {}", errors.join("; "))) + } + + fn delivery_stats(&self) -> Vec { + self.sinks + .iter() + .flat_map(|sink| sink.delivery_stats()) + .collect() + } +} + +// ── Process-wide registry ─────────────────────────────────────────────────── + +/// The active sink. `RwLock` rather than `OnceLock` because the settings UI can +/// reconfigure instrumentation without a restart. +static GLOBAL_SINK: RwLock>> = RwLock::new(None); + +#[cfg(test)] +pub(crate) static GLOBAL_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Install `sink` as the process-wide destination, replacing any previous one. +pub fn set_global_sink(sink: Arc) { + match GLOBAL_SINK.write() { + Ok(mut guard) => *guard = Some(sink), + Err(poisoned) => *poisoned.into_inner() = Some(sink), + } +} + +/// Remove the process-wide sink, disabling instrumentation. +pub fn clear_global_sink() { + match GLOBAL_SINK.write() { + Ok(mut guard) => *guard = None, + Err(poisoned) => *poisoned.into_inner() = None, + } +} + +/// The active sink, if instrumentation is configured. +#[must_use] +pub fn global_sink() -> Option> { + match GLOBAL_SINK.read() { + Ok(guard) => guard.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + } +} + +/// Run a short operation while holding the global sink registry read lock. +/// +/// Used when cloning the sink must be atomic with related bookkeeping, such as +/// registering an active turn before shutdown can clear the registry. +pub(crate) fn with_global_sink( + operation: impl FnOnce(&Arc) -> T, +) -> Option { + match GLOBAL_SINK.read() { + Ok(guard) => guard.as_ref().map(operation), + Err(poisoned) => poisoned.into_inner().as_ref().map(operation), + } +} + +/// Whether instrumentation is active. +/// +/// Call this before doing any work to build an event: when no sink is +/// installed, instrumentation must cost nothing beyond this check. +#[must_use] +pub fn is_enabled() -> bool { + match GLOBAL_SINK.read() { + Ok(guard) => guard.is_some(), + Err(poisoned) => poisoned.into_inner().is_some(), + } +} + +/// Send `event` to the active sink, if any. +pub fn record(event: Event) { + if let Some(sink) = global_sink() { + sink.record(event); + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use std::sync::Mutex; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId, TraceRecord}, + }; + + struct CollectingSink { + name: String, + events: Mutex>, + } + + impl CollectingSink { + fn new(name: &str) -> Arc { + Arc::new(Self { + name: name.to_string(), + events: Mutex::new(Vec::new()), + }) + } + + fn count(&self) -> usize { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + } + } + + #[async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + struct FailingSink; + + #[async_trait] + impl ObservationSink for FailingSink { + fn name(&self) -> &str { + "failing" + } + + fn record(&self, _event: Event) {} + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Err(anyhow::anyhow!("backend unreachable")) + } + } + + fn sample_event() -> Event { + Event::ObservationEnd(Box::new(ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "llm-call", + ))) + } + + #[test] + fn fanout_delivers_to_every_sink() { + let a = CollectingSink::new("a"); + let b = CollectingSink::new("b"); + let c = CollectingSink::new("c"); + let fanout = SinkFanout::new(vec![a.clone(), b.clone(), c.clone()]); + + fanout.record(sample_event()); + + assert_eq!(a.count(), 1); + assert_eq!(b.count(), 1); + assert_eq!(c.count(), 1); + } + + #[test] + fn fanout_name_lists_every_backend() { + let fanout = SinkFanout::new(vec![ + CollectingSink::new("langfuse"), + CollectingSink::new("otlp"), + ]); + assert_eq!(fanout.name(), "langfuse+otlp"); + assert_eq!(fanout.len(), 2); + assert!(!fanout.is_empty()); + } + + #[test] + fn empty_fanout_accepts_events_without_panicking() { + let fanout = SinkFanout::new(Vec::new()); + fanout.record(sample_event()); + assert!(fanout.is_empty()); + } + + #[tokio::test] + async fn fanout_flush_reports_every_failure() { + let fanout = SinkFanout::new(vec![ + CollectingSink::new("ok"), + Arc::new(FailingSink), + Arc::new(FailingSink), + ]); + + let error = fanout + .flush(Duration::from_millis(10)) + .await + .expect_err("failing sinks must surface an error"); + let rendered = error.to_string(); + + // Both failures must be reported, not just the first. + assert_eq!(rendered.matches("backend unreachable").count(), 2); + } + + #[tokio::test] + async fn fanout_flush_succeeds_when_all_sinks_succeed() { + let fanout = SinkFanout::new(vec![CollectingSink::new("a"), CollectingSink::new("b")]); + assert!(fanout.flush(Duration::from_millis(10)).await.is_ok()); + } + + // The global registry is process-wide state, so these assertions live in a + // single test to stay independent of test execution order. + #[test] + fn global_registry_install_record_and_clear() { + let _guard = GLOBAL_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let sink = CollectingSink::new("global"); + + assert!(!is_enabled()); + record(sample_event()); + assert_eq!(sink.count(), 0, "events must drop while no sink installed"); + + set_global_sink(sink.clone()); + assert!(is_enabled()); + assert!(global_sink().is_some()); + + record(sample_event()); + assert_eq!(sink.count(), 1); + + clear_global_sink(); + assert!(!is_enabled()); + record(sample_event()); + assert_eq!(sink.count(), 1, "events must drop after clearing the sink"); + } + + #[test] + fn trace_event_reports_owning_trace() { + let trace = TraceRecord::new("turn"); + let id = trace.id.clone(); + let event = Event::Trace(Box::new(trace)); + assert_eq!(event.trace_id(), &id); + } +} diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs new file mode 100644 index 0000000000..b7d2f8d0f1 --- /dev/null +++ b/crates/observability/tests/end_to_end.rs @@ -0,0 +1,377 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] +// Every case here drives the OTLP transport, which a build without that feature +// does not have. The Langfuse cases use the Langfuse *profile* over the same +// transport, so `otlp` is the only feature they need. +#![cfg(feature = "otlp")] +//! End-to-end tests over the real export path. +//! +//! The unit tests check the mapping in isolation; these drive a recorder +//! through a live `BatchSink` into an HTTP server and assert on the bytes that +//! actually leave the process. The content-separation guarantee is only worth +//! anything if it holds on the wire, not just in a mapping function. + +use std::{sync::Arc, time::Duration}; + +use { + moltis_observability::{ + BatchConfig, BatchSink, ExportProfile, ObservationKind, ObservationSink, RecorderSettings, + TokenUsage, TraceScope, TurnRecorder, + exporters::otlp::{OtlpConfig, OtlpTransport}, + }, + serde_json::Value, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }, +}; + +/// The prompt text used throughout, so leak assertions are unambiguous. +const SECRET_PROMPT: &str = "please deploy the production database migration"; +const SECRET_ANSWER: &str = "migration applied to prod-db-01"; + +fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:channel:telegram:account:private-bot:peer:user:99".into()), + user_id: Some("telegram:99".into()), + tags: vec!["channel:telegram".into()], + environment: Some("production".into()), + release: Some("20260726.01".into()), + version: None, + } +} + +/// Spin up a mock collector and return it plus the captured request bodies. +async fn collector() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/traces")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + server +} + +/// Every OTLP body the mock server received, parsed. +async fn captured_bodies(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter_map(|req| serde_json::from_slice::(&req.body).ok()) + .collect() +} + +/// Install a sink pointed at `server` with `profile`, run one scripted agent +/// turn through the recorder, then flush and return the raw request bodies. +async fn run_turn_against(server: &MockServer, profile: ExportProfile) -> Vec { + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "test".into(), + endpoint: format!("{}/v1/traces", server.uri()), + headers: Default::default(), + timeout: Duration::from_secs(5), + service_name: "moltis".into(), + service_version: "20260726.01".into(), + environment: Some("production".into()), + profile, + })); + let batch_sink = Arc::new(BatchSink::spawn(transport, BatchConfig { + flush_interval: Duration::from_millis(20), + ..Default::default() + })); + + { + // Deliberately not the global sink: these tests run in parallel in one + // process, and a shared global means one test's spans land in another + // test's collector. + let recorder = TurnRecorder::begin_with_sink( + batch_sink.clone(), + "agent-run", + scope(), + RecorderSettings::default(), + ) + .expect("recorder should start"); + recorder.set_input(Value::String(SECRET_PROMPT.into())); + recorder.set_metadata("tenant", serde_json::json!("acme")); + + let mut generation = recorder.step(ObservationKind::Generation, "anthropic/claude-opus-4"); + generation.set_model("claude-opus-4"); + generation.set_input(serde_json::json!([{ "role": "user", "content": SECRET_PROMPT }])); + generation.mark_first_token(); + generation.set_usage(TokenUsage::from_provider_totals(120, 40, 800, 15)); + generation.set_output(Value::String(SECRET_ANSWER.into())); + let generation_id = generation.id(); + generation.finish(); + + let mut tool = recorder.step_under(ObservationKind::Tool, "exec", generation_id); + tool.set_input(serde_json::json!({ "cmd": "psql", "api_key": "sk-live-abcdefghijkl" })); + tool.set_output(Value::String(SECRET_ANSWER.into())); + tool.finish(); + + recorder.set_output(Value::String(SECRET_ANSWER.into())); + recorder.finish(); + } + + batch_sink + .flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + captured_bodies(server).await +} + +/// Every span across every captured request. +fn all_spans(bodies: &[Value]) -> Vec<&Value> { + bodies + .iter() + .filter_map(|b| b["resourceSpans"].as_array()) + .flatten() + .filter_map(|rs| rs["scopeSpans"].as_array()) + .flatten() + .filter_map(|ss| ss["spans"].as_array()) + .flatten() + .collect() +} + +fn span_attr<'a>(span: &'a Value, key: &str) -> Option<&'a Value> { + span["attributes"] + .as_array()? + .iter() + .find(|kv| kv["key"] == key) + .map(|kv| &kv["value"]) +} + +#[tokio::test] +async fn langfuse_profile_delivers_the_full_conversation() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + + assert!(!bodies.is_empty(), "expected at least one export request"); + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // Langfuse is useless without the conversation, so it must be present. + assert!( + raw.contains(SECRET_PROMPT), + "prompt missing from Langfuse export" + ); + assert!( + raw.contains(SECRET_ANSWER), + "completion missing from Langfuse export" + ); + + // Immutable OTLP receives only the completed generation. + let spans = all_spans(&bodies); + let generation = spans + .iter() + .find(|s| span_attr(s, "langfuse.observation.model.name").is_some()) + .expect("completed generation span present"); + + assert_eq!( + span_attr(generation, "langfuse.observation.model.name"), + Some(&serde_json::json!({ "stringValue": "claude-opus-4" })) + ); + assert!(span_attr(generation, "langfuse.observation.completion_start_time").is_some()); + + let root = spans + .iter() + .find(|span| span.get("parentSpanId").is_none()) + .expect("root span present"); + assert_eq!( + span_attr(root, "langfuse.observation.type"), + Some(&serde_json::json!({ "stringValue": "agent" })) + ); + assert_ne!(root["startTimeUnixNano"], root["endTimeUnixNano"]); + assert!(span_attr(root, "langfuse.observation.input").is_some()); + assert!(span_attr(root, "langfuse.observation.output").is_some()); +} + +#[tokio::test] +async fn generic_otlp_profile_never_puts_conversation_on_the_wire() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::otel_generic()).await; + + assert!(!bodies.is_empty(), "expected at least one export request"); + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // The guarantee that makes it safe to point this at a shared APM. + assert!( + !raw.contains(SECRET_PROMPT), + "prompt text reached a generic OTLP backend" + ); + assert!( + !raw.contains(SECRET_ANSWER), + "completion text reached a generic OTLP backend" + ); + assert!( + !raw.contains("langfuse."), + "langfuse-specific attributes reached a generic OTLP backend" + ); + // Per-user cardinality is a billing and compliance problem in an APM. + assert!(!raw.contains("telegram:99"), "user id reached an APM"); + assert!( + !raw.contains("private-bot"), + "channel account id reached an APM" + ); + assert!( + !raw.contains("gen_ai.conversation.id"), + "channel session key reached an APM" + ); +} + +#[tokio::test] +async fn generic_otlp_profile_still_delivers_operational_signal() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::otel_generic()).await; + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // Withholding content must not mean withholding everything: latency, + // model and token counts are the whole point of the APM export. + assert!(raw.contains("claude-opus-4"), "model missing"); + assert!( + raw.contains("gen_ai.usage.input_tokens"), + "token counts missing" + ); + assert!(raw.contains("moltis.input.bytes"), "payload sizes missing"); + assert!(!raw.contains("gen_ai.conversation.id")); +} + +#[tokio::test] +async fn tool_credentials_are_redacted_even_for_langfuse() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // The tool argument carried both a benign key and an API key. Redaction + // runs before serialization, so the credential must never reach the wire + // even on the backend that receives everything else. + assert!( + !raw.contains("sk-live-abcdefghijkl"), + "tool credential leaked into the export" + ); + assert!(raw.contains("REDACTED"), "expected a redaction marker"); + assert!(raw.contains("psql"), "benign tool argument should survive"); +} + +#[tokio::test] +async fn the_span_tree_is_correctly_nested() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let spans = all_spans(&bodies); + + let trace_ids: std::collections::HashSet<&str> = + spans.iter().filter_map(|s| s["traceId"].as_str()).collect(); + assert_eq!( + trace_ids.len(), + 1, + "one turn must produce exactly one trace" + ); + + let root = spans + .iter() + .find(|s| s.get("parentSpanId").is_none()) + .expect("a root span must exist"); + let root_id = root["spanId"].as_str().expect("root has an id"); + + // The generation hangs off the root, and the tool hangs off the + // generation. A flat tree would lose which call issued which tool. + let generation = spans + .iter() + .find(|s| s["name"] == "anthropic/claude-opus-4") + .expect("generation span present"); + assert_eq!(generation["parentSpanId"].as_str(), Some(root_id)); + + let tool = spans + .iter() + .find(|s| s["name"] == "exec") + .expect("tool span present"); + assert_eq!( + tool["parentSpanId"].as_str(), + generation["spanId"].as_str(), + "tool must nest under the generation that called it" + ); +} + +#[tokio::test] +async fn every_completed_wire_id_is_exported_once() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let spans = all_spans(&bodies); + let ids: std::collections::HashSet<(&str, &str)> = spans + .iter() + .filter_map(|span| Some((span["traceId"].as_str()?, span["spanId"].as_str()?))) + .collect(); + + assert_eq!(spans.len(), 3, "root, generation, and tool expected"); + assert_eq!(ids.len(), spans.len(), "duplicate OTLP wire span ID"); + + for span in &spans { + assert_eq!( + span_attr(span, "langfuse.trace.name"), + Some(&serde_json::json!({ "stringValue": "agent-run" })) + ); + let metadata = span_attr(span, "langfuse.trace.metadata") + .and_then(|value| value["stringValue"].as_str()) + .map(|value| serde_json::from_str::(value).expect("valid metadata")) + .expect("filterable trace metadata"); + assert_eq!(metadata["tenant"], "acme"); + } +} + +#[tokio::test] +async fn cache_tokens_survive_the_round_trip_unsummed() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let spans = all_spans(&bodies); + + let usage = spans + .iter() + .find_map(|s| span_attr(s, "langfuse.observation.usage_details")) + .and_then(|v| v["stringValue"].as_str()) + .map(|s| serde_json::from_str::(s).expect("valid json")) + .expect("usage details present on the wire"); + + // Folding cache reads into `input` would inflate the priced fresh-input + // count by nearly an order of magnitude on a cache-heavy run. + assert_eq!(usage["input"], 120); + assert_eq!(usage["input_cache_read"], 800); + assert_eq!(usage["input_cache_write"], 15); + assert_eq!(usage["total"], 975); +} + +#[tokio::test] +async fn an_unreachable_backend_does_not_stall_the_turn() { + // Port 1 is reserved and refuses connections immediately. + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "dead".into(), + endpoint: "http://127.0.0.1:1/v1/traces".into(), + timeout: Duration::from_millis(50), + profile: ExportProfile::langfuse(), + ..Default::default() + })); + let batch_sink = Arc::new(BatchSink::spawn(transport, BatchConfig { + max_retries: 1, + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(2), + flush_interval: Duration::from_millis(20), + ..Default::default() + })); + let started = std::time::Instant::now(); + let recorder = TurnRecorder::begin_with_sink( + batch_sink.clone(), + "agent-run", + scope(), + RecorderSettings::default(), + ) + .expect("recorder should start"); + for _ in 0..200 { + recorder.step(ObservationKind::Tool, "exec").finish(); + } + recorder.finish(); + let elapsed = started.elapsed(); + + // Recording is a queue push; a dead backend must not be felt by the agent. + assert!( + elapsed < Duration::from_secs(1), + "recording blocked for {elapsed:?} against a dead backend" + ); +} diff --git a/crates/providers/src/anthropic.rs b/crates/providers/src/anthropic.rs index b171fbb37a..a2b58ea6a6 100644 --- a/crates/providers/src/anthropic.rs +++ b/crates/providers/src/anthropic.rs @@ -10,8 +10,8 @@ use {async_trait::async_trait, futures::StreamExt, secrecy::ExposeSecret, tokio_ use tracing::{debug, trace, warn}; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, ContentPart, LlmProvider, StreamEvent, - ToolCall, ToolChoice, Usage, UserContent, + AgentToolControls, ChatMessage, CompletionResponse, ContentPart, InputTokenAccounting, + LlmProvider, StreamEvent, ToolCall, ToolChoice, Usage, UserContent, }; pub struct AnthropicProvider { @@ -823,16 +823,17 @@ impl LlmProvider for AnthropicProvider { let tool_calls = parse_tool_calls(&content); - let usage = Usage { - input_tokens: resp["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32, - output_tokens: resp["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32, - cache_read_tokens: resp["usage"]["cache_read_input_tokens"] + let usage = Usage::from_input_tokens( + InputTokenAccounting::Exclusive, + resp["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32, + resp["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32, + resp["usage"]["cache_read_input_tokens"] .as_u64() .unwrap_or(0) as u32, - cache_write_tokens: resp["usage"]["cache_creation_input_tokens"] + resp["usage"]["cache_creation_input_tokens"] .as_u64() .unwrap_or(0) as u32, - }; + ); if usage.cache_read_tokens > 0 || usage.cache_write_tokens > 0 { debug!( @@ -1053,12 +1054,13 @@ impl LlmProvider for AnthropicProvider { } } "message_stop" => { - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Exclusive, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, - }); + )); return; } "error" => { diff --git a/crates/providers/src/async_openai_provider.rs b/crates/providers/src/async_openai_provider.rs index 61f83af5c5..33e801d84d 100644 --- a/crates/providers/src/async_openai_provider.rs +++ b/crates/providers/src/async_openai_provider.rs @@ -172,10 +172,14 @@ impl LlmProvider for AsyncOpenAiProvider { let usage = response .usage .as_ref() - .map(|u| Usage { - input_tokens: u.prompt_tokens, - output_tokens: u.completion_tokens, - ..Default::default() + .map(|u| { + Usage::from_input_tokens( + moltis_agents::model::InputTokenAccounting::Inclusive, + u.prompt_tokens, + u.completion_tokens, + 0, + 0, + ) }) .unwrap_or_default(); @@ -231,11 +235,13 @@ impl LlmProvider for AsyncOpenAiProvider { } } if let Some(ref u) = response.usage { - yield StreamEvent::Done(Usage { - input_tokens: u.prompt_tokens, - output_tokens: u.completion_tokens, - ..Default::default() - }); + yield StreamEvent::Done(Usage::from_input_tokens( + moltis_agents::model::InputTokenAccounting::Inclusive, + u.prompt_tokens, + u.completion_tokens, + 0, + 0, + )); return; } } diff --git a/crates/providers/src/genai_provider.rs b/crates/providers/src/genai_provider.rs index 439d70e4c6..54434e9b05 100644 --- a/crates/providers/src/genai_provider.rs +++ b/crates/providers/src/genai_provider.rs @@ -3,7 +3,8 @@ use std::pin::Pin; use {async_trait::async_trait, futures::StreamExt, tokio_stream::Stream}; use moltis_agents::model::{ - ChatMessage, CompletionResponse, LlmProvider, StreamEvent, Usage, UserContent, + ChatMessage, CompletionResponse, InputTokenAccounting, LlmProvider, StreamEvent, Usage, + UserContent, }; /// Provider backed by the `genai` crate (supports Anthropic, OpenAI, Gemini, @@ -11,13 +12,19 @@ use moltis_agents::model::{ pub struct GenaiProvider { model: String, provider_name: String, + input_token_accounting: InputTokenAccounting, client: genai::Client, } impl GenaiProvider { /// Create a new `GenaiProvider` with an explicit API key passed via /// `AuthResolver`, avoiding the need to set environment variables. - pub fn new(model: String, provider_name: String, api_key: secrecy::Secret) -> Self { + pub fn new( + model: String, + provider_name: String, + input_token_accounting: InputTokenAccounting, + api_key: secrecy::Secret, + ) -> Self { use secrecy::ExposeSecret; // Expose the secret once to hand it to genai's auth resolver. let key = api_key.expose_secret().clone(); @@ -29,12 +36,16 @@ impl GenaiProvider { Self { model, provider_name, + input_token_accounting, client, } } } -fn genai_usage_to_usage(u: &genai::chat::Usage) -> Usage { +fn genai_usage_to_usage( + input_token_accounting: InputTokenAccounting, + u: &genai::chat::Usage, +) -> Usage { let (cache_read, cache_write) = u .prompt_tokens_details .as_ref() @@ -45,12 +56,13 @@ fn genai_usage_to_usage(u: &genai::chat::Usage) -> Usage { ) }) .unwrap_or((0, 0)); - Usage { - input_tokens: u.prompt_tokens.unwrap_or(0) as u32, - output_tokens: u.completion_tokens.unwrap_or(0) as u32, - cache_read_tokens: cache_read, - cache_write_tokens: cache_write, - } + Usage::from_input_tokens( + input_token_accounting, + u.prompt_tokens.unwrap_or(0) as u32, + u.completion_tokens.unwrap_or(0) as u32, + cache_read, + cache_write, + ) } fn build_genai_messages(messages: &[ChatMessage]) -> Vec { @@ -103,7 +115,7 @@ impl LlmProvider for GenaiProvider { .map_err(|e| anyhow::anyhow!("{e}"))?; let text = chat_res.first_text().map(|s| s.to_string()); - let usage = genai_usage_to_usage(&chat_res.usage); + let usage = genai_usage_to_usage(self.input_token_accounting, &chat_res.usage); Ok(CompletionResponse { text, @@ -147,7 +159,7 @@ impl LlmProvider for GenaiProvider { Ok(ChatStreamEvent::End(end)) => { let usage = end.captured_usage .as_ref() - .map(genai_usage_to_usage) + .map(|usage| genai_usage_to_usage(self.input_token_accounting, usage)) .unwrap_or_default(); yield StreamEvent::Done(usage); return; @@ -162,3 +174,30 @@ impl LlmProvider for GenaiProvider { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalized_prompt_usage_removes_cache_buckets_from_fresh_input() { + let usage = genai::chat::Usage { + prompt_tokens: Some(100), + prompt_tokens_details: Some(genai::chat::PromptTokensDetails { + cached_tokens: Some(30), + cache_creation_tokens: Some(20), + audio_tokens: None, + }), + completion_tokens: Some(10), + total_tokens: Some(110), + ..Default::default() + }; + + let converted = genai_usage_to_usage(InputTokenAccounting::Inclusive, &usage); + + assert_eq!(converted.input_tokens, 50); + assert_eq!(converted.cache_read_tokens, 30); + assert_eq!(converted.cache_write_tokens, 20); + assert_eq!(converted.output_tokens, 10); + } +} diff --git a/crates/providers/src/openai/provider/completion.rs b/crates/providers/src/openai/provider/completion.rs index c39fc3dcb2..eb2e3e86e8 100644 --- a/crates/providers/src/openai/provider/completion.rs +++ b/crates/providers/src/openai/provider/completion.rs @@ -15,7 +15,8 @@ use crate::{ }; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, Usage, decode_tool_call_arguments_from_str, + AgentToolControls, ChatMessage, CompletionResponse, InputTokenAccounting, Usage, + decode_tool_call_arguments_from_str, }; use { @@ -550,12 +551,13 @@ impl OpenAiProvider { Ok(CompletionResponse { text, tool_calls, - usage: Usage { + usage: Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, - }, + ), }) } } diff --git a/crates/providers/src/openai_codex.rs b/crates/providers/src/openai_codex.rs index 3f8ecfcc02..7c45e0ef56 100644 --- a/crates/providers/src/openai_codex.rs +++ b/crates/providers/src/openai_codex.rs @@ -12,8 +12,9 @@ use { }; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, LlmProvider, ReasoningEffort, StreamEvent, - ToolCall, Usage, UserContent, decode_tool_call_arguments_from_str, + AgentToolControls, ChatMessage, CompletionResponse, InputTokenAccounting, LlmProvider, + ReasoningEffort, StreamEvent, ToolCall, Usage, UserContent, + decode_tool_call_arguments_from_str, }; use crate::openai_compat::to_responses_api_tools; @@ -605,12 +606,13 @@ impl LlmProvider for OpenAiCodexProvider { Ok(CompletionResponse { text, tool_calls, - usage: Usage { + usage: Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }, + 0, + ), }) } @@ -755,12 +757,13 @@ impl LlmProvider for OpenAiCodexProvider { for index in tool_calls.keys() { yield StreamEvent::ToolCallComplete { index: *index }; } - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }); + 0, + )); return; } @@ -842,12 +845,13 @@ impl LlmProvider for OpenAiCodexProvider { for index in tool_calls.keys() { yield StreamEvent::ToolCallComplete { index: *index }; } - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }); + 0, + )); return; } "error" | "response.failed" => { diff --git a/crates/providers/src/openai_compat/provider.rs b/crates/providers/src/openai_compat/provider.rs index 662e8a102c..0e045d4230 100644 --- a/crates/providers/src/openai_compat/provider.rs +++ b/crates/providers/src/openai_compat/provider.rs @@ -7,8 +7,8 @@ use std::collections::{HashMap, HashSet}; use { moltis_agents::model::{ - ChatMessage, CompletionResponse, StreamEvent, ToolCall, Usage, UserContent, - decode_tool_call_arguments_with_diagnostic, extract_tool_call_metadata, + ChatMessage, CompletionResponse, InputTokenAccounting, StreamEvent, ToolCall, Usage, + UserContent, decode_tool_call_arguments_with_diagnostic, extract_tool_call_metadata, }, serde::Serialize, tracing::trace, @@ -510,36 +510,59 @@ fn usage_object_from_payload(payload: &serde_json::Value) -> Option<&serde_json: /// - Cache fields may be top-level or nested in `*_tokens_details`. #[must_use] pub fn parse_openai_compat_usage(usage: &serde_json::Value) -> Usage { - Usage { - input_tokens: usage_field_u32(usage, &[ - &["prompt_tokens"], - &["promptTokens"], - &["input_tokens"], - &["inputTokens"], - ]), - output_tokens: usage_field_u32(usage, &[ + let prompt_tokens = usage_field_u32(usage, &[&["prompt_tokens"], &["promptTokens"]]); + let input_tokens = usage_field_u32(usage, &[&["input_tokens"], &["inputTokens"]]); + let cache_read_tokens = usage_field_u32(usage, &[ + &["prompt_tokens_details", "cached_tokens"], + &["promptTokensDetails", "cachedTokens"], + &["input_tokens_details", "cached_tokens"], + &["inputTokensDetails", "cachedTokens"], + &["cache_read_input_tokens"], + &["cacheReadInputTokens"], + &["input_tokens_details", "cache_read_input_tokens"], + &["inputTokensDetails", "cacheReadInputTokens"], + ]); + let cache_write_tokens = usage_field_u32(usage, &[ + &["cache_creation_input_tokens"], + &["cacheCreationInputTokens"], + &["input_tokens_details", "cache_creation_input_tokens"], + &["inputTokensDetails", "cacheCreationInputTokens"], + ]); + + // `prompt_tokens` and OpenAI's nested `cached_tokens` use inclusive + // accounting. Anthropic-compatible cache counters are siblings of an + // already-exclusive `input_tokens` value. + let has_openai_cache_details = usage + .pointer("/prompt_tokens_details/cached_tokens") + .or_else(|| usage.pointer("/promptTokensDetails/cachedTokens")) + .or_else(|| usage.pointer("/input_tokens_details/cached_tokens")) + .or_else(|| usage.pointer("/inputTokensDetails/cachedTokens")) + .is_some(); + let (accounting, reported_input_tokens) = if prompt_tokens > 0 || has_openai_cache_details { + ( + InputTokenAccounting::Inclusive, + if prompt_tokens > 0 { + prompt_tokens + } else { + input_tokens + }, + ) + } else { + (InputTokenAccounting::Exclusive, input_tokens) + }; + + Usage::from_input_tokens( + accounting, + reported_input_tokens, + usage_field_u32(usage, &[ &["completion_tokens"], &["completionTokens"], &["output_tokens"], &["outputTokens"], ]), - cache_read_tokens: usage_field_u32(usage, &[ - &["prompt_tokens_details", "cached_tokens"], - &["promptTokensDetails", "cachedTokens"], - &["input_tokens_details", "cached_tokens"], - &["inputTokensDetails", "cachedTokens"], - &["cache_read_input_tokens"], - &["cacheReadInputTokens"], - &["input_tokens_details", "cache_read_input_tokens"], - &["inputTokensDetails", "cacheReadInputTokens"], - ]), - cache_write_tokens: usage_field_u32(usage, &[ - &["cache_creation_input_tokens"], - &["cacheCreationInputTokens"], - &["input_tokens_details", "cache_creation_input_tokens"], - &["inputTokensDetails", "cacheCreationInputTokens"], - ]), - } + cache_read_tokens, + cache_write_tokens, + ) } /// Parse usage from an OpenAI-compatible payload, checking common nesting variants. diff --git a/crates/providers/src/openai_compat/tests/mod.rs b/crates/providers/src/openai_compat/tests/mod.rs index 2916581fc0..0c703a278a 100644 --- a/crates/providers/src/openai_compat/tests/mod.rs +++ b/crates/providers/src/openai_compat/tests/mod.rs @@ -1,11 +1,50 @@ mod schema_normalization; use super::{ - normalize_tool_call_arguments_from_schemas, parse_responses_completion, parse_tool_calls, - sanitize_schema_for_openai_compat, strict_mode::patch_schema_for_strict_mode, to_openai_tools, - to_responses_api_tools, + normalize_tool_call_arguments_from_schemas, parse_openai_compat_usage, + parse_responses_completion, parse_tool_calls, sanitize_schema_for_openai_compat, + strict_mode::patch_schema_for_strict_mode, to_openai_tools, to_responses_api_tools, }; +#[test] +fn openai_usage_removes_cached_tokens_from_flat_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "prompt_tokens": 1_000, + "completion_tokens": 50, + "prompt_tokens_details": {"cached_tokens": 800} + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.output_tokens, 50); +} + +#[test] +fn responses_usage_removes_cached_tokens_from_flat_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "input_tokens": 1_000, + "output_tokens": 50, + "input_tokens_details": {"cached_tokens": 800} + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); +} + +#[test] +fn anthropic_compatible_usage_keeps_exclusive_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "input_tokens": 200, + "output_tokens": 50, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 20 + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); +} + /// Recursively assert that every `required` entry has a corresponding key in /// `properties`. Panics with `path` context on the first orphaned entry. fn assert_no_orphaned_required(schema: &serde_json::Value, path: &str) { diff --git a/crates/providers/src/registry/registration.rs b/crates/providers/src/registry/registration.rs index 3d661ecbb0..3f7756c8cf 100644 --- a/crates/providers/src/registry/registration.rs +++ b/crates/providers/src/registry/registration.rs @@ -457,6 +457,9 @@ impl ProviderRegistry { let provider = Arc::new(genai_provider::GenaiProvider::new( model_id.clone(), genai_provider_name.clone(), + // genai normalizes every adapter to inclusive prompt tokens, + // including Anthropic's otherwise-exclusive API counters. + moltis_agents::model::InputTokenAccounting::Inclusive, resolved_key, )); self.register( diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 8e4726fbcb..6994a72081 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -108,18 +108,21 @@ impl SlackOutbound { } /// Native Slack streaming using chat.startStream/appendStream/stopStream. + /// + /// Returns the timestamp of the message Slack created for the stream, so a + /// reaction on a natively streamed reply resolves like any other. async fn send_stream_native( &self, account_id: &str, to: &str, thread_ts: Option<&str>, stream: &mut StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let (bot_token, api_base_url, throttle) = self.get_native_stream_config(account_id)?; let http = moltis_common::http_client::build_default_http_client(); - let stream_id = - start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; + let started = start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; + let stream_id = started.stream_id; let mut pending = String::new(); let mut last_append = tokio::time::Instant::now(); @@ -168,22 +171,31 @@ impl SlackOutbound { } } - // Finalize the stream. - if let Err(e) = stop_native_stream(&http, &api_base_url, &bot_token, &stream_id).await { - warn!(account_id, to, "chat.stopStream failed: {e}"); - } + // Finalize the stream. Either call may carry the message timestamp; + // prefer the one from startStream and fall back to stopStream. + let stopped_ts = + match stop_native_stream(&http, &api_base_url, &bot_token, &stream_id).await { + Ok(ts) => ts, + Err(e) => { + warn!(account_id, to, "chat.stopStream failed: {e}"); + None + }, + }; - Ok(()) + Ok(started.ts.or(stopped_ts).into_iter().collect()) } /// Edit-in-place streaming: post → throttled edits → final update. + /// + /// Returns the timestamps of the messages the reply occupies, which are + /// what a later reaction is keyed on. async fn send_stream_edit_in_place( &self, account_id: &str, to: &str, thread_ts: Option<&str>, stream: &mut StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let (client, token) = self.get_session(account_id)?; let throttle = self.get_edit_throttle(account_id); @@ -258,35 +270,94 @@ impl SlackOutbound { } if accumulated.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let final_text = markdown_to_slack(&accumulated); let chunks = chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); match &sent_ts { Some(ts) => { + ids.push(ts.to_string()); if let Some(first) = chunks.first() && let Err(e) = update_message(&client, &token, to, ts, first).await { warn!(account_id, to, "failed to finalize stream message: {e}"); } for chunk in chunks.iter().skip(1) { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send overflow chunk: {e}"); + match post_message(&client, &token, to, chunk, thread_ts).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send overflow chunk: {e}"), } } }, None => { for chunk in &chunks { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send stream message: {e}"); + match post_message(&client, &token, to, chunk, thread_ts).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send stream message: {e}"), } } }, } - Ok(()) + Ok(ids) + } + + /// Drive a stream in whichever mode the account is configured for, + /// returning the timestamps of the messages the reply occupies. + /// + /// All three modes report ids, so a reaction resolves the same way however + /// the account happens to be configured. + async fn send_stream_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: &mut StreamReceiver, + ) -> ChannelResult> { + let stream_mode = self.get_stream_mode(account_id); + let thread_ts = self.get_thread_ts(account_id, to, reply_to); + + match stream_mode { + StreamMode::Native => { + self.send_stream_native(account_id, to, thread_ts.as_deref(), stream) + .await + }, + StreamMode::EditInPlace => { + self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), stream) + .await + }, + StreamMode::Off => { + // Streaming disabled — accumulate and send once. + let mut accumulated = String::new(); + loop { + match stream.recv().await { + Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { + accumulated.push_str(&chunk) + }, + Some(StreamEvent::Error(e)) => { + accumulated.push_str(&format!("\n\n:warning: {e}")); + break; + }, + Some(StreamEvent::Done) | None => break, + } + } + let mut ids = Vec::new(); + if !accumulated.is_empty() { + let (client, token) = self.get_session(account_id)?; + let final_text = markdown_to_slack(&accumulated); + for chunk in chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN) { + match post_message(&client, &token, to, chunk, thread_ts.as_deref()).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send stream message: {e}"), + } + } + } + Ok(ids) + }, + } } } @@ -486,6 +557,66 @@ impl ChannelOutbound for SlackOutbound { Ok(()) } + /// Slack identifies a message by its `ts`, which is what a + /// `reaction_added` event carries back. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let (client, token) = self.get_session(account_id)?; + let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let slack_text = markdown_to_slack(text); + + let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); + let mut ids = Vec::new(); + for chunk in chunks { + let ts = post_message(&client, &token, to, chunk, thread_ts.as_deref()).await?; + ids.push(ts.to_string()); + } + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + moltis_metrics::channels::MESSAGES_SENT_TOTAL, + moltis_metrics::labels::CHANNEL => "slack" + ) + .increment(1); + + Ok(ids) + } + + /// Slack has no suffix rendering, so the inherited `send_text_with_suffix` + /// drops it and posts the text alone. Route the reporting variant to + /// `send_text_reporting_ids` rather than the trait default, which would + /// throw away ids Slack can perfectly well supply. + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + _suffix_html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + self.send_text_reporting_ids(account_id, to, text, reply_to) + .await + } + + /// Slack renders no HTML either, so the inherited `send_html` posts the + /// markup as text. Report its ids rather than inheriting the default that + /// drops them. + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + self.send_text_reporting_ids(account_id, to, html, reply_to) + .await + } + async fn send_media( &self, account_id: &str, @@ -661,13 +792,22 @@ impl ChannelOutbound for SlackOutbound { /// Start a native Slack stream via `chat.startStream`. /// /// Returns `(stream_id, channel)` on success. +/// What `chat.startStream` hands back. +struct NativeStream { + stream_id: String, + /// Timestamp of the message Slack created to hold the stream, when the API + /// reports one. This is the id a later reaction is keyed on, so dropping it + /// would leave natively streamed replies unattributable. + ts: Option, +} + async fn start_native_stream( http: &reqwest::Client, api_base_url: &str, bot_token: &str, channel: &str, thread_ts: Option<&str>, -) -> ChannelResult { +) -> ChannelResult { let mut body = serde_json::json!({ "channel": channel }); if let Some(ts) = thread_ts { body["thread_ts"] = serde_json::json!(ts); @@ -696,10 +836,13 @@ async fn start_native_stream( ))); } - json.get("stream_id") + let stream_id = json + .get("stream_id") .and_then(|v| v.as_str()) .map(String::from) - .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing stream_id")) + .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing stream_id"))?; + let ts = json.get("ts").and_then(|v| v.as_str()).map(String::from); + Ok(NativeStream { stream_id, ts }) } /// Append text to a native Slack stream via `chat.appendStream`. @@ -742,12 +885,15 @@ async fn append_native_stream( } /// Finalize a native Slack stream via `chat.stopStream`. +/// Finalize a native stream, reporting the resulting message timestamp when +/// Slack includes one — a fallback for the `chat.startStream` response that +/// omits it. async fn stop_native_stream( http: &reqwest::Client, api_base_url: &str, bot_token: &str, stream_id: &str, -) -> ChannelResult<()> { +) -> ChannelResult> { let body = serde_json::json!({ "stream_id": stream_id }); let resp = http @@ -773,7 +919,7 @@ async fn stop_native_stream( ))); } - Ok(()) + Ok(json.get("ts").and_then(|v| v.as_str()).map(String::from)) } #[async_trait] @@ -785,47 +931,20 @@ impl ChannelStreamOutbound for SlackOutbound { reply_to: Option<&str>, mut stream: StreamReceiver, ) -> ChannelResult<()> { - let stream_mode = self.get_stream_mode(account_id); - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + self.send_stream_ids(account_id, to, reply_to, &mut stream) + .await?; + Ok(()) + } - match stream_mode { - StreamMode::Native => { - self.send_stream_native(account_id, to, thread_ts.as_deref(), &mut stream) - .await - }, - StreamMode::EditInPlace => { - self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), &mut stream) - .await - }, - StreamMode::Off => { - // Streaming disabled — accumulate and send once. - let mut accumulated = String::new(); - loop { - match stream.recv().await { - Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { - accumulated.push_str(&chunk) - }, - Some(StreamEvent::Error(e)) => { - accumulated.push_str(&format!("\n\n:warning: {e}")); - break; - }, - Some(StreamEvent::Done) | None => break, - } - } - if !accumulated.is_empty() { - let (client, token) = self.get_session(account_id)?; - let final_text = markdown_to_slack(&accumulated); - for chunk in chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN) { - if let Err(e) = - post_message(&client, &token, to, chunk, thread_ts.as_deref()).await - { - warn!(account_id, to, "failed to send stream message: {e}"); - } - } - } - Ok(()) - }, - } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + mut stream: StreamReceiver, + ) -> ChannelResult> { + self.send_stream_ids(account_id, to, reply_to, &mut stream) + .await } async fn is_stream_enabled(&self, account_id: &str) -> bool { diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index ba8b535679..5e174b8a5e 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -652,9 +652,8 @@ pub(crate) async fn handle_reaction_event( /// Core reaction handling shared by Socket Mode and the Events API. /// -/// Always emits a [`ChannelEvent::ReactionChange`] for observers; additionally -/// routes the reaction into the agent as a synthetic message when -/// `reaction_triggers` is enabled and the reaction is eligible. +/// Emits authorized additions and user-scoped removals to observers; +/// additionally routes eligible additions into the agent as synthetic messages. pub(crate) async fn dispatch_reaction( account_id: &str, user_id: &str, @@ -676,25 +675,6 @@ pub(crate) async fn dispatch_reaction( } }; - let Some(sink) = event_sink else { - return; - }; - - // Always surface the raw reaction change to observers (web UI, hooks). - sink.emit(ChannelEvent::ReactionChange { - channel_type: ChannelType::Slack, - account_id: account_id.to_string(), - chat_id: channel_id.clone(), - message_id: message_ts.clone(), - user_id: user_id.to_string(), - emoji: emoji.to_string(), - added, - }) - .await; - - // Optionally route the reaction into the agent as a message. The bot's own - // acknowledgment reactions (👀/✅/❌) are always ignored to avoid loops. - // // Fail closed: if the bot user id is unknown we cannot distinguish the bot's // own reactions, so treat the reactor as "self" and skip. Triggering here // would let the bot's own ACK reactions fire agent turns and loop. @@ -708,16 +688,6 @@ pub(crate) async fn dispatch_reaction( true }, }; - if !reaction_should_trigger( - config.reaction_triggers, - added, - is_self, - emoji, - &config.reaction_trigger_emojis, - ) { - return; - } - let is_dm = channel_id.starts_with('D'); let access_granted = check_access( is_dm, @@ -728,14 +698,40 @@ pub(crate) async fn dispatch_reaction( &config.allowlist, &config.channel_allowlist, ); - if !access_granted { + if is_self || (added && !access_granted) { debug!( account_id, - user_id, "slack reaction trigger denied by access control" + user_id, "slack reaction denied by access control" ); return; } + let Some(sink) = event_sink else { + return; + }; + + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Slack, + account_id: account_id.to_string(), + chat_id: channel_id.clone(), + message_id: message_ts.clone(), + user_id: user_id.to_string(), + emoji: emoji.to_string(), + added, + }) + .await; + + // Optionally route the reaction into the agent as a message. + if !reaction_should_trigger( + config.reaction_triggers, + added, + is_self, + emoji, + &config.reaction_trigger_emojis, + ) { + return; + } + // Thread the synthetic message under the reacted message so the agent sees // the original content as thread context (dispatch fetches thread history). let reply_to = ChannelReplyTarget { diff --git a/crates/swift-bridge/src/state.rs b/crates/swift-bridge/src/state.rs index e8719365ea..779c81f686 100644 --- a/crates/swift-bridge/src/state.rs +++ b/crates/swift-bridge/src/state.rs @@ -197,14 +197,24 @@ pub(crate) static HTTPD: Mutex> = Mutex::new(None); pub(crate) fn stop_httpd_handle(handle: HttpdHandle, log_target: &str, stop_message: &str) { emit_log("INFO", log_target, stop_message); - let _ = handle.shutdown_tx.send(()); + let HttpdHandle { + shutdown_tx, + server_task, + state, + .. + } = handle; + let _ = shutdown_tx.send(()); BRIDGE.runtime.block_on(async { - if let Err(error) = handle.server_task.await { + if let Err(error) = server_task.await { emit_log( "WARN", log_target, &format!("httpd task join failed during shutdown: {error}"), ); } + state + .instrumentation + .shutdown(std::time::Duration::from_secs(5)) + .await; }); } diff --git a/crates/telegram/src/bot.rs b/crates/telegram/src/bot.rs index 1a743676c3..cd7f4ff823 100644 --- a/crates/telegram/src/bot.rs +++ b/crates/telegram/src/bot.rs @@ -111,6 +111,10 @@ pub async fn start_polling( AllowedUpdate::Message, AllowedUpdate::EditedMessage, AllowedUpdate::CallbackQuery, + // Telegram only delivers reaction updates when they are + // explicitly requested; without this the bot never learns + // that anyone reacted. + AllowedUpdate::MessageReaction, ]) .await; @@ -158,6 +162,10 @@ pub async fn start_polling( ); } }, + UpdateKind::MessageReaction(reaction) => { + handlers::handle_message_reaction(reaction, &aid, &poll_accounts) + .await; + }, UpdateKind::CallbackQuery(query) => { debug!( account_id = aid, diff --git a/crates/telegram/src/handlers.rs b/crates/telegram/src/handlers.rs index e96e1f05f0..e63bafc1a7 100644 --- a/crates/telegram/src/handlers.rs +++ b/crates/telegram/src/handlers.rs @@ -5,5 +5,7 @@ #[path = "handlers/implementation.rs"] mod implementation; +#[path = "handlers/reactions.rs"] +mod reactions; -pub use self::implementation::*; +pub use self::{implementation::*, reactions::handle_message_reaction}; diff --git a/crates/telegram/src/handlers/reactions.rs b/crates/telegram/src/handlers/reactions.rs new file mode 100644 index 0000000000..617e7b5196 --- /dev/null +++ b/crates/telegram/src/handlers/reactions.rs @@ -0,0 +1,221 @@ +//! Inbound reaction updates. +//! +//! Telegram reports reactions as a whole-set diff rather than an add/remove +//! event: each update carries the reaction lists before and after the change. +//! The interesting part is therefore the difference between them. + +use std::sync::Arc; + +use { + moltis_channels::{ChannelEvent, ChannelType}, + moltis_common::types::ChatType, + teloxide::types::{MessageReactionUpdated, ReactionType}, + tracing::debug, +}; + +use crate::{access, state::AccountStateMap}; + +/// Emoji for a reaction, or `None` for kinds a vocabulary cannot match. +fn reaction_token(reaction: &ReactionType) -> Option { + match reaction { + ReactionType::Emoji { emoji } => Some(emoji.clone()), + // Custom emoji are identified by an opaque id with no readable name, + // so there is nothing a feedback vocabulary could match against. + _ => None, + } +} + +/// Reactions present in `after` but not `before`, and vice versa. +fn diff(before: &[ReactionType], after: &[ReactionType]) -> (Vec, Vec) { + let old: Vec = before.iter().filter_map(reaction_token).collect(); + let new: Vec = after.iter().filter_map(reaction_token).collect(); + + let added = new + .iter() + .filter(|e| !old.contains(e)) + .cloned() + .collect::>(); + let removed = old + .iter() + .filter(|e| !new.contains(e)) + .cloned() + .collect::>(); + (added, removed) +} + +fn ordered_changes(before: &[ReactionType], after: &[ReactionType]) -> Vec<(String, bool)> { + let (added, removed) = diff(before, after); + removed + .into_iter() + .map(|emoji| (emoji, false)) + .chain(added.into_iter().map(|emoji| (emoji, true))) + .collect() +} + +/// Handle a `message_reaction` update. +pub async fn handle_message_reaction( + update: MessageReactionUpdated, + account_id: &str, + accounts: &AccountStateMap, +) { + // Anonymous channel reactions carry no user; without one, two different + // people's votes would collapse onto a single score id. + let Some(user) = update.actor.user() else { + return; + }; + + let (config, event_sink) = { + let accts = accounts.read().unwrap_or_else(|e| e.into_inner()); + match accts.get(account_id) { + Some(state) => (state.config.clone(), state.event_sink.clone()), + None => return, + } + }; + let Some(sink) = event_sink else { + return; + }; + + let (chat_type, group_id) = match update.chat.kind { + teloxide::types::ChatKind::Private(_) => (ChatType::Dm, None), + teloxide::types::ChatKind::Public(ref public) => { + let group_id = Some(update.chat.id.0.to_string()); + match public.kind { + teloxide::types::PublicChatKind::Channel(_) => (ChatType::Channel, group_id), + _ => (ChatType::Group, group_id), + } + }, + }; + let peer_id = user.id.0.to_string(); + let access_granted = access::check_access( + &config, + &chat_type, + &peer_id, + user.username.as_deref(), + group_id.as_deref(), + // Reactions do not need a fresh mention, but all other DM/group policy + // and allowlist checks still apply. + true, + ) + .is_ok(); + + let changes = ordered_changes(&update.old_reaction, &update.new_reaction); + if changes.is_empty() { + return; + } + + debug!( + account_id, + changes = changes.len(), + "telegram reaction update" + ); + + let emit = |emoji: String, was_added: bool| { + let sink = Arc::clone(&sink); + let account_id = account_id.to_string(); + let chat_id = update.chat.id.0.to_string(); + let message_id = update.message_id.0.to_string(); + let user_id = peer_id.clone(); + async move { + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Telegram, + account_id, + chat_id, + message_id, + user_id, + emoji, + added: was_added, + }) + .await; + } + }; + + for (emoji, added) in changes { + // A removal can only retract this user's own score, so preserve source + // truth even if an allowlist changed after the score was recorded. + if added && !access_granted { + debug!(account_id, user_id = %peer_id, "telegram reaction denied by access control"); + continue; + } + emit(emoji, added).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn emoji(raw: &str) -> ReactionType { + ReactionType::Emoji { + emoji: raw.to_string(), + } + } + + #[test] + fn a_new_reaction_is_reported_as_added() { + let (added, removed) = diff(&[], &[emoji("\u{1f44d}")]); + + assert_eq!(added, vec!["\u{1f44d}".to_string()]); + assert!(removed.is_empty()); + } + + #[test] + fn clearing_a_reaction_is_reported_as_removed() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[]); + + assert!(added.is_empty()); + assert_eq!(removed, vec!["\u{1f44d}".to_string()]); + } + + #[test] + fn switching_reactions_reports_both_sides() { + // Telegram sends one update for a swap; scoring it needs the retract + // and the new vote, not just whichever happened to be last. + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[emoji("\u{1f44e}")]); + + assert_eq!(added, vec!["\u{1f44e}".to_string()]); + assert_eq!(removed, vec!["\u{1f44d}".to_string()]); + } + + #[test] + fn switching_reactions_retracts_before_setting_the_replacement() { + assert_eq!( + ordered_changes(&[emoji("\u{1f44d}")], &[emoji("\u{1f44e}")]), + vec![ + ("\u{1f44d}".to_string(), false), + ("\u{1f44e}".to_string(), true) + ] + ); + } + + #[test] + fn an_unchanged_reaction_set_produces_nothing() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[emoji("\u{1f44d}")]); + + assert!(added.is_empty()); + assert!(removed.is_empty()); + } + + #[test] + fn adding_alongside_an_existing_reaction_only_reports_the_new_one() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[ + emoji("\u{1f44d}"), + emoji("\u{2764}"), + ]); + + assert_eq!(added, vec!["\u{2764}".to_string()]); + assert!(removed.is_empty()); + } + + #[test] + fn custom_emoji_are_skipped() { + // Identified by an opaque id with no readable name, so a vocabulary + // has nothing to match. + let custom = ReactionType::CustomEmoji { + custom_emoji_id: teloxide::types::CustomEmojiId("12345".to_string()), + }; + let (added, removed) = diff(&[], &[custom]); + + assert!(added.is_empty()); + assert!(removed.is_empty()); + } +} diff --git a/crates/telegram/src/outbound/send.rs b/crates/telegram/src/outbound/send.rs index 5c31687ace..986237ceb1 100644 --- a/crates/telegram/src/outbound/send.rs +++ b/crates/telegram/src/outbound/send.rs @@ -21,66 +21,53 @@ use crate::{ use super::{TelegramOutbound, retry::RequestResultExt}; -#[async_trait] -impl ChannelOutbound for TelegramOutbound { - async fn send_text( +impl TelegramOutbound { + /// Send raw HTML chunks, returning every message id they produced. + async fn send_html_ids( &self, account_id: &str, to: &str, - text: &str, + html: &str, reply_to: Option<&str>, - ) -> Result<()> { + ) -> Result> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; let rp = self.reply_params(account_id, reply_to); - // Send typing indicator - let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; - - let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); - info!( - account_id, - chat_id = to, - reply_to = ?reply_to, - text_len = text.len(), - chunk_count = chunks.len(), - "telegram outbound text send start" - ); - - for chunk in chunks.iter() { - let reply_params = rp.as_ref(); - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - chunk, - reply_params, - false, - ) - .await?; + // Send raw HTML chunks without markdown conversion. + let chunks = markdown::chunk_message(html, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); + for chunk in &chunks { + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); } - - info!( - account_id, - chat_id = to, - reply_to = ?reply_to, - text_len = text.len(), - chunk_count = chunks.len(), - "telegram outbound text sent" - ); - Ok(()) + Ok(ids) } - async fn send_text_with_suffix( + /// Send `text` with `suffix_html` appended, returning every message id it + /// produced. + /// + /// Shared by both trait entry points so the chunking and suffix-overflow + /// rules exist once; the non-reporting one simply discards the ids. + async fn send_text_with_suffix_ids( &self, account_id: &str, to: &str, text: &str, suffix_html: &str, reply_to: Option<&str>, - ) -> Result<()> { + ) -> Result> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; let rp = self.reply_params(account_id, reply_to); @@ -101,6 +88,7 @@ impl ChannelOutbound for TelegramOutbound { "telegram outbound text+suffix send start" ); + let mut ids = Vec::with_capacity(chunks.len()); for (i, chunk) in chunks.iter().enumerate() { let content = if i == last_idx { // Append suffix to the last chunk. If it would exceed the limit, @@ -110,29 +98,36 @@ impl ChannelOutbound for TelegramOutbound { combined } else { // Send this chunk first, then the suffix as a separate message. - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - chunk, - rp.as_ref(), - false, - ) - .await?; + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); // Send suffix as the final message (no reply threading). - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - suffix_html, - rp.as_ref(), - true, - ) - .await?; + let suffix_id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + suffix_html, + rp.as_ref(), + true, + ) + .await?; + // A reader rating "the bot's answer" may land on the + // logbook, and it maps back to the same turn, so linking it + // recovers a score that would otherwise be dropped. + ids.push(suffix_id.0.to_string()); info!( account_id, chat_id = to, @@ -142,22 +137,24 @@ impl ChannelOutbound for TelegramOutbound { chunk_count = chunks.len(), "telegram outbound text+suffix sent (separate suffix message)" ); - return Ok(()); + return Ok(ids); } } else { chunk.clone() }; - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - &content, - rp.as_ref(), - false, - ) - .await?; + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + &content, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); } info!( @@ -169,23 +166,38 @@ impl ChannelOutbound for TelegramOutbound { chunk_count = chunks.len(), "telegram outbound text+suffix sent" ); - Ok(()) + Ok(ids) } +} - async fn send_html( +#[async_trait] +impl ChannelOutbound for TelegramOutbound { + async fn send_text( &self, account_id: &str, to: &str, - html: &str, + text: &str, reply_to: Option<&str>, ) -> Result<()> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; let rp = self.reply_params(account_id, reply_to); - // Send raw HTML chunks without markdown conversion. - let chunks = markdown::chunk_message(html, TELEGRAM_MAX_MESSAGE_LEN); - for chunk in &chunks { + // Send typing indicator + let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; + + let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); + info!( + account_id, + chat_id = to, + reply_to = ?reply_to, + text_len = text.len(), + chunk_count = chunks.len(), + "telegram outbound text send start" + ); + + for chunk in chunks.iter() { + let reply_params = rp.as_ref(); self.send_chunk_with_fallback( &bot, account_id, @@ -193,14 +205,112 @@ impl ChannelOutbound for TelegramOutbound { chat_id, thread_id, chunk, - rp.as_ref(), + reply_params, false, ) .await?; } + + info!( + account_id, + chat_id = to, + reply_to = ?reply_to, + text_len = text.len(), + chunk_count = chunks.len(), + "telegram outbound text sent" + ); + Ok(()) + } + + /// Telegram splits long replies into several messages, so every chunk id + /// is reported: a reader may react to any of them. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + let bot = self.get_bot(account_id)?; + let (chat_id, thread_id) = parse_chat_target(to)?; + let rp = self.reply_params(account_id, reply_to); + + let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; + + let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); + for chunk in chunks.iter() { + let reply_params = rp.as_ref(); + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + reply_params, + false, + ) + .await?; + ids.push(id.0.to_string()); + } + + info!( + account_id, + chat_id = to, + chunk_count = ids.len(), + "telegram outbound text sent with ids" + ); + Ok(ids) + } + + async fn send_text_with_suffix( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result<()> { + self.send_text_with_suffix_ids(account_id, to, text, suffix_html, reply_to) + .await?; Ok(()) } + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text_with_suffix_ids(account_id, to, text, suffix_html, reply_to) + .await + } + + async fn send_html( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result<()> { + self.send_html_ids(account_id, to, html, reply_to).await?; + Ok(()) + } + + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_html_ids(account_id, to, html, reply_to).await + } + async fn send_typing(&self, account_id: &str, to: &str) -> Result<()> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; diff --git a/crates/telegram/src/outbound/stream.rs b/crates/telegram/src/outbound/stream.rs index 632d6db54a..61c6262426 100644 --- a/crates/telegram/src/outbound/stream.rs +++ b/crates/telegram/src/outbound/stream.rs @@ -512,6 +512,7 @@ impl TelegramOutbound { Ok(()) } + /// Send the chunks that did not fit the edited message, reporting their ids. async fn send_remaining_final_chunks( &self, bot: &Bot, @@ -521,23 +522,26 @@ impl TelegramOutbound { thread_id: Option, reply_params: Option<&ReplyParameters>, final_state: &StreamFinalState, - ) -> Result<()> { + ) -> Result> { let chunks = markdown::chunk_markdown_html(&final_state.accumulated, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::new(); for chunk in chunks.into_iter().skip(1) { - self.send_chunk_with_fallback( - bot, - account_id, - to, - chat_id, - thread_id, - &chunk, - reply_params, - true, - ) - .await?; + let id = self + .send_chunk_with_fallback( + bot, + account_id, + to, + chat_id, + thread_id, + &chunk, + reply_params, + true, + ) + .await?; + ids.push(id.0.to_string()); } - Ok(()) + Ok(ids) } async fn finish_final_stream_message( @@ -552,9 +556,9 @@ impl TelegramOutbound { progress_message_id: &mut Option, final_state: &mut StreamFinalState, throttle: Duration, - ) -> Result<()> { + ) -> Result> { if final_state.accumulated.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let display = final_state.current_final_html(); @@ -667,15 +671,19 @@ fn trim_to_recent_chars(text: &mut String, max_chars: usize) -> bool { true } -#[async_trait] -impl ChannelStreamOutbound for TelegramOutbound { - async fn send_stream( +impl TelegramOutbound { + /// Drive an edit-in-place stream, returning the ids of the messages the + /// final reply ended up occupying. + /// + /// Both trait entry points share this so the streaming state machine + /// exists once; [`ChannelStreamOutbound::send_stream`] discards the ids. + async fn send_stream_ids( &self, account_id: &str, to: &str, reply_to: Option<&str>, mut stream: StreamReceiver, - ) -> Result<()> { + ) -> Result> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; let rp = self.reply_params(account_id, reply_to); @@ -693,12 +701,15 @@ impl ChannelStreamOutbound for TelegramOutbound { typing_interval.tick().await; let mut flush_interval = tokio::time::interval(throttle); flush_interval.tick().await; + // Chunks that spilled past the edited message; the edited message's own + // id is read off `final_state` once the loop ends. + let mut overflow_ids: Vec = Vec::new(); loop { tokio::select! { event = stream.recv() => { let Some(event) = event else { - self.finish_final_stream_message( + overflow_ids = self.finish_final_stream_message( &bot, account_id, to, @@ -836,7 +847,7 @@ impl ChannelStreamOutbound for TelegramOutbound { } }, StreamEvent::Done => { - self.finish_final_stream_message( + overflow_ids = self.finish_final_stream_message( &bot, account_id, to, @@ -915,9 +926,37 @@ impl ChannelStreamOutbound for TelegramOutbound { .await; } + let mut ids = Vec::with_capacity(overflow_ids.len() + 1); + ids.extend(final_state.message_id.map(|id| id.0.to_string())); + ids.extend(overflow_ids); + Ok(ids) + } +} + +#[async_trait] +impl ChannelStreamOutbound for TelegramOutbound { + async fn send_stream( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result<()> { + self.send_stream_ids(account_id, to, reply_to, stream) + .await?; Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream_ids(account_id, to, reply_to, stream).await + } + async fn is_stream_enabled(&self, account_id: &str) -> bool { let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); accounts diff --git a/crates/web/src/assets/css/components.css b/crates/web/src/assets/css/components.css index 0d95e14437..4dc63ca764 100644 --- a/crates/web/src/assets/css/components.css +++ b/crates/web/src/assets/css/components.css @@ -1243,6 +1243,10 @@ -webkit-mask-image: url("../icons/masks/mask-b9bf0315417c03e4.svg"); mask-image: url("../icons/masks/mask-b9bf0315417c03e4.svg"); } +.settings-nav-item[data-section="instrumentation"]::before { + -webkit-mask-image: url("../icons/masks/instrumentation.svg"); + mask-image: url("../icons/masks/instrumentation.svg"); +} .settings-nav-item[data-section="logs"]::before { -webkit-mask-image: url("../icons/masks/mask-5883df2e24ebc841.svg"); mask-image: url("../icons/masks/mask-5883df2e24ebc841.svg"); diff --git a/crates/web/src/assets/icons/masks/instrumentation.svg b/crates/web/src/assets/icons/masks/instrumentation.svg new file mode 100644 index 0000000000..7c9aa8eb7a --- /dev/null +++ b/crates/web/src/assets/icons/masks/instrumentation.svg @@ -0,0 +1 @@ + diff --git a/crates/web/src/assets/icons/masks/mask-thumbs-down.svg b/crates/web/src/assets/icons/masks/mask-thumbs-down.svg new file mode 100644 index 0000000000..9f8f7e445e --- /dev/null +++ b/crates/web/src/assets/icons/masks/mask-thumbs-down.svg @@ -0,0 +1 @@ + diff --git a/crates/web/src/assets/icons/masks/mask-thumbs-up.svg b/crates/web/src/assets/icons/masks/mask-thumbs-up.svg new file mode 100644 index 0000000000..1470413343 --- /dev/null +++ b/crates/web/src/assets/icons/masks/mask-thumbs-up.svg @@ -0,0 +1 @@ + diff --git a/crates/web/ui/e2e/helpers.js b/crates/web/ui/e2e/helpers.js index aa42c4ca6c..2aaf43e4fc 100644 --- a/crates/web/ui/e2e/helpers.js +++ b/crates/web/ui/e2e/helpers.js @@ -250,7 +250,14 @@ async function waitForChatSessionReady(page) { ); } -function isRetryableRpcError(message) { +function isRetryableRpcError(error) { + // The client localizes its own errors before returning them, so + // "WebSocket not connected" reaches us as "Service temporarily + // unavailable. Please try again." Matching the raw English here made this + // retry dead code, and the flake it exists to absorb surfaced as a failed + // assertion instead. The code survives localization, so match on that. + if (error?.code === "UNAVAILABLE") return true; + const message = error?.message; if (typeof message !== "string") return false; return message.includes("WebSocket not connected") || message.includes("WebSocket disconnected"); } @@ -298,7 +305,7 @@ async function sendRpcFromPage(page, method, params) { .catch((error) => ({ ok: false, error: { message: error?.message || String(error) } })); if (lastResponse?.ok) return lastResponse; - if (!isRetryableRpcError(lastResponse?.error?.message)) return lastResponse; + if (!isRetryableRpcError(lastResponse?.error)) return lastResponse; } console.log(`[sendRpc] ${method} FAILED after 3 attempts, last: ${lastResponse?.error?.message?.slice(0, 100)}`); return lastResponse; diff --git a/crates/web/ui/e2e/specs/feedback.spec.js b/crates/web/ui/e2e/specs/feedback.spec.js new file mode 100644 index 0000000000..11eef4b6b6 --- /dev/null +++ b/crates/web/ui/e2e/specs/feedback.spec.js @@ -0,0 +1,77 @@ +const { expect, test } = require("../base-test"); +const { navigateAndWait, watchPageErrors, expectRpcOk, sendRpcFromPage, waitForWsConnected } = require("../helpers"); + +test.describe("Reaction feedback", () => { + test("status RPC responds", async ({ page }) => { + await navigateAndWait(page, "/"); + await waitForWsConnected(page); + + const response = await expectRpcOk(page, "feedback.status", {}); + expect(typeof response.payload.enabled).toBe("boolean"); + expect(typeof response.payload.retention_days).toBe("number"); + }); + + test("reports feedback as unavailable while instrumentation is off", async ({ page }) => { + await navigateAndWait(page, "/"); + await waitForWsConnected(page); + + // A thumb that goes nowhere is worse than no thumb, so the control is + // gated on something actually collecting the score. + const response = await expectRpcOk(page, "feedback.status", {}); + expect(response.payload.enabled).toBe(false); + }); + + test("submitting without a session key is rejected", async ({ page }) => { + await navigateAndWait(page, "/"); + await waitForWsConnected(page); + + const response = await sendRpcFromPage(page, "feedback.submit", { + messageId: "run-1", + signal: "positive", + }); + + expect(response.ok).toBe(false); + }); + + test("submitting an unknown signal is rejected", async ({ page }) => { + await navigateAndWait(page, "/"); + await waitForWsConnected(page); + + // The wire format must not be able to smuggle an arbitrary value into + // the score. + const response = await sendRpcFromPage(page, "feedback.submit", { + sessionKey: "agent:main:main", + messageId: "run-1", + signal: "0.7", + }); + + expect(response.ok).toBe(false); + }); + + test("a thumb on an unlinked message reports why rather than failing silently", async ({ page }) => { + await navigateAndWait(page, "/"); + await waitForWsConnected(page); + + const response = await sendRpcFromPage(page, "feedback.submit", { + sessionKey: "agent:main:main", + messageId: "run-that-never-existed", + signal: "positive", + }); + + // The RPC itself succeeds; the outcome explains that the turn is not + // attributable, which is not an error the user can retry away. + expect(response.ok).toBe(true); + expect(response.payload.ok).toBe(false); + expect(["unknown_message", "disabled"]).toContain(response.payload.outcome); + }); + + test("chat page loads with no JS errors when feedback is unavailable", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/"); + + // The action bar appends thumbs asynchronously; a failed or disabled + // status check must not surface as a page error. + await expect(page.locator("#chatInput")).toBeVisible(); + expect(pageErrors).toEqual([]); + }); +}); diff --git a/crates/web/ui/e2e/specs/instrumentation.spec.js b/crates/web/ui/e2e/specs/instrumentation.spec.js new file mode 100644 index 0000000000..584b9f6e3f --- /dev/null +++ b/crates/web/ui/e2e/specs/instrumentation.spec.js @@ -0,0 +1,78 @@ +const { expect, test } = require("../base-test"); +const { navigateAndWait, watchPageErrors, expectRpcOk, waitForWsConnected } = require("../helpers"); + +test.describe("Instrumentation settings", () => { + test("page loads with the section heading", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/settings/instrumentation"); + + await expect(page.getByRole("heading", { name: "Instrumentation", exact: true })).toBeVisible(); + expect(pageErrors).toEqual([]); + }); + + test("shows a card for every backend", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // All three backends are always listed, including the disabled ones, so + // an operator can see what is available without reading the docs first. + await expect(page.getByText("Langfuse", { exact: true })).toBeVisible(); + await expect(page.getByText("Datadog", { exact: true })).toBeVisible(); + await expect(page.getByText("OpenTelemetry (Grafana, Honeycomb, Collector)", { exact: true })).toBeVisible(); + }); + + test("explains that backends receive different data", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // The Langfuse/APM content asymmetry is the single most important thing + // for an operator to understand before enabling anything, so it must be + // on the page itself rather than only in the docs. + await expect(page.getByText("What each backend receives", { exact: true })).toBeVisible(); + }); + + test("reports instrumentation as off by default", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // Defaulting to on would ship conversation content to a third party + // without the operator having chosen to. + await expect(page.getByText(/Instrumentation is off/)).toBeVisible(); + }); + + test("test-connection button is disabled while Langfuse is off", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + const button = page.getByRole("button", { name: "Test connection" }); + await expect(button).toBeVisible(); + await expect(button).toBeDisabled(); + }); + + test("status RPC responds", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + await waitForWsConnected(page); + await expectRpcOk(page, "instrumentation.status", {}); + }); + + test("shows honest delivery status before exporters are enabled", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + await expect(page.getByText("Delivery", { exact: true })).toBeVisible(); + await expect(page.getByText("no exporters running", { exact: true })).toBeVisible(); + }); + + test("status RPC never returns the secret key", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + await waitForWsConnected(page); + const res = await expectRpcOk(page, "instrumentation.status", {}); + + // The UI only ever needs to know whether a key is configured. Returning + // the value would put it in every browser devtools network log. + const serialized = JSON.stringify(res); + expect(serialized).not.toContain('secret_key"'); + expect(serialized).toContain("secret_key_set"); + }); + + test("page has no JS errors", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/settings/instrumentation"); + expect(pageErrors).toEqual([]); + }); +}); diff --git a/crates/web/ui/e2e/specs/sessions.spec.js b/crates/web/ui/e2e/specs/sessions.spec.js index 84c1ae9613..4b210be209 100644 --- a/crates/web/ui/e2e/specs/sessions.spec.js +++ b/crates/web/ui/e2e/specs/sessions.spec.js @@ -97,6 +97,7 @@ test.describe("Session management", () => { await page.clock.install({ time: new Date(2026, 6, 23, 23, 58) }); const pageErrors = await navigateAndWait(page, "/"); await waitForWsConnected(page); + await expect(page.locator('#sessionList .session-item[data-session-key="main"]')).toBeVisible(); await page.clock.pauseAt(new Date(2026, 6, 23, 23, 59, 59, 500)); const expected = await page.evaluate(() => { diff --git a/crates/web/ui/e2e/specs/settings-nav.spec.js b/crates/web/ui/e2e/specs/settings-nav.spec.js index 5dcd534f03..f9d163b339 100644 --- a/crates/web/ui/e2e/specs/settings-nav.spec.js +++ b/crates/web/ui/e2e/specs/settings-nav.spec.js @@ -24,9 +24,7 @@ test.describe("Settings navigation", () => { await expect(page.getByRole("heading", { name: "User Profile", exact: true })).toBeVisible(); }); - test("settings nav keeps distinct icons for nodes, remote access, network audit, and openclaw import", async ({ - page, - }) => { + test("settings nav keeps distinct icons for systems and integrations", async ({ page }) => { const pageErrors = watchPageErrors(page); await navigateAndWait(page, "/settings/profile"); await expect(page.locator(".settings-sidebar-nav")).toBeVisible(); @@ -58,6 +56,7 @@ test.describe("Settings navigation", () => { networkAudit: readRuleMask('.settings-nav-item[data-section="network-audit"]::before'), mcp: readRuleMask('.settings-nav-item[data-section="mcp"]::before'), openclawImport: readRuleMask('.settings-nav-item[data-section="import"]::before'), + instrumentation: readRuleMask('.settings-nav-item[data-section="instrumentation"]::before'), }; }); @@ -73,6 +72,7 @@ test.describe("Settings navigation", () => { expect(hasMask(masks.tools)).toBeTruthy(); expect(hasMask(masks.remoteAccess)).toBeTruthy(); expect(hasMask(masks.networkAudit)).toBeTruthy(); + expect(hasMask(masks.instrumentation)).toBeTruthy(); expect(hasMask(masks.mcp)).toBeTruthy(); expect(masks.remoteAccess).not.toBe(masks.networkAudit); @@ -670,6 +670,7 @@ test.describe("Settings navigation", () => { "Terminal", "Monitoring", "Logs", + "Instrumentation", ...presentOptionalItems(["GraphQL"]), "Configuration", ]; diff --git a/crates/web/ui/input.css b/crates/web/ui/input.css index 8d22b13b6b..5cc405ce01 100644 --- a/crates/web/ui/input.css +++ b/crates/web/ui/input.css @@ -1960,6 +1960,10 @@ -webkit-mask-image: url("./icons/masks/mask-b9bf0315417c03e4.svg"); mask-image: url("./icons/masks/mask-b9bf0315417c03e4.svg"); } + .icon-instrumentation { + -webkit-mask-image: url("./icons/masks/instrumentation.svg"); + mask-image: url("./icons/masks/instrumentation.svg"); + } .icon-server { -webkit-mask-image: url("./icons/masks/mask-7a1361dacb5b8427.svg"); mask-image: url("./icons/masks/mask-7a1361dacb5b8427.svg"); @@ -2212,6 +2216,14 @@ -webkit-mask-image: url("./icons/masks/mask-git-fork.svg"); mask-image: url("./icons/masks/mask-git-fork.svg"); } + .icon-thumbs-up { + -webkit-mask-image: url("./icons/masks/mask-thumbs-up.svg"); + mask-image: url("./icons/masks/mask-thumbs-up.svg"); + } + .icon-thumbs-down { + -webkit-mask-image: url("./icons/masks/mask-thumbs-down.svg"); + mask-image: url("./icons/masks/mask-thumbs-down.svg"); + } .icon-list-plus { -webkit-mask-image: url("./icons/masks/mask-list-plus.svg"); mask-image: url("./icons/masks/mask-list-plus.svg"); diff --git a/crates/web/ui/package-lock.json b/crates/web/ui/package-lock.json index 796f7649d1..bbc2436c3d 100644 --- a/crates/web/ui/package-lock.json +++ b/crates/web/ui/package-lock.json @@ -1606,9 +1606,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1626,9 +1623,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1646,9 +1640,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1666,9 +1657,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1686,9 +1674,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1706,9 +1691,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4131,9 +4113,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4155,9 +4134,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4179,9 +4155,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4203,9 +4176,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/crates/web/ui/src/components/forms/SectionLayout.tsx b/crates/web/ui/src/components/forms/SectionLayout.tsx index 1df282dc26..b247f30447 100644 --- a/crates/web/ui/src/components/forms/SectionLayout.tsx +++ b/crates/web/ui/src/components/forms/SectionLayout.tsx @@ -126,7 +126,12 @@ export function StatusMessage({ error, success, className }: StatusMessageProps) const color = error ? "var(--error)" : "var(--accent)"; const text = error ?? success; return ( -
+
{text}
); diff --git a/crates/web/ui/src/message-actions.ts b/crates/web/ui/src/message-actions.ts index 1a9683e1a8..f0e248b76b 100644 --- a/crates/web/ui/src/message-actions.ts +++ b/crates/web/ui/src/message-actions.ts @@ -8,6 +8,7 @@ import { isChatAtBottom, scrollChatToBottom } from "./chat-ui"; import * as gon from "./gon"; import { sendRpc } from "./helpers"; +import { buildFeedbackButtons, feedbackStatus } from "./message-feedback"; import { renderPersistedAudio } from "./message-voice"; import { copyToClipboard, showToast } from "./ui"; @@ -138,6 +139,17 @@ export function appendMessageActions(ctx: MessageActionContext): void { bar.appendChild(voiceBtn); } + // ── Feedback thumbs ────────────────────────────────────── + // Appended asynchronously: availability depends on whether any backend is + // collecting scores, and the action bar must not wait on that check. + void feedbackStatus().then((status) => { + if (!(status?.enabled && status.instrumentation_active)) return; + if (!bar.isConnected) return; + const buttons = buildFeedbackButtons({ sessionKey, runId: ctx.runId }, actionButton); + if (!buttons) return; + for (const button of buttons) bar.appendChild(button); + }); + // ── Fork button ────────────────────────────────────────── const forkBtn = actionButton("icon-git-fork", "Fork into new session"); forkBtn.addEventListener("click", () => { diff --git a/crates/web/ui/src/message-feedback.ts b/crates/web/ui/src/message-feedback.ts new file mode 100644 index 0000000000..893e582064 --- /dev/null +++ b/crates/web/ui/src/message-feedback.ts @@ -0,0 +1,97 @@ +// ── Thumbs up/down feedback on an assistant message ────────── +// +// Submits a "user-feedback" score against the trace that produced the +// message. The control only appears when instrumentation is actually +// collecting scores — a thumb that goes nowhere is worse than no thumb. + +import { sendRpc } from "./helpers"; +import { showToast } from "./ui"; + +export type FeedbackSignal = "positive" | "negative"; + +interface FeedbackStatus { + enabled: boolean; + instrumentation_active: boolean; + retention_days: number; +} + +let statusPromise: Promise | null = null; + +/** Feedback availability, fetched once per page load. */ +export function feedbackStatus(): Promise { + if (!statusPromise) { + statusPromise = sendRpc("feedback.status", {}).then((result) => { + if (!result?.ok) return null; + return (result.payload as FeedbackStatus | undefined) ?? null; + }); + } + return statusPromise; +} + +/** Reset the cached status, for tests and after a settings change. */ +export function resetFeedbackStatus(): void { + statusPromise = null; +} + +export interface FeedbackContext { + sessionKey: string; + runId?: string; +} + +/** + * Build the thumbs up/down pair, or `null` when there is nothing to attribute + * a score to. + * + * A message with no run id predates the correlation table, so a thumb on it + * could not be tied to a trace. + */ +export function buildFeedbackButtons( + ctx: FeedbackContext, + makeButton: (iconClass: string, title: string) => HTMLButtonElement, +): HTMLElement[] | null { + if (!ctx.runId) return null; + + let active: FeedbackSignal | null = null; + const up = makeButton("icon-thumbs-up", "Good response"); + const down = makeButton("icon-thumbs-down", "Bad response"); + + const submit = async (signal: FeedbackSignal, button: HTMLButtonElement): Promise => { + // Clicking the active thumb withdraws the opinion rather than + // re-asserting it. + const clearing = active === signal; + const result = await sendRpc("feedback.submit", { + sessionKey: ctx.sessionKey, + messageId: ctx.runId, + signal: clearing ? "clear" : signal, + }); + + const payload = result?.payload as { ok?: boolean; outcome?: string } | undefined; + if (!(result?.ok && payload?.ok)) { + // Say why rather than failing silently: "unknown_message" means the + // turn aged out of the retention window, which is not the user's + // fault and not something a retry fixes. + const reason = + payload?.outcome === "unknown_message" + ? "This message is too old to rate" + : payload?.outcome === "disabled" + ? "Feedback collection is turned off" + : "Could not record feedback"; + showToast(reason, "error"); + return; + } + + up.classList.remove("msg-action-btn-active"); + down.classList.remove("msg-action-btn-active"); + active = clearing ? null : signal; + if (!clearing) button.classList.add("msg-action-btn-active"); + up.setAttribute("aria-pressed", String(active === "positive")); + down.setAttribute("aria-pressed", String(active === "negative")); + }; + + up.setAttribute("aria-pressed", "false"); + down.setAttribute("aria-pressed", "false"); + up.addEventListener("click", () => void submit("positive", up)); + down.addEventListener("click", () => void submit("negative", down)); + + return [up, down]; +} diff --git a/crates/web/ui/src/pages/SettingsPage.tsx b/crates/web/ui/src/pages/SettingsPage.tsx index 5f689e8c13..0faa31c07a 100644 --- a/crates/web/ui/src/pages/SettingsPage.tsx +++ b/crates/web/ui/src/pages/SettingsPage.tsx @@ -42,6 +42,7 @@ import { ConfigSection, GraphqlSection } from "./sections/ConfigSection"; import { EnvironmentSection } from "./sections/EnvironmentSection"; import { IdentitySection } from "./sections/IdentitySection"; import { ImportSection } from "./sections/ImportSection"; +import { InstrumentationSection } from "./sections/InstrumentationSection"; import { MemorySection } from "./sections/MemorySection"; import { NotificationsSection } from "./sections/NotificationsSection"; import { PhoneSection } from "./sections/PhoneSection"; @@ -200,6 +201,11 @@ const sections: SectionItem[] = [ { id: "terminal", label: "Terminal", page: true }, { id: "monitoring", label: "Monitoring", page: true }, { id: "logs", label: "Logs", page: true }, + { + id: "instrumentation", + label: "Instrumentation", + icon: , + }, { id: "graphql", label: "GraphQL" }, { id: "config", label: "Configuration" }, ]; @@ -410,6 +416,7 @@ function SettingsPage(): VNode { {section === "phone" ? : null} {section === "notifications" ? : null} {section === "import" ? : null} + {section === "instrumentation" ? : null} {section === "graphql" ? : null} {section === "config" ? : null}
diff --git a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx new file mode 100644 index 0000000000..bbfe1dbf98 --- /dev/null +++ b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx @@ -0,0 +1,366 @@ +// ── Instrumentation section ───────────────────────────────── +// +// One page for every tracing backend, because they are all fed from a single +// instrumentation pass in the agent runtime. The page is deliberately explicit +// about the fact that Langfuse and the APM backends receive *different* data: +// that asymmetry is the design, not an accident, and an operator who does not +// know about it will either send prompts somewhere they should not, or wonder +// why their Datadog spans look empty. + +import type { VNode } from "preact"; +import { useEffect, useState } from "preact/hooks"; +import { Badge, Loading, SectionHeading, StatusMessage, SubHeading } from "../../components/forms"; +import { sendRpc } from "../../helpers"; +import { connected } from "../../signals"; +import { rerender } from "./_shared"; + +type ContentMode = "full" | "metadata_only" | "none"; + +interface SkippedBackend { + name: string; + reason: string; +} + +interface LangfuseConfig { + enabled: boolean; + host: string; + public_key: string; + secret_key_set: boolean; + capture_input: boolean; + capture_output: boolean; + capture_tool_io: boolean; +} + +interface OtlpConfig { + enabled: boolean; + endpoint: string; + content: ContentMode; + emit_user_id: boolean; +} + +interface DatadogConfig { + enabled: boolean; + endpoint: string; + service: string; + api_key_set: boolean; + content: ContentMode; +} + +interface InstrumentationConfig { + enabled: boolean; + environment: string; + sample_rate: number; + queue_capacity: number; + langfuse: LangfuseConfig; + otlp: OtlpConfig; + datadog: DatadogConfig; +} + +interface InstrumentationStatus { + active: boolean; + backends: string[]; + skipped: SkippedBackend[]; + delivery: DeliveryStats[]; + config: InstrumentationConfig; +} + +interface DeliveryStats { + name: string; + accepted: number; + dropped_queue_full: number; + dropped_failed: number; + delivered: number; + retries: number; + last_success_at: string | null; + last_error: string | null; + last_error_at: string | null; +} + +interface TestResult { + ok: boolean; + error?: string; +} + +const CONTENT_LABELS: Record = { + full: "Full conversation", + metadata_only: "Metadata only", + none: "Nothing", +}; + +// ── Read-only field rows ──────────────────────────────────── + +interface RowProps { + label: string; + value: string; + hint?: string; +} + +function Row({ label, value, hint }: RowProps): VNode { + return ( +
+ {label} + + {value} + {hint ? {hint} : null} + +
+ ); +} + +function deliveryLabel(name: string): string { + if (name === "langfuse") return "Langfuse traces"; + if (name === "langfuse-scores") return "Langfuse scores"; + if (name === "otlp") return "OpenTelemetry"; + if (name === "datadog") return "Datadog"; + return name; +} + +function formatDeliveryTime(value?: string | null): string { + if (!value) return "never"; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); +} + +interface DeliveryRowsProps { + delivery: DeliveryStats[]; +} + +function DeliveryRows({ delivery }: DeliveryRowsProps): VNode { + if (delivery.length === 0) { + return ; + } + + return ( + <> + {delivery.map((stats) => { + const failures = `${stats.dropped_queue_full} queue drops, ${stats.dropped_failed} failed, ${stats.retries} retries`; + const latest = stats.last_error + ? `Last success: ${formatDeliveryTime(stats.last_success_at)}. Last error: ${stats.last_error} (${formatDeliveryTime(stats.last_error_at)}).` + : `Last success: ${formatDeliveryTime(stats.last_success_at)}.`; + return ( + + ); + })} + + ); +} + +interface BackendCardProps { + title: string; + purpose: string; + enabled: boolean; + running: boolean; + skippedReason?: string; + children: VNode | VNode[]; +} + +function BackendCard({ title, purpose, enabled, running, skippedReason, children }: BackendCardProps): VNode { + return ( +
+
+ {title} + {running ? ( + + ) : enabled ? ( + + ) : ( + + )} +
+

{purpose}

+ {skippedReason ? ( +

Enabled in config but did not start: {skippedReason}

+ ) : null} + {children} +
+ ); +} + +// ── Section ───────────────────────────────────────────────── + +interface BackendsProps { + config: InstrumentationConfig; + status: InstrumentationStatus; + testing: boolean; + onTestLangfuse: () => void; +} + +/** The three backend cards, extracted so the section body stays readable. */ +function Backends({ config, status, testing, onTestLangfuse }: BackendsProps): VNode { + const skippedFor = (name: string): string | undefined => status.skipped.find((s) => s.name === name)?.reason; + const isRunning = (name: string): boolean => status.backends.includes(name); + + const captures = + [ + config.langfuse.capture_input ? "input" : null, + config.langfuse.capture_output ? "output" : null, + config.langfuse.capture_tool_io ? "tool I/O" : null, + ] + .filter(Boolean) + .join(", ") || "structure only"; + + return ( + <> + + + + + + + + + + + + + + + + + + + + + + ); +} + +/** Explains why backends receive different data. */ +function ProfileExplainer(): VNode { + return ( +
+ +

+ Langfuse gets completed observations with the full conversation, token usage and session context. It infers cost + from its current model pricing. OTLP and Datadog get operational shape only: latency, errors, model and token + counts, with payload sizes instead of payloads. Prompt bodies in an APM mean unbounded span size, cardinality + pressure, per-byte ingest billing, and conversation content in a system nobody scoped for it. +

+
+ ); +} + +export function InstrumentationSection(): VNode { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(null); + const [testing, setTesting] = useState(false); + + useEffect(() => { + if (!connected.value) return; + setLoading(true); + setError(null); + sendRpc("instrumentation.status", {}) + .then((res) => { + if (res.payload) setStatus(res.payload); + else setError(res.error?.message ?? "Failed to load instrumentation status"); + }) + .catch((e: Error) => setError(e.message)) + .finally(() => { + setLoading(false); + rerender(); + }); + }, [connected.value]); + + function onTestLangfuse(): void { + setTesting(true); + setError(null); + setSuccess(null); + rerender(); + sendRpc("instrumentation.test", { backend: "langfuse" }) + .then((res) => { + if (res.payload?.ok) setSuccess("Langfuse reachable and credentials accepted."); + else setError(res.payload?.error ?? res.error?.message ?? "Connection test failed"); + }) + .catch((e: Error) => setError(e.message)) + .finally(() => { + setTesting(false); + rerender(); + }); + } + + if (loading) return ; + if (!status) { + return ( +
+ + +
+ ); + } + + const { config } = status; + + return ( +
+ + +

+ Exports what each agent run did (LLM calls, tool calls, retrievals) to an external backend. Configured in{" "} + moltis.toml under [instrumentation], or in Settings then Configuration, and applied on + restart. This page is read-only: it shows what is running and lets you verify connectivity. +

+ + {config.enabled ? null : ( +

+ Instrumentation is off. It stays off until you enable it explicitly, because turning it on sends data about + your conversations to a third party. +

+ )} + + + + +
+ + + 0 ? status.backends.join(", ") : "none"} /> + + + + +
+ + +
+ ); +} diff --git a/crates/web/ui/src/types/rpc-methods.ts b/crates/web/ui/src/types/rpc-methods.ts index d543f7dd04..b6ac2b97e0 100644 --- a/crates/web/ui/src/types/rpc-methods.ts +++ b/crates/web/ui/src/types/rpc-methods.ts @@ -60,6 +60,11 @@ export interface RpcMethodMap { // ── Exec ──────────────────────────────────────────────────── "exec.approval.resolve": unknown; + + // ── Feedback ──────────────────────────────────────────────── + "feedback.status": unknown; + "feedback.submit": unknown; + "external_agents.bind": unknown; "external_agents.list": ExternalAgentInfo[]; "external_agents.status": unknown; @@ -80,6 +85,10 @@ export interface RpcMethodMap { "hooks.reload": unknown; "hooks.save": unknown; + // ── Instrumentation ───────────────────────────────────────── + "instrumentation.status": unknown; + "instrumentation.test": unknown; + // ── Location ──────────────────────────────────────────────── "location.result": unknown; diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 3500f903c5..4e2364b0e9 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -91,6 +91,7 @@ - [Streaming](streaming.md) - [SQLite Migrations](sqlite-migration.md) - [Metrics & Tracing](metrics-and-tracing.md) +- [Instrumentation](instrumentation.md) - [Tool Registry](tool-registry.md) - [Tool Policy](tool-policy.md) - [Agent Presets](agent-presets.md) diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md new file mode 100644 index 0000000000..f66d6633ac --- /dev/null +++ b/docs/src/instrumentation.md @@ -0,0 +1,396 @@ +# Instrumentation + +Moltis can export what a completed agent run did — its LLM calls, tool +invocations and retrievals, with timings, token usage and errors — to an +external backend. + +Three backends are supported, and they are configured together under one +`[instrumentation]` section because they are fed from a single instrumentation +pass in the agent runtime: + +| Backend | What it is for | +| --- | --- | +| **Langfuse** | LLM observability: prompts, completions, sessions, token usage, inferred cost, and reaction feedback | +| **OTLP** | Any OpenTelemetry backend — Grafana Tempo/Alloy, Honeycomb, an OpenTelemetry Collector | +| **Datadog** | Datadog APM, through the Datadog Agent's OTLP intake | + +> **Instrumentation is disabled by default.** Unlike every other Moltis feature +> flag, it stays off until you turn it on: enabling it sends data about your +> conversations to a third party. Read [What gets sent](#what-gets-sent) before +> switching it on. + +## Backends receive different data, on purpose + +This is the most important thing to understand about the design. + +Langfuse is an LLM-native product. Moltis sends completed traces, conversation +content, token usage, session and user attribution, and reaction feedback so +Langfuse can provide LLM observability and infer costs. + +Datadog and Grafana are infrastructure tools. Sending them prompt bodies is +actively harmful: + +- span size becomes unbounded, and most vendors bill per ingested byte; +- prompt text as a span attribute is a cardinality problem in a trace index; +- it copies user conversation content into a system that was never scoped, + reviewed, or access-controlled for it. + +So Moltis applies a different **export profile** per backend: + +| | Langfuse | OTLP / Datadog | +| --- | --- | --- | +| Prompts, completions, tool arguments and results | ✅ sent | ❌ **never sent** by default | +| Payload sizes (`moltis.input.bytes`) | — | ✅ sent | +| Observation type (`AGENT`, `TOOL`, `RETRIEVER`, …) | ✅ `langfuse.observation.type` | ✅ as `gen_ai.operation.name` | +| Model, latency, errors | ✅ | ✅ | +| Token counts | ✅ usage details with cache split; Langfuse infers cost | ✅ counts only | +| End-user id | ✅ | ❌ off by default (cardinality) | +| Session id | ✅ | ✅ | +| Tags | ✅ | ✅ OTLP · ❌ Datadog (billing) | + +You can raise an APM's content level deliberately with +`content = "full"`, or lower it to `none`. You cannot accidentally end up +sending prompts to Datadog: the default for any newly configured OTLP-family +backend is `metadata_only`. + +**For Grafana specifically:** most of what you want is already available +without this feature. Moltis exposes a Prometheus endpoint at `/metrics` with +`moltis_llm_*` counters for tokens, latency and time-to-first-token. Scrape +that for dashboards and alerts; use the OTLP exporter here when you want +per-run traces to correlate a latency spike with a specific agent run. + +Moltis does not currently integrate Langfuse Prompt Management, datasets or +dataset runs, evaluators, or media uploads. User reaction scores are the only +evaluation signal sent directly to Langfuse. + +## Quick start: Langfuse + +Prefer the process environment for the secret key: + +```sh +export MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY='sk-lf-...' +``` + +```toml +[instrumentation] +enabled = true +environment = "production" + +[instrumentation.langfuse] +enabled = true +host = "https://cloud.langfuse.com" # or your self-hosted URL +public_key = "pk-lf-..." +``` + +Keys come from your Langfuse project settings. For a self-hosted deployment, +point `host` at your own instance — no data leaves your network. + +Once Moltis has restarted, use **Settings → Instrumentation → Test connection** +in the web UI to verify the host is reachable and the credentials are accepted +before relying on it. + +## Quick start: Grafana / OpenTelemetry Collector + +```toml +[instrumentation] +enabled = true + +[instrumentation.otlp] +enabled = true +endpoint = "http://localhost:4318/v1/traces" +content = "metadata_only" # default; "full" or "none" also accepted + +[instrumentation.otlp.headers] +Authorization = "Basic ..." # if your collector requires auth +``` + +## Quick start: Datadog + +Run the Datadog Agent with OTLP ingest enabled and point Moltis at it. This is +the recommended setup — the Agent handles batching, retries and your API key, +so no Datadog credential needs to live in the Moltis config at all. + +```toml +[instrumentation] +enabled = true + +[instrumentation.datadog] +enabled = true +endpoint = "http://localhost:4318/v1/traces" +service = "moltis" +``` + +To post to Datadog's intake directly instead, set `endpoint` to the regional +OTLP intake URL and supply `api_key`. + +## Where the settings live + +Instrumentation is configured in `moltis.toml` under `[instrumentation]`, either +by editing the file directly or through **Settings → Configuration** in the web +UI, which edits the same TOML. Either way the change takes effect on restart. + +**Settings → Instrumentation is a read-only view.** It reports what is actually +running — active backends, any that failed to start and why, delivery counters, +the capture policy in force, and whether credentials are present — and it can +test the Langfuse connection. It deliberately has no fields to edit: turning +instrumentation on sends conversation data to a third party, so that decision +lives in the config file next to the credential it needs, not behind a toggle in +a browser. + +## Build features + +Both exporter families are compiled in by default, so nothing here needs a +rebuild — only the config above switches them on. To leave one out of a build: + +```bash +# Langfuse only (drops the generic OTLP and Datadog exporters). +cargo build --no-default-features --features "web-ui,tls,langfuse" + +# No exporter at all. +cargo build --no-default-features --features "web-ui,tls" +``` + +`langfuse` implies `otlp`, because Langfuse ingests traces over the OTLP wire +format, and `otlp` also carries Datadog. A backend enabled in config that the +running binary was not built with is reported in **Settings → Instrumentation** +as skipped with the reason `not compiled into this build`, rather than failing +quietly. + +## Running several backends at once + +Backends are independent. Enabling all three sends full traces to Langfuse and +operational spans to Grafana and Datadog from the same run, each with its own +content policy. A backend that fails to start is reported in the settings UI +with a reason; the others keep running. + +## What gets sent + +With Langfuse enabled and default settings, each completed agent run produces: + +- a trace named `agent-run` whose root observation has type **`AGENT`** and + carries the user's message and final answer; +- a **`GENERATION`** observation for each completed LLM call, with the full + message array sent to the provider, completion text, model, token usage split + into fresh / cache-read / cache-write buckets, and time-to-first-token; +- a completed **`TOOL`** or **`RETRIEVER`** observation for each tool call, + with arguments and results; +- the session key as the session id, so a Langfuse session matches a Moltis + conversation; +- the channel sender as the user id, namespaced per channel (`telegram:42`). + +Root and child observations are immutable and exported exactly once, after +completion. Moltis does not send in-progress observation updates. Cancelled or +timed-out work is closed as an error rather than exported as successful work. + +You can narrow this without turning the backend off: + +```toml +[instrumentation.langfuse] +capture_input = false # drop turn and LLM inputs +capture_output = false # drop turn and LLM outputs +capture_tool_io = false # drop tool arguments and results +``` + +`capture_tool_io` is separate from the others because tool arguments are the +likeliest place for a credential to appear — a `curl` command with a bearer +token, a connection string, an API key passed to a script. + +### Redaction + +Redaction runs before anything is serialized, in two passes: + +1. **By key name** — any object key containing `password`, `secret`, `token`, + `api_key`, `authorization`, `credential`, `private_key` and similar has its + value replaced with `[REDACTED]`, at any nesting depth. +2. **By value shape** — string values that look like credentials are replaced + even under an innocuous key, because tool output is unstructured. Covers + `sk-`, `ghp_`, `github_pat_`, `xoxb-`, `AKIA`, `AIza`, `Bearer …`, PEM + private key blocks and others. + +Add your own key patterns; they extend the built-in list rather than replacing +it, so you cannot accidentally weaken the baseline: + +```toml +[instrumentation] +redact = ["customer_ref", "internal_id"] +``` + +Redaction is a strong mitigation, not a guarantee. It cannot detect a secret +that looks like ordinary prose. If your agents routinely handle sensitive data, +prefer `capture_tool_io = false`, a self-hosted Langfuse, or both. + +### Transport security + +Plaintext `http://` endpoints are refused for non-loopback hosts. These +payloads carry conversation content and a credential in a header; shipping them +unencrypted across a network is not something you should be able to do by +mistyping a URL. `http://localhost` and `http://127.0.0.1` are allowed, for a +local Agent or collector. + +## Sampling and volume + +```toml +[instrumentation] +sample_rate = 1.0 # fraction of turns traced, 0.0-1.0 +queue_capacity = 10000 # bounded export queue +flush_interval_ms = 5000 +max_batch_bytes = 3000000 +``` + +Sampling is per-turn: a sampled turn is traced completely, so you never get a +trace with holes in it. + +Telemetry never blocks an agent. Events go into a bounded queue and are +**dropped** if it fills, rather than slowing a user's turn. Drops are counted +and shown in the settings UI — if you see them, raise `queue_capacity` or lower +`sample_rate`. + +## Configuration reference + +### `[instrumentation]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | Master switch. Off by default. | +| `environment` | `"production"` | Reported to every backend. | +| `release` | running version | Build identifier. | +| `sample_rate` | `1.0` | Fraction of turns traced. | +| `redact` | `[]` | Extra key patterns to redact. | +| `queue_capacity` | `10000` | Bounded export queue depth. | +| `flush_interval_ms` | `5000` | Maximum time an event waits before export. | +| `max_batch_bytes` | `3000000` | Forced flush threshold. | + +### `[instrumentation.langfuse]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `host` | `https://cloud.langfuse.com` | Set to your self-hosted URL to keep data on-premises. | +| `public_key` | `""` | | +| `secret_key` | unset | Prefer `MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY`; a config value is also accepted and is never logged. | +| `capture_input` | `true` | | +| `capture_output` | `true` | | +| `capture_tool_io` | `true` | | +| `timeout_secs` | `10` | | + +### `[instrumentation.otlp]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `endpoint` | `""` | Full traces URL, e.g. `http://localhost:4318/v1/traces`. | +| `headers` | `{}` | Extra headers, typically auth. | +| `content` | `"metadata_only"` | `full`, `metadata_only`, or `none`. | +| `emit_user_id` | `false` | High-cardinality in an APM index. | +| `timeout_secs` | `10` | | + +### `[instrumentation.datadog]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `endpoint` | `http://localhost:4318/v1/traces` | The Datadog Agent's OTLP intake. | +| `api_key` | unset | Only needed when posting to the intake directly. | +| `service` | `"moltis"` | Datadog service name. | +| `content` | `"metadata_only"` | | +| `timeout_secs` | `10` | | + +### `[instrumentation.feedback]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `true` | Collect reaction feedback. Gated by the master switch. | +| `positive` | `[]` | Reaction tokens counted as approval. Empty uses the built-in list. | +| `negative` | `[]` | Reaction tokens counted as disapproval. Empty uses the built-in list. | +| `link_retention_days` | `30` | How long a reply stays attributable to its turn. | + +## User feedback + +A thumbs up or down on a reply becomes a `user-feedback` score with Langfuse +data type `BOOLEAN`, attached to the trace that produced the reply. The Scores +API represents the value as `1.0` for positive and `0.0` for negative. + +Feedback works in three places: + +- **Telegram, Discord and Slack** — react to any message the bot sent. +- **The web UI** — thumbs appear in the action bar under each assistant + message, but only while instrumentation is active and feedback is enabled. + A control that goes nowhere is worse than no control. + +Reactions accept raw emoji or shortcodes, and skin-tone and presentation +selectors are ignored when matching, so `👍`, `👍🏾`, `:+1:` and `thumbsup` are +one signal. The default vocabulary is deliberately narrow; a default that +swallowed every positive-looking emoji would turn release parties into quality +data. Override either list to change it: + +```toml +[instrumentation.feedback] +positive = ["👍", "🎉", "ship-it"] +``` + +Setting one list leaves the other on its defaults. + +**Changing your mind works.** Score ids are derived from the trace and the +reacting user. Switching from 👍 to 👎 submits the same id, so Langfuse replaces +the vote instead of recording both. Removing the reaction deletes the score +through the dedicated Scores API. Creates, replacements and deletions share one +ordered delivery queue, preventing an older queued create from recreating a +score after deletion. Two reacting users still count separately. + +### Why a reaction sometimes does nothing + +- **The reply is older than `link_retention_days`.** Attribution comes from a + correlation table written when the reply is sent, and it is pruned on that + schedule. The web UI says so explicitly rather than failing silently. +- **The reply predates instrumentation being switched on.** There was no trace + to attribute it to. +- **The channel cannot report the ids of messages it sends.** Telegram, Discord + and Slack can; other channels deliver the message but lose feedback + attribution. + +## Cost + +Moltis exports the model id and token usage details, including separate +cache-read and cache-write buckets, but it does not compute or export a cost of +its own. Langfuse infers cost from its own versioned model definitions and +pricing tiers, which stay current without Moltis shipping a price table that +would go stale and then override them. + +Unknown, private or custom models may show no cost until their pricing is +configured in Langfuse. + +## How it works + +Trace observations are sent to Langfuse using OTLP/HTTP JSON at +`/api/public/otel/v1/traces`. Every OTLP request includes +`x-langfuse-ingestion-version: 4`, selecting Langfuse's v4 ingestion contract. +The exporter sends only completed, immutable root and child observations. + +OTLP cannot represent feedback scores. Moltis therefore creates, replaces and +deletes `BOOLEAN` feedback through Langfuse's dedicated Scores API at +`/api/public/scores`, using a separate ordered sink. + +Spans carry two attribute vocabularies depending on the profile: + +- `langfuse.*` — Langfuse's mapping for trace context, observation taxonomy, + content, and detailed token usage. +- `gen_ai.*` — the OpenTelemetry GenAI semantic conventions, understood by + Grafana, Datadog, Honeycomb and other OTel consumers. + +## Troubleshooting + +**No traces appear.** Check `enabled = true` at both the `[instrumentation]` +level and the backend level — the master switch gates everything. Then use +Test connection in the settings UI. + +**Traces appear but token counts look wrong.** Moltis reports fresh input +tokens separately from cache reads and writes, because Langfuse prices them +differently. A run with heavy prompt caching will show a low `input` and a +large `input_cache_read`; that is correct, not a bug. + +**Events are being dropped.** The export queue is full — the backend is slower +than the agent is producing events. Raise `queue_capacity`, lower +`sample_rate`, or check backend latency. + +**A backend shows as skipped.** The settings UI gives the reason: missing +credentials, an empty endpoint, or a rejected plaintext URL. diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index 103d924b46..75f088d690 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -210,8 +210,17 @@ strip_all_features_flag() { changed_files() { if [[ "$LOCAL_ONLY" -eq 0 ]]; then - gh pr diff "$PR_NUMBER" --repo "$BASE_REPO" --name-only - return + # `gh pr diff` answers HTTP 406 once a PR exceeds 20,000 diff lines, and it + # exits 0 while printing nothing usable. Left alone that turns the targeted + # contexts into no-ops that still report "passed", so a large PR would be + # the least tested one. Fall back to the local range instead. + local pr_files="" + if pr_files="$(gh pr diff "$PR_NUMBER" --repo "$BASE_REPO" --name-only 2>/dev/null)" \ + && [[ -n "$pr_files" ]]; then + printf '%s\n' "$pr_files" + return + fi + echo "[changed-files] gh pr diff unavailable for #${PR_NUMBER}; using the local branch range." >&2 fi local base_ref="${LOCAL_VALIDATE_BASE_REF:-$BASE_REF_NAME}" @@ -274,10 +283,14 @@ build_targeted_rust_test_cmd() { local base_cmd base_cmd="$(nextest_base_cmd_for_package "$package")" - if [[ "$file" == */tests/*.rs ]]; then + if [[ "$file" =~ ^crates/[^/]+/tests/[^/]+\.rs$ ]]; then local test_name test_name="$(basename "$file" .rs)" commands+=("$base_cmd --test $test_name") + elif [[ "$file" == */tests/*.rs ]]; then + # Nested source test modules do not map to Cargo --test targets, and a + # basename such as `mod` is not a reliable nextest filter. + commands+=("$base_cmd") elif grep -Eq '#\[(tokio::)?test\]' "$file" 2>/dev/null; then local filter_name filter_name="$(basename "$file" .rs)" @@ -286,6 +299,9 @@ build_targeted_rust_test_cmd() { done < <(changed_files) if [[ "${#commands[@]}" -eq 0 ]]; then + # Announced here, in the main log: the context's own output is captured and + # discarded on success, so a skip would otherwise look like a green run. + echo "[local/test] no changed Rust tests detected; nothing to run." >&2 printf '%s' 'echo "No changed Rust tests detected; skipping local/test."' return fi @@ -311,6 +327,7 @@ build_targeted_e2e_cmd() { done < <(changed_files) if [[ "${#specs[@]}" -eq 0 ]]; then + echo "[local/e2e] no changed Playwright specs detected; nothing to run." >&2 printf '%s' 'echo "No changed Playwright specs detected; skipping local/e2e."' return fi