From a5172dc2c45cf44c2cff41842eeb6297b7aefb45 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 05:12:41 +0000 Subject: [PATCH 01/33] feat(observability): add instrumentation core with per-backend export profiles Moltis had no trace export of any kind. The workspace declared four opentelemetry crates that no code used, and nothing recorded what an agent run actually did beyond Prometheus counters. This adds moltis-observability: one instrumentation pass in the agent runtime, fanned out to any number of backends through an ObservationSink. The central design point is that backends do not receive the same payload. Langfuse is an LLM-native product whose cost, session and prompt-version views are built on conversation content, so it gets prompts, completions, tool arguments and results, the full observation taxonomy, cache-split token usage and managed-prompt linkage. Datadog and Grafana are operational tools: prompt bodies there mean unbounded span size, cardinality pressure, per-byte ingest billing, and user conversation content sitting in a system nobody scoped to hold it. They get latency, errors, model and token counts, with payload sizes instead of payloads and no per-user cardinality. An ExportProfile carries that policy and the mapping layer consults it before emitting any attribute; the default profile is the conservative one so a newly configured backend cannot silently receive conversation content. Transport is OTLP/JSON for every backend. That is Langfuse's own modern ingest path, used by its v3+ Python and v4+ JS SDKs, and the only one carrying the AGENT/TOOL/RETRIEVER observation types: the native ingestion API's observation-create event is marked deprecated upstream and its supported span-create/generation-create pair cannot express them. The same transport reaches Grafana Tempo, Honeycomb and Datadog's OTLP intake. Langfuse's REST ingestion API is still used for scores, which OTLP cannot express, and for the connection test. Telemetry never blocks a turn: record() is synchronous and try_send-based, dropping on a saturated queue and counting the drops rather than applying backpressure to the agent loop. Retries use exponential backoff with full jitter so instances that hit one rate limit do not retry in lockstep. Redaction runs before serialization on both key names and value shapes, since tool results are unstructured and a leaked credential rarely lands under a key called "password". Plaintext http is refused for non-loopback endpoints because these payloads carry content and a credential header. Config defaults to disabled, unlike every other Moltis feature flag: turning this on ships conversation content to a third party and should be a deliberate act. 135 tests cover the profile split, redaction, batching and backpressure, retry classification, and the OTLP wire encoding. --- Cargo.lock | 24 + Cargo.toml | 3 + crates/config/src/lib.rs | 17 +- crates/config/src/schema.rs | 8 +- crates/config/src/schema/instrumentation.rs | 419 ++++++++ crates/config/src/validate/schema_map.rs | 48 + crates/observability/Cargo.toml | 36 + crates/observability/src/builder.rs | 542 ++++++++++ .../observability/src/exporters/langfuse.rs | 522 +++++++++ crates/observability/src/exporters/mod.rs | 7 + .../src/exporters/otlp/attributes.rs | 106 ++ .../src/exporters/otlp/mapping.rs | 998 ++++++++++++++++++ .../observability/src/exporters/otlp/mod.rs | 386 +++++++ .../observability/src/exporters/otlp/wire.rs | 311 ++++++ crates/observability/src/lib.rs | 40 + crates/observability/src/model.rs | 590 +++++++++++ crates/observability/src/profile.rs | 213 ++++ crates/observability/src/recorder.rs | 754 +++++++++++++ crates/observability/src/redact.rs | 319 ++++++ crates/observability/src/runtime.rs | 606 +++++++++++ crates/observability/src/sink.rs | 311 ++++++ 21 files changed, 6250 insertions(+), 10 deletions(-) create mode 100644 crates/config/src/schema/instrumentation.rs create mode 100644 crates/observability/Cargo.toml create mode 100644 crates/observability/src/builder.rs create mode 100644 crates/observability/src/exporters/langfuse.rs create mode 100644 crates/observability/src/exporters/mod.rs create mode 100644 crates/observability/src/exporters/otlp/attributes.rs create mode 100644 crates/observability/src/exporters/otlp/mapping.rs create mode 100644 crates/observability/src/exporters/otlp/mod.rs create mode 100644 crates/observability/src/exporters/otlp/wire.rs create mode 100644 crates/observability/src/lib.rs create mode 100644 crates/observability/src/model.rs create mode 100644 crates/observability/src/profile.rs create mode 100644 crates/observability/src/recorder.rs create mode 100644 crates/observability/src/redact.rs create mode 100644 crates/observability/src/runtime.rs create mode 100644 crates/observability/src/sink.rs diff --git a/Cargo.lock b/Cargo.lock index 34c2b4736..00ab9f938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8029,6 +8029,30 @@ dependencies = [ "uuid", ] +[[package]] +name = "moltis-observability" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.22.1", + "futures", + "moltis-common", + "moltis-config", + "moltis-metrics", + "rand 0.10.1", + "reqwest 0.13.2", + "secrecy 0.8.0", + "serde", + "serde_json", + "thiserror 2.0.18", + "time", + "tokio", + "tracing", + "uuid", + "wiremock", +] + [[package]] name = "moltis-onboarding" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index aeb9e23b2..de289c1fa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ default-members = [ "crates/node-host", "crates/nostr", "crates/oauth", + "crates/observability", "crates/onboarding", "crates/openclaw-import", "crates/plugins", @@ -106,6 +107,7 @@ members = [ "crates/node-host", "crates/nostr", "crates/oauth", + "crates/observability", "crates/onboarding", "crates/openclaw-import", "crates/plugins", @@ -401,6 +403,7 @@ moltis-network-filter = { default-features = false, path = "crates/network-fil moltis-node-host = { path = "crates/node-host" } moltis-nostr = { path = "crates/nostr" } moltis-oauth = { path = "crates/oauth" } +moltis-observability = { path = "crates/observability" } moltis-onboarding = { path = "crates/onboarding" } moltis-openclaw-import = { path = "crates/openclaw-import" } moltis-plugins = { path = "crates/plugins" } diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index d161d8412..30dc0d9ca 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -56,15 +56,16 @@ pub use { AgentIdentity, AgentMemoryWriteMode, AgentPreset, AgentRuntimeLimitSource, AgentRuntimeLimits, AgentsConfig, AuthConfig, CacheRetention, CalDavAccountConfig, CalDavConfig, ChannelToolPolicyOverride, ChannelsConfig, ChatConfig, CodeIndexTomlConfig, - CompactionConfig, CompactionMode, GeoLocation, GroupToolPolicy, HeartbeatConfig, - HomeAssistantAccountConfig, HomeAssistantConfig, MemoryBackend, MemoryCitationsMode, + CompactionConfig, CompactionMode, ContentCaptureMode, DatadogSettings, GeoLocation, + GroupToolPolicy, HeartbeatConfig, HomeAssistantAccountConfig, HomeAssistantConfig, + InstrumentationConfig, LangfuseSettings, MemoryBackend, MemoryCitationsMode, MemoryProvider, MemoryScope, MemorySearchMergeStrategy, MemoryStyle, MessageQueueMode, - ModePreset, ModesConfig, MoltisConfig, NgrokConfig, PresetMemoryConfig, PresetToolPolicy, - PromptMemoryMode, ResolvedIdentity, SessionAccessPolicyConfig, SessionExportMode, Timezone, - ToolMode, ToolPolicyConfig, ToolRegistryMode, UserProfile, UserProfileWriteMode, - VoiceConfig, VoiceElevenLabsConfig, VoiceOpenAiConfig, VoiceSttConfig, VoiceSttProvider, - VoiceTtsConfig, VoiceTtsProvider, VoiceWhisperConfig, VoiceWhisperLocalConfig, WireApi, - parse_byte_size, + ModePreset, ModesConfig, MoltisConfig, NgrokConfig, OtlpSettings, PresetMemoryConfig, + PresetToolPolicy, PromptMemoryMode, ResolvedIdentity, SessionAccessPolicyConfig, + SessionExportMode, Timezone, ToolMode, ToolPolicyConfig, ToolRegistryMode, UserProfile, + UserProfileWriteMode, VoiceConfig, VoiceElevenLabsConfig, VoiceOpenAiConfig, + VoiceSttConfig, VoiceSttProvider, VoiceTtsConfig, VoiceTtsProvider, VoiceWhisperConfig, + VoiceWhisperLocalConfig, WireApi, parse_byte_size, }, validate::{Diagnostic, Severity, ValidationResult}, }; diff --git a/crates/config/src/schema.rs b/crates/config/src/schema.rs index eb4339d13..044907666 100644 --- a/crates/config/src/schema.rs +++ b/crates/config/src/schema.rs @@ -15,6 +15,8 @@ mod chat; mod code_index; #[path = "schema/hooks.rs"] mod hooks; + +mod instrumentation; #[path = "schema/memory.rs"] mod memory; #[path = "schema/modes.rs"] @@ -33,8 +35,8 @@ mod tools; mod voice; pub use { - agents::*, chat::*, code_index::*, hooks::*, memory::*, modes::*, phone::*, providers::*, - runtime::*, system::*, tools::*, voice::*, + agents::*, chat::*, code_index::*, hooks::*, instrumentation::*, memory::*, modes::*, phone::*, + providers::*, runtime::*, system::*, tools::*, voice::*, }; // ── Reasoning effort ────────────────────────────────────────────────────── @@ -305,6 +307,8 @@ pub struct MoltisConfig { pub auth: AuthConfig, pub graphql: GraphqlConfig, pub metrics: MetricsConfig, + /// Agent instrumentation exported to Langfuse, OTLP collectors and Datadog. + pub instrumentation: InstrumentationConfig, pub identity: AgentIdentity, pub user: UserProfile, pub hooks: Option, diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs new file mode 100644 index 000000000..f17151027 --- /dev/null +++ b/crates/config/src/schema/instrumentation.rs @@ -0,0 +1,419 @@ +//! Instrumentation (tracing/observability) configuration. +//! +//! One `[instrumentation]` section configures every backend, because they are +//! fed from a single instrumentation pass in the agent runtime. Each backend +//! has its own sub-table and its own content policy — see the `profile` module +//! in `moltis-observability` for why an LLM observability product and an +//! infrastructure APM must not receive the same payload. + +use { + secrecy::Secret, + serde::{Deserialize, Serialize}, +}; + +use crate::schema::{deserialize_option_secret, serialize_option_secret}; + +fn default_true() -> bool { + true +} + +fn default_sample_rate() -> f64 { + 1.0 +} + +fn default_flush_interval_ms() -> u64 { + 5_000 +} + +fn default_queue_capacity() -> usize { + 10_000 +} + +fn default_max_batch_bytes() -> usize { + 3_000_000 +} + +fn default_timeout_secs() -> u64 { + 10 +} + +fn default_langfuse_host() -> String { + "https://cloud.langfuse.com".to_string() +} + +/// How much conversation content a backend receives. +/// +/// Mirrors `moltis_observability::profile::ContentCapture`; kept as a separate +/// type so `moltis-config` does not depend on the observability crate, and +/// converted at the boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContentCaptureMode { + /// Send prompts, completions, tool arguments and tool results. + Full, + /// Send structural metadata only: sizes, counts, model, timings. + #[default] + MetadataOnly, + /// Send neither content nor content-derived metadata. + None, +} + +impl ContentCaptureMode { + /// Accepted values, for config validation messages. + pub const ALL: &'static [&'static str] = &["full", "metadata_only", "none"]; +} + +/// Top-level instrumentation configuration. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct InstrumentationConfig { + /// Master switch. Disabled by default: every backend here ships data off + /// the machine, so it must be an explicit choice rather than something a + /// user discovers after the fact. + #[serde(default)] + pub enabled: bool, + /// Deployment environment reported to every backend. + #[serde(default = "default_environment")] + pub environment: String, + /// Build release identifier. Defaults to the running Moltis version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub release: Option, + /// Fraction of turns to trace, in `0.0..=1.0`. + #[serde(default = "default_sample_rate")] + pub sample_rate: f64, + /// Extra object keys whose values are redacted before export. Merged with + /// the built-in list; it cannot be narrowed. + #[serde(default)] + pub redact: Vec, + /// Bounded export queue depth. Events are dropped once this fills rather + /// than blocking the agent loop. + #[serde(default = "default_queue_capacity")] + pub queue_capacity: usize, + /// Maximum time an event waits before being flushed. + #[serde(default = "default_flush_interval_ms")] + pub flush_interval_ms: u64, + /// Maximum estimated batch size in bytes before a forced flush. + #[serde(default = "default_max_batch_bytes")] + pub max_batch_bytes: usize, + /// Langfuse backend. + #[serde(default)] + pub langfuse: LangfuseSettings, + /// Generic OTLP backend (Grafana Tempo/Alloy, Honeycomb, a collector). + #[serde(default)] + pub otlp: OtlpSettings, + /// Datadog backend, via Datadog's OTLP intake. + #[serde(default)] + pub datadog: DatadogSettings, +} + +fn default_environment() -> String { + "production".to_string() +} + +impl Default for InstrumentationConfig { + fn default() -> Self { + Self { + enabled: false, + environment: default_environment(), + release: None, + sample_rate: default_sample_rate(), + redact: Vec::new(), + queue_capacity: default_queue_capacity(), + flush_interval_ms: default_flush_interval_ms(), + max_batch_bytes: default_max_batch_bytes(), + langfuse: LangfuseSettings::default(), + otlp: OtlpSettings::default(), + datadog: DatadogSettings::default(), + } + } +} + +impl InstrumentationConfig { + /// Whether any backend is switched on. + #[must_use] + pub fn any_backend_enabled(&self) -> bool { + self.enabled && (self.langfuse.enabled || self.otlp.enabled || self.datadog.enabled) + } +} + +/// Langfuse-specific settings. +/// +/// Langfuse is LLM-native: it wants the whole conversation, and its cost, +/// session and prompt-version views are built on that. Content capture is +/// therefore on by default here, unlike the APM backends. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct LangfuseSettings { + /// Whether to export to Langfuse. + #[serde(default)] + pub enabled: bool, + /// Base host. Set to a self-hosted URL to keep data on-premises. + #[serde(default = "default_langfuse_host")] + pub host: String, + /// Project public key. + #[serde(default)] + pub public_key: String, + /// Project secret key. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_option_secret", + deserialize_with = "deserialize_option_secret" + )] + pub secret_key: Option>, + /// Whether to send turn and step inputs. + #[serde(default = "default_true")] + pub capture_input: bool, + /// Whether to send turn and step outputs. + #[serde(default = "default_true")] + pub capture_output: bool, + /// Whether to send tool arguments and results. Separate from the input and + /// output switches because tool arguments are the likeliest place for + /// credentials to appear. + #[serde(default = "default_true")] + pub capture_tool_io: bool, + /// Per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +impl Default for LangfuseSettings { + fn default() -> Self { + Self { + enabled: false, + host: default_langfuse_host(), + public_key: String::new(), + secret_key: None, + capture_input: true, + capture_output: true, + capture_tool_io: true, + timeout_secs: default_timeout_secs(), + } + } +} + +/// Generic OTLP backend settings. +/// +/// Targets an OpenTelemetry collector, Grafana Alloy/Tempo, Honeycomb, or +/// anything else speaking OTLP/HTTP. Content capture defaults to +/// `metadata_only`: these are operational tools, and prompt bodies there mean +/// unbounded span size, cardinality pressure, per-byte ingest billing, and +/// conversation content sitting in a system nobody scoped for it. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct OtlpSettings { + /// Whether to export over OTLP. + #[serde(default)] + pub enabled: bool, + /// Full traces endpoint, e.g. `http://localhost:4318/v1/traces`. + #[serde(default)] + pub endpoint: String, + /// Extra headers, typically for authentication. + #[serde(default)] + pub headers: std::collections::BTreeMap, + /// How much conversation content this backend receives. + #[serde(default)] + pub content: ContentCaptureMode, + /// Whether to attach end-user identity. Off by default: high-cardinality + /// in an APM's index and often a compliance question. + #[serde(default)] + pub emit_user_id: bool, + /// Per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +impl Default for OtlpSettings { + fn default() -> Self { + Self { + enabled: false, + endpoint: String::new(), + headers: std::collections::BTreeMap::new(), + content: ContentCaptureMode::MetadataOnly, + emit_user_id: false, + timeout_secs: default_timeout_secs(), + } + } +} + +/// Datadog backend settings. +/// +/// Datadog ingests OTLP either through the Datadog Agent (default +/// `http://localhost:4318/v1/traces`) or its direct intake. Tags are dropped +/// for this backend because Datadog indexes span tags and bills on custom +/// metric cardinality. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct DatadogSettings { + /// Whether to export to Datadog. + #[serde(default)] + pub enabled: bool, + /// OTLP endpoint exposed by the Datadog Agent or intake. + #[serde(default = "default_datadog_endpoint")] + pub endpoint: String, + /// Datadog API key, required only when posting to the intake directly + /// rather than through a local Agent. + #[serde( + default, + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_option_secret", + deserialize_with = "deserialize_option_secret" + )] + pub api_key: Option>, + /// Datadog service name. + #[serde(default = "default_datadog_service")] + pub service: String, + /// How much conversation content this backend receives. + #[serde(default)] + pub content: ContentCaptureMode, + /// Per-request timeout in seconds. + #[serde(default = "default_timeout_secs")] + pub timeout_secs: u64, +} + +fn default_datadog_endpoint() -> String { + "http://localhost:4318/v1/traces".to_string() +} + +fn default_datadog_service() -> String { + "moltis".to_string() +} + +impl Default for DatadogSettings { + fn default() -> Self { + Self { + enabled: false, + endpoint: default_datadog_endpoint(), + api_key: None, + service: default_datadog_service(), + content: ContentCaptureMode::MetadataOnly, + timeout_secs: default_timeout_secs(), + } + } +} + +#[cfg(test)] +mod tests { + use secrecy::ExposeSecret; + + use super::*; + + #[test] + fn instrumentation_is_off_by_default() { + // Unlike every other Moltis feature flag: enabling this ships + // conversation content to a third party. + let config = InstrumentationConfig::default(); + assert!(!config.enabled); + assert!(!config.langfuse.enabled); + assert!(!config.otlp.enabled); + assert!(!config.datadog.enabled); + assert!(!config.any_backend_enabled()); + } + + #[test] + fn master_switch_gates_every_backend() { + let mut config = InstrumentationConfig { + enabled: false, + ..Default::default() + }; + config.langfuse.enabled = true; + + assert!( + !config.any_backend_enabled(), + "master switch must override per-backend flags" + ); + + config.enabled = true; + assert!(config.any_backend_enabled()); + } + + #[test] + fn langfuse_captures_content_by_default_but_apms_do_not() { + // The asymmetry is the point: Langfuse is useless without content, + // an APM is actively harmed by it. + assert!(LangfuseSettings::default().capture_input); + assert!(LangfuseSettings::default().capture_tool_io); + assert_eq!( + OtlpSettings::default().content, + ContentCaptureMode::MetadataOnly + ); + assert_eq!( + DatadogSettings::default().content, + ContentCaptureMode::MetadataOnly + ); + } + + #[test] + fn apm_backends_do_not_emit_user_id_by_default() { + assert!(!OtlpSettings::default().emit_user_id); + } + + #[test] + fn secret_key_round_trips_through_toml() { + let toml = r#" + enabled = true + host = "https://self-hosted.example.com" + public_key = "pk-lf-1" + secret_key = "sk-lf-1" + "#; + let parsed: LangfuseSettings = toml::from_str(toml).expect("valid toml"); + + assert_eq!(parsed.host, "https://self-hosted.example.com"); + assert_eq!( + parsed.secret_key.as_ref().map(ExposeSecret::expose_secret), + Some(&"sk-lf-1".to_string()) + ); + } + + #[test] + fn debug_output_does_not_leak_the_secret_key() { + let settings = LangfuseSettings { + secret_key: Some(Secret::new("sk-lf-supersecret".to_string())), + ..Default::default() + }; + let rendered = format!("{settings:?}"); + + assert!( + !rendered.contains("sk-lf-supersecret"), + "secret leaked into Debug: {rendered}" + ); + } + + #[test] + fn datadog_defaults_to_the_local_agent_endpoint() { + // The overwhelmingly common deployment: Agent on localhost, no API key + // needed in Moltis at all. + let settings = DatadogSettings::default(); + assert_eq!(settings.endpoint, "http://localhost:4318/v1/traces"); + assert!(settings.api_key.is_none()); + } + + #[test] + fn content_mode_parses_every_documented_value() { + for value in ContentCaptureMode::ALL { + let toml = format!("content = \"{value}\""); + toml::from_str::(&toml) + .unwrap_or_else(|e| panic!("{value} should parse: {e}")); + } + } + + #[test] + fn partial_config_fills_in_defaults() { + // Operators write a two-line section; everything else must default. + let config: InstrumentationConfig = toml::from_str( + r#" + enabled = true + [langfuse] + enabled = true + public_key = "pk" + "#, + ) + .expect("valid toml"); + + assert!(config.enabled); + assert_eq!(config.sample_rate, 1.0); + assert_eq!(config.environment, "production"); + assert_eq!(config.langfuse.host, default_langfuse_host()); + assert_eq!(config.queue_capacity, 10_000); + } +} diff --git a/crates/config/src/validate/schema_map.rs b/crates/config/src/validate/schema_map.rs index 6416856ea..828c42cc3 100644 --- a/crates/config/src/validate/schema_map.rs +++ b/crates/config/src/validate/schema_map.rs @@ -509,6 +509,54 @@ pub(super) fn build_schema_map() -> KnownKeys { ("labels", Map(Box::new(Leaf))), ])), ), + ( + "instrumentation", + Struct(HashMap::from([ + ("enabled", Leaf), + ("environment", Leaf), + ("release", Leaf), + ("sample_rate", Leaf), + ("redact", Leaf), + ("queue_capacity", Leaf), + ("flush_interval_ms", Leaf), + ("max_batch_bytes", Leaf), + ( + "langfuse", + Struct(HashMap::from([ + ("enabled", Leaf), + ("host", Leaf), + ("public_key", Leaf), + ("secret_key", Leaf), + ("capture_input", Leaf), + ("capture_output", Leaf), + ("capture_tool_io", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ( + "otlp", + Struct(HashMap::from([ + ("enabled", Leaf), + ("endpoint", Leaf), + ("headers", Map(Box::new(Leaf))), + ("content", Leaf), + ("emit_user_id", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ( + "datadog", + Struct(HashMap::from([ + ("enabled", Leaf), + ("endpoint", Leaf), + ("api_key", Leaf), + ("service", Leaf), + ("content", Leaf), + ("timeout_secs", Leaf), + ])), + ), + ])), + ), ( "identity", Struct(HashMap::from([ diff --git a/crates/observability/Cargo.toml b/crates/observability/Cargo.toml new file mode 100644 index 000000000..9981b5752 --- /dev/null +++ b/crates/observability/Cargo.toml @@ -0,0 +1,36 @@ +[package] +edition.workspace = true +name = "moltis-observability" +version.workspace = true + +[features] +default = ["langfuse", "otlp"] +langfuse = [] +metrics = ["dep:moltis-metrics"] +otlp = [] + +[dependencies] +anyhow = { workspace = true } +async-trait = { workspace = true } +base64 = { workspace = true } +futures = { workspace = true } +moltis-common = { workspace = true } +moltis-config = { workspace = true } +moltis-metrics = { optional = true, workspace = true } +rand = { workspace = true } +reqwest = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +time = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +uuid = { workspace = true } + +[dev-dependencies] +tokio = { features = ["full", "test-util"], workspace = true } +wiremock = { workspace = true } + +[lints] +workspace = true diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs new file mode 100644 index 000000000..503d35401 --- /dev/null +++ b/crates/observability/src/builder.rs @@ -0,0 +1,542 @@ +//! Construction of live sinks from `[instrumentation]` configuration. + +use std::{sync::Arc, time::Duration}; + +use { + moltis_config::{ + ContentCaptureMode, DatadogSettings, InstrumentationConfig, LangfuseSettings, OtlpSettings, + }, + secrecy::ExposeSecret, + tracing::{info, warn}, +}; + +use crate::{ + exporters::{ + langfuse::{LangfuseClient, LangfuseConfig}, + otlp::{OtlpConfig, OtlpTransport}, + }, + profile::{ContentCapture, ExportProfile}, + recorder::RecorderSettings, + redact::RedactionPolicy, + runtime::{BatchConfig, BatchSink}, + sink::{ObservationSink, SinkFanout}, +}; + +/// Convert the config-level content mode to the profile-level one. +impl From for ContentCapture { + fn from(mode: ContentCaptureMode) -> Self { + match mode { + ContentCaptureMode::Full => Self::Full, + ContentCaptureMode::MetadataOnly => Self::MetadataOnly, + ContentCaptureMode::None => Self::None, + } + } +} + +/// The result of building instrumentation from config. +pub struct BuiltInstrumentation { + /// Fanout over every enabled backend. + pub sink: Arc, + /// Settings the agent runtime applies when recording. + pub recorder_settings: RecorderSettings, + /// Langfuse REST client, present when the Langfuse backend is enabled. + /// Used for scores and the settings UI connection test. + pub langfuse: Option>, + /// Names of the backends that were built. + pub backends: Vec, +} + +/// Reasons a backend was skipped, surfaced in the settings UI rather than only +/// in logs — a silently disabled exporter is indistinguishable from a broken one. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct SkippedBackend { + /// Backend name. + pub name: String, + /// Why it was not built. + pub reason: String, +} + +/// Outcome of a build attempt. +pub struct BuildOutcome { + /// Built instrumentation, or `None` when nothing was enabled. + pub built: Option, + /// Backends that were enabled but could not be built. + pub skipped: Vec, +} + +/// Batching parameters shared by every backend. +fn batch_config(config: &InstrumentationConfig) -> BatchConfig { + BatchConfig { + max_batch_bytes: config.max_batch_bytes, + flush_interval: Duration::from_millis(config.flush_interval_ms), + queue_capacity: config.queue_capacity, + ..Default::default() + } +} + +/// Validate a backend endpoint. +/// +/// Plaintext HTTP is refused for non-loopback hosts: these payloads carry +/// conversation content and credentials in a header, and shipping them +/// unencrypted across a network is not a trade-off an operator should make by +/// typing a URL. Loopback is allowed for a local Agent or collector. +fn validate_endpoint(raw: &str) -> Result<(), String> { + let Ok(url) = reqwest::Url::parse(raw) else { + return Err(format!("`{raw}` is not a valid URL")); + }; + match url.scheme() { + "https" => Ok(()), + "http" => { + let host = url.host_str().unwrap_or_default(); + let is_loopback = host == "localhost" + || host == "::1" + || host + .parse::() + .is_ok_and(|ip| ip.is_loopback()); + if is_loopback { + Ok(()) + } else { + Err(format!( + "refusing plaintext http for non-loopback host `{host}`; use https" + )) + } + }, + other => Err(format!("unsupported URL scheme `{other}`")), + } +} + +/// Build the Langfuse backend. +fn build_langfuse( + settings: &LangfuseSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result<(Arc, Arc), String> { + if settings.public_key.trim().is_empty() { + return Err("public_key is not set".to_string()); + } + let Some(secret) = settings.secret_key.as_ref() else { + return Err("secret_key is not set".to_string()); + }; + if secret.expose_secret().trim().is_empty() { + return Err("secret_key is empty".to_string()); + } + validate_endpoint(&settings.host)?; + + let langfuse_config = LangfuseConfig { + host: settings.host.clone(), + public_key: settings.public_key.clone(), + secret_key: secrecy::SecretString::new(secret.expose_secret().clone()), + environment: Some(config.environment.clone()), + release: Some(release.to_string()), + timeout: Duration::from_secs(settings.timeout_secs), + }; + + let transport = Arc::new(langfuse_config.build_transport(release.to_string())); + let sink = Arc::new(BatchSink::spawn(transport, batch_config(config))); + let client = Arc::new(LangfuseClient::new(langfuse_config)); + Ok((sink, client)) +} + +/// Build a generic OTLP backend. +fn build_otlp( + settings: &OtlpSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result, String> { + if settings.endpoint.trim().is_empty() { + return Err("endpoint is not set".to_string()); + } + validate_endpoint(&settings.endpoint)?; + + let profile = ExportProfile { + content: settings.content.into(), + emit_user_id: settings.emit_user_id, + ..ExportProfile::otel_generic() + }; + + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "otlp".to_string(), + endpoint: settings.endpoint.clone(), + headers: settings.headers.clone(), + timeout: Duration::from_secs(settings.timeout_secs), + service_name: "moltis".to_string(), + service_version: release.to_string(), + environment: Some(config.environment.clone()), + profile, + })); + Ok(Arc::new(BatchSink::spawn(transport, batch_config(config)))) +} + +/// Build the Datadog backend. +fn build_datadog( + settings: &DatadogSettings, + config: &InstrumentationConfig, + release: &str, +) -> Result, String> { + if settings.endpoint.trim().is_empty() { + return Err("endpoint is not set".to_string()); + } + validate_endpoint(&settings.endpoint)?; + + let mut headers = std::collections::BTreeMap::new(); + if let Some(key) = &settings.api_key { + headers.insert("DD-API-KEY".to_string(), key.expose_secret().clone()); + } + + let profile = ExportProfile { + content: settings.content.into(), + ..ExportProfile::datadog() + }; + + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "datadog".to_string(), + endpoint: settings.endpoint.clone(), + headers, + timeout: Duration::from_secs(settings.timeout_secs), + service_name: settings.service.clone(), + service_version: release.to_string(), + environment: Some(config.environment.clone()), + profile, + })); + Ok(Arc::new(BatchSink::spawn(transport, batch_config(config)))) +} + +/// Build every enabled backend. +/// +/// A backend that cannot be built is reported in `skipped` and the others still +/// start: one misconfigured exporter must not take instrumentation down. +/// +/// Must be called from within a Tokio runtime — each backend spawns a +/// background export task. +#[must_use] +pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { + let mut skipped = Vec::new(); + + if !config.enabled { + return BuildOutcome { + built: None, + skipped, + }; + } + + let mut sinks: Vec> = Vec::new(); + let mut backends = Vec::new(); + let mut langfuse_client = None; + + if config.langfuse.enabled { + match build_langfuse(&config.langfuse, config, release) { + Ok((sink, client)) => { + sinks.push(sink); + langfuse_client = Some(client); + backends.push("langfuse".to_string()); + }, + Err(reason) => { + warn!(backend = "langfuse", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "langfuse".to_string(), + reason, + }); + }, + } + } + + if config.otlp.enabled { + match build_otlp(&config.otlp, config, release) { + Ok(sink) => { + sinks.push(sink); + backends.push("otlp".to_string()); + }, + Err(reason) => { + warn!(backend = "otlp", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "otlp".to_string(), + reason, + }); + }, + } + } + + if config.datadog.enabled { + match build_datadog(&config.datadog, config, release) { + Ok(sink) => { + sinks.push(sink); + backends.push("datadog".to_string()); + }, + Err(reason) => { + warn!(backend = "datadog", %reason, "instrumentation backend disabled"); + skipped.push(SkippedBackend { + name: "datadog".to_string(), + reason, + }); + }, + } + } + + if sinks.is_empty() { + return BuildOutcome { + built: None, + skipped, + }; + } + + info!(backends = ?backends, "instrumentation enabled"); + + // Capture switches live on the Langfuse settings: it is the only backend + // that receives bodies at all, and the APM profiles gate content + // independently through their own `content` mode. + let recorder_settings = RecorderSettings { + redaction: RedactionPolicy::from_needles(&config.redact), + capture_input: config.langfuse.capture_input, + capture_output: config.langfuse.capture_output, + capture_tool_io: config.langfuse.capture_tool_io, + sample_rate: config.sample_rate.clamp(0.0, 1.0), + }; + + BuildOutcome { + built: Some(BuiltInstrumentation { + sink: Arc::new(SinkFanout::new(sinks)), + recorder_settings, + langfuse: langfuse_client, + backends, + }), + skipped, + } +} + +#[cfg(test)] +mod tests { + use secrecy::Secret; + + use super::*; + + fn enabled_config() -> InstrumentationConfig { + InstrumentationConfig { + enabled: true, + ..Default::default() + } + } + + fn valid_langfuse() -> LangfuseSettings { + LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + } + } + + #[tokio::test] + async fn disabled_config_builds_nothing() { + let outcome = build(&InstrumentationConfig::default(), "test"); + assert!(outcome.built.is_none()); + assert!(outcome.skipped.is_empty()); + } + + #[tokio::test] + async fn master_switch_off_skips_enabled_backends_silently() { + let config = InstrumentationConfig { + enabled: false, + langfuse: valid_langfuse(), + ..Default::default() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + // Not "skipped": the operator turned the whole thing off deliberately. + assert!(outcome.skipped.is_empty()); + } + + #[tokio::test] + async fn langfuse_builds_with_valid_credentials() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + ..enabled_config() + }; + let outcome = build(&config, "20260726.01"); + let built = outcome.built.expect("langfuse should build"); + + assert_eq!(built.backends, vec!["langfuse"]); + assert!(built.langfuse.is_some(), "REST client needed for scores"); + } + + #[tokio::test] + async fn langfuse_without_credentials_is_skipped_with_a_reason() { + let config = InstrumentationConfig { + langfuse: LangfuseSettings { + enabled: true, + public_key: String::new(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert!(outcome.skipped[0].reason.contains("public_key")); + } + + #[tokio::test] + async fn one_broken_backend_does_not_disable_the_others() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + otlp: OtlpSettings { + enabled: true, + endpoint: String::new(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + let built = outcome.built.expect("langfuse should still build"); + + assert_eq!(built.backends, vec!["langfuse"]); + assert_eq!(outcome.skipped.len(), 1); + assert_eq!(outcome.skipped[0].name, "otlp"); + } + + #[tokio::test] + async fn multiple_backends_fan_out_together() { + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + otlp: OtlpSettings { + enabled: true, + endpoint: "http://localhost:4318/v1/traces".into(), + ..Default::default() + }, + datadog: DatadogSettings { + enabled: true, + ..Default::default() + }, + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert_eq!(built.backends, vec!["langfuse", "otlp", "datadog"]); + assert_eq!(built.sink.name(), "langfuse+otlp+datadog"); + } + + #[tokio::test] + async fn plaintext_http_is_refused_for_remote_hosts() { + // Conversation content plus a credential header over the open network. + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "http://collector.example.com/v1/traces".into(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert!(outcome.skipped[0].reason.contains("plaintext http")); + } + + #[tokio::test] + async fn plaintext_http_is_allowed_for_loopback() { + // The common Datadog Agent and local-collector deployment. + for endpoint in [ + "http://localhost:4318/v1/traces", + "http://127.0.0.1:4318/v1/traces", + ] { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: endpoint.to_string(), + ..Default::default() + }, + ..enabled_config() + }; + assert!( + build(&config, "test").built.is_some(), + "{endpoint} should be accepted" + ); + } + } + + #[tokio::test] + async fn malformed_endpoints_are_rejected() { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "not a url".into(), + ..Default::default() + }, + ..enabled_config() + }; + let outcome = build(&config, "test"); + assert!(outcome.skipped[0].reason.contains("not a valid URL")); + } + + #[tokio::test] + async fn sample_rate_is_clamped_into_range() { + let config = InstrumentationConfig { + sample_rate: 7.5, + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + assert_eq!(built.recorder_settings.sample_rate, 1.0); + } + + #[tokio::test] + async fn capture_switches_flow_through_to_the_recorder() { + let config = InstrumentationConfig { + langfuse: LangfuseSettings { + capture_tool_io: false, + ..valid_langfuse() + }, + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert!(!built.recorder_settings.capture_tool_io); + assert!(built.recorder_settings.capture_input); + } + + #[tokio::test] + async fn configured_redaction_needles_extend_the_defaults() { + let config = InstrumentationConfig { + redact: vec!["customer_ref".into()], + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + let policy = &built.recorder_settings.redaction; + + assert!(policy.is_sensitive_key("customer_ref")); + assert!(policy.is_sensitive_key("password")); + } + + #[tokio::test] + async fn datadog_api_key_is_sent_as_a_header_not_a_query_param() { + let config = InstrumentationConfig { + datadog: DatadogSettings { + enabled: true, + endpoint: "https://otlp.datadoghq.com/v1/traces".into(), + api_key: Some(Secret::new("dd-key".to_string())), + ..Default::default() + }, + ..enabled_config() + }; + assert!(build(&config, "test").built.is_some()); + } + + #[test] + fn content_modes_convert_to_the_matching_profile_policy() { + assert_eq!( + ContentCapture::from(ContentCaptureMode::Full), + ContentCapture::Full + ); + assert_eq!( + ContentCapture::from(ContentCaptureMode::MetadataOnly), + ContentCapture::MetadataOnly + ); + assert_eq!( + ContentCapture::from(ContentCaptureMode::None), + ContentCapture::None + ); + } +} diff --git a/crates/observability/src/exporters/langfuse.rs b/crates/observability/src/exporters/langfuse.rs new file mode 100644 index 000000000..5d279bc2c --- /dev/null +++ b/crates/observability/src/exporters/langfuse.rs @@ -0,0 +1,522 @@ +//! Langfuse backend. +//! +//! Traces reach Langfuse over OTLP — that is Langfuse's own modern ingest path, +//! the one its v3+ Python and v4+ JS SDKs use, and the only one that carries the +//! full observation taxonomy (`AGENT`, `TOOL`, `RETRIEVER`, ...). The native +//! `/api/public/ingestion` event `observation-create` is marked deprecated +//! upstream and its non-deprecated `span-create`/`generation-create` pair +//! cannot express the newer types at all. +//! +//! What OTLP *cannot* express is scores, so those go over the ingestion API. +//! This module owns both, plus the credential handling and the connection test +//! behind the settings UI's "Test connection" button. + +use std::{collections::BTreeMap, time::Duration}; + +use { + base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}, + secrecy::{ExposeSecret, SecretString}, + serde::Serialize, + tracing::debug, +}; + +use { + super::otlp::{OtlpConfig, OtlpTransport}, + crate::{ + model::{ScoreRecord, ScoreValue}, + profile::ExportProfile, + }, +}; + +/// Path Langfuse serves OTLP traces on, appended to the configured host. +pub const OTEL_TRACES_PATH: &str = "/api/public/otel/v1/traces"; +/// Batch ingestion path, used for scores. +pub const INGESTION_PATH: &str = "/api/public/ingestion"; +/// Unauthenticated liveness probe. +pub const HEALTH_PATH: &str = "/api/public/health"; + +/// Langfuse connection settings. +#[derive(Clone)] +pub struct LangfuseConfig { + /// Base host, e.g. `https://cloud.langfuse.com` or a self-hosted URL. + pub host: String, + /// Project public key (Basic auth username). + pub public_key: String, + /// Project secret key (Basic auth password). + pub secret_key: SecretString, + /// Deployment environment. + pub environment: Option, + /// Build release identifier. + pub release: Option, + /// Per-request timeout. + pub timeout: Duration, +} + +// Manual impl: the derived one would print the secret key. +impl std::fmt::Debug for LangfuseConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LangfuseConfig") + .field("host", &self.host) + .field("public_key", &self.public_key) + .field("secret_key", &"[REDACTED]") + .field("environment", &self.environment) + .field("release", &self.release) + .field("timeout", &self.timeout) + .finish() + } +} + +impl LangfuseConfig { + /// Join `path` onto the configured host, tolerating a trailing slash. + #[must_use] + pub fn url(&self, path: &str) -> String { + format!("{}{path}", self.host.trim_end_matches('/')) + } + + /// `Authorization` header value for Basic auth. + /// + /// Langfuse authenticates with the public key as username and the secret + /// key as password. + #[must_use] + pub fn basic_auth_header(&self) -> String { + let raw = format!("{}:{}", self.public_key, self.secret_key.expose_secret()); + format!("Basic {}", BASE64.encode(raw)) + } + + /// Headers every authenticated Langfuse request carries. + #[must_use] + pub fn auth_headers(&self) -> BTreeMap { + let mut headers = BTreeMap::new(); + headers.insert("Authorization".to_string(), self.basic_auth_header()); + // Langfuse surfaces these in its own diagnostics. + headers.insert("X-Langfuse-Sdk-Name".to_string(), "moltis".to_string()); + headers.insert( + "X-Langfuse-Sdk-Version".to_string(), + env!("CARGO_PKG_VERSION").to_string(), + ); + headers.insert("X-Langfuse-Public-Key".to_string(), self.public_key.clone()); + headers + } + + /// Build the OTLP transport that carries traces to Langfuse. + /// + /// The profile is fixed to [`ExportProfile::langfuse`]: content capture is + /// the entire point of sending to Langfuse, and letting it be configured + /// down to `MetadataOnly` would produce traces with no conversation in + /// them, which is worse than not exporting at all. + #[must_use] + pub fn build_transport(&self, service_version: String) -> OtlpTransport { + OtlpTransport::new(OtlpConfig { + name: "langfuse".to_string(), + endpoint: self.url(OTEL_TRACES_PATH), + headers: self.auth_headers(), + timeout: self.timeout, + service_name: "moltis".to_string(), + service_version, + environment: self.environment.clone(), + profile: ExportProfile::langfuse(), + }) + } +} + +// ── Scores ────────────────────────────────────────────────────────────────── + +/// A `score-create` event in the ingestion batch envelope. +#[derive(Debug, Serialize)] +struct IngestionEvent<'a> { + id: String, + timestamp: String, + #[serde(rename = "type")] + kind: &'static str, + body: ScoreBody<'a>, +} + +/// Langfuse `ScoreBody`. +#[derive(Debug, Serialize)] +struct ScoreBody<'a> { + id: &'a str, + #[serde(rename = "traceId")] + trace_id: &'a str, + #[serde(rename = "observationId", skip_serializing_if = "Option::is_none")] + observation_id: Option<&'a str>, + name: &'a str, + value: &'a ScoreValue, + #[serde(rename = "dataType")] + data_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + comment: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + environment: Option<&'a str>, +} + +/// The ingestion request envelope. +#[derive(Debug, Serialize)] +struct IngestionRequest<'a> { + batch: Vec>, +} + +/// Client for the Langfuse REST surfaces that OTLP cannot express. +pub struct LangfuseClient { + config: LangfuseConfig, + http: reqwest::Client, +} + +impl LangfuseClient { + /// Build a client over the workspace HTTP client, so the configured + /// upstream proxy applies. + #[must_use] + pub fn new(config: LangfuseConfig) -> Self { + Self { + config, + http: moltis_common::http_client::build_default_http_client(), + } + } + + /// Verify that the host is reachable and the credentials are accepted. + /// + /// Backs the settings UI's "Test connection" button. The health endpoint is + /// unauthenticated, so credentials are checked with an empty ingestion + /// batch: it is the cheapest authenticated call that creates no data. + pub async fn test_connection(&self) -> anyhow::Result<()> { + let health = self + .http + .get(self.config.url(HEALTH_PATH)) + .timeout(self.config.timeout) + .send() + .await + .map_err(|e| anyhow::anyhow!("cannot reach Langfuse at {}: {e}", self.config.host))?; + + if !health.status().is_success() { + return Err(anyhow::anyhow!( + "Langfuse health check failed with HTTP {}", + health.status() + )); + } + + let auth = self + .http + .post(self.config.url(INGESTION_PATH)) + .headers(self.header_map()) + .timeout(self.config.timeout) + .json(&serde_json::json!({ "batch": [] })) + .send() + .await + .map_err(|e| anyhow::anyhow!("Langfuse credential check failed: {e}"))?; + + let status = auth.status(); + match status { + s if s.is_success() => Ok(()), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN => Err( + anyhow::anyhow!("Langfuse rejected the credentials (HTTP {status})"), + ), + other => Err(anyhow::anyhow!( + "unexpected response from Langfuse: HTTP {other}" + )), + } + } + + /// Submit scores. Langfuse upserts on score id. + pub async fn submit_scores(&self, scores: &[ScoreRecord]) -> anyhow::Result<()> { + if scores.is_empty() { + return Ok(()); + } + + let now = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(); + + let batch: Vec> = scores + .iter() + .map(|score| IngestionEvent { + id: uuid::Uuid::new_v4().to_string(), + timestamp: now.clone(), + kind: "score-create", + body: ScoreBody { + id: &score.id, + trace_id: &score.trace_id.0, + observation_id: score.observation_id.as_ref().map(|o| o.0.as_str()), + name: &score.name, + value: &score.value, + data_type: match score.value { + ScoreValue::Numeric(_) => "NUMERIC", + ScoreValue::Categorical(_) => "CATEGORICAL", + }, + comment: score.comment.as_deref(), + environment: score + .environment + .as_deref() + .or(self.config.environment.as_deref()), + }, + }) + .collect(); + + let response = self + .http + .post(self.config.url(INGESTION_PATH)) + .headers(self.header_map()) + .timeout(self.config.timeout) + .json(&IngestionRequest { batch }) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "Langfuse rejected scores: HTTP {status}: {}", + body.chars().take(512).collect::() + )); + } + + // The ingestion endpoint reports per-event outcomes in a 207, so a 2xx + // alone does not mean every score landed. + let body: serde_json::Value = response.json().await.unwrap_or_default(); + if let Some(errors) = body.get("errors").and_then(|e| e.as_array()) + && !errors.is_empty() + { + return Err(anyhow::anyhow!( + "Langfuse rejected {} of {} scores: {}", + errors.len(), + scores.len(), + serde_json::to_string(errors).unwrap_or_default() + )); + } + + debug!(count = scores.len(), "submitted scores to Langfuse"); + Ok(()) + } + + /// Authenticated header map. + fn header_map(&self) -> reqwest::header::HeaderMap { + let mut map = reqwest::header::HeaderMap::new(); + for (key, value) in self.config.auth_headers() { + let Ok(name) = key.parse::() else { + continue; + }; + let Ok(mut val) = reqwest::header::HeaderValue::from_str(&value) else { + continue; + }; + val.set_sensitive(true); + map.insert(name, val); + } + map + } +} + +#[cfg(test)] +mod tests { + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{body_json_schema, header, method, path}, + }; + + use { + super::*, + crate::model::{ObservationId, TraceId}, + }; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + #[test] + fn basic_auth_encodes_public_and_secret_key() { + let header = config("https://example.com".into()).basic_auth_header(); + let encoded = header.strip_prefix("Basic ").expect("basic scheme"); + let decoded = + String::from_utf8(BASE64.decode(encoded).expect("valid base64")).expect("valid utf-8"); + + assert_eq!(decoded, "pk-lf-test:sk-lf-secret"); + } + + #[test] + fn debug_never_prints_the_secret_key() { + let rendered = format!("{:?}", config("https://example.com".into())); + + assert!( + !rendered.contains("sk-lf-secret"), + "secret leaked: {rendered}" + ); + assert!(rendered.contains("[REDACTED]")); + // The public key is not a secret and is useful in diagnostics. + assert!(rendered.contains("pk-lf-test")); + } + + #[test] + fn url_join_tolerates_a_trailing_slash_on_the_host() { + let with = config("https://example.com/".into()); + let without = config("https://example.com".into()); + + assert_eq!(with.url(OTEL_TRACES_PATH), without.url(OTEL_TRACES_PATH)); + assert_eq!( + without.url(OTEL_TRACES_PATH), + "https://example.com/api/public/otel/v1/traces" + ); + } + + #[test] + fn transport_targets_the_otel_endpoint_with_the_langfuse_profile() { + let transport = config("https://cloud.langfuse.com".into()).build_transport("1.0".into()); + + assert_eq!( + transport.endpoint(), + "https://cloud.langfuse.com/api/public/otel/v1/traces" + ); + } + + #[tokio::test] + async fn test_connection_accepts_a_healthy_authenticated_host() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .and(header( + "authorization", + "Basic cGstbGYtdGVzdDpzay1sZi1zZWNyZXQ=", + )) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], "errors": [] + }))) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect("healthy host should pass"); + } + + #[tokio::test] + async fn test_connection_reports_bad_credentials_distinctly() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect_err("401 should fail"); + + // The operator needs to know it is the keys, not the host. + assert!( + error.to_string().contains("rejected the credentials"), + "unhelpful message: {error}" + ); + } + + #[tokio::test] + async fn test_connection_reports_an_unreachable_host_distinctly() { + let error = LangfuseClient::new(config("http://127.0.0.1:1".into())) + .test_connection() + .await + .expect_err("connection refused should fail"); + + assert!( + error.to_string().contains("cannot reach Langfuse"), + "unhelpful message: {error}" + ); + } + + #[tokio::test] + async fn scores_post_as_score_create_events() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .and(body_json_schema::) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [{ "id": "1", "status": 201 }], "errors": [] + }))) + .expect(1) + .mount(&server) + .await; + + let mut score = ScoreRecord::new( + TraceId("trace-1".into()), + "user-feedback", + ScoreValue::Numeric(1.0), + ); + score.observation_id = Some(ObservationId("obs-1".into())); + score.comment = Some("helpful".into()); + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) + .await + .expect("score submission should succeed"); + } + + #[tokio::test] + async fn partial_rejection_in_a_207_is_surfaced_as_an_error() { + let server = MockServer::start().await; + // A 2xx alone does not mean the data landed: the ingestion endpoint + // reports per-event outcomes in the body. + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], + "errors": [{ "id": "1", "status": 400, "message": "invalid value" }] + }))) + .mount(&server) + .await; + + let score = ScoreRecord::new( + TraceId("trace-1".into()), + "user-feedback", + ScoreValue::Numeric(1.0), + ); + let error = LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) + .await + .expect_err("partial failure should surface"); + + assert!(error.to_string().contains("rejected 1 of 1"), "{error}"); + } + + #[tokio::test] + async fn empty_score_batches_skip_the_request() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[]) + .await + .expect("empty batch is a no-op"); + } + + #[test] + fn categorical_and_numeric_scores_declare_their_data_type() { + let numeric = ScoreValue::Numeric(0.5); + let categorical = ScoreValue::Categorical("helpful".into()); + + assert_eq!( + serde_json::to_value(&numeric).expect("serializable"), + serde_json::json!(0.5) + ); + assert_eq!( + serde_json::to_value(&categorical).expect("serializable"), + serde_json::json!("helpful") + ); + } +} diff --git a/crates/observability/src/exporters/mod.rs b/crates/observability/src/exporters/mod.rs new file mode 100644 index 000000000..a34e48cf7 --- /dev/null +++ b/crates/observability/src/exporters/mod.rs @@ -0,0 +1,7 @@ +//! Backend exporters. + +#[cfg(feature = "langfuse")] +pub mod langfuse; + +#[cfg(feature = "otlp")] +pub mod otlp; diff --git a/crates/observability/src/exporters/otlp/attributes.rs b/crates/observability/src/exporters/otlp/attributes.rs new file mode 100644 index 000000000..e0b1ab172 --- /dev/null +++ b/crates/observability/src/exporters/otlp/attributes.rs @@ -0,0 +1,106 @@ +//! Attribute names emitted on every exported span. +//! +//! Two vocabularies are written side by side: +//! +//! * `gen_ai.*` — the OpenTelemetry GenAI semantic conventions, understood by +//! Grafana, Datadog, Honeycomb and anything else that speaks OTel. +//! * `langfuse.*` — Langfuse's own mapping, which carries the richer +//! observation taxonomy (AGENT/TOOL/RETRIEVER/...), usage and cost detail, +//! and managed-prompt linkage that the GenAI conventions have no slot for. +//! +//! Emitting both costs a few hundred bytes per span and means a single +//! instrumentation pass serves every backend. Unknown attributes are ignored by +//! conformant backends, so the extra keys are inert where unrecognised. +//! +//! The `langfuse.*` names are taken verbatim from +//! `packages/shared/src/server/otel/attributes.ts` in the Langfuse source, +//! which is ahead of the published documentation page. + +// ── Langfuse: trace scope ─────────────────────────────────────────────────── + +/// Trace display name. +pub const TRACE_NAME: &str = "langfuse.trace.name"; +/// End-user identity. Primary spelling; `langfuse.user.id` is the legacy alias. +pub const USER_ID: &str = "user.id"; +/// Session grouping. Primary spelling; `langfuse.session.id` is the legacy alias. +pub const SESSION_ID: &str = "session.id"; +/// Filterable trace labels. +pub const TRACE_TAGS: &str = "langfuse.trace.tags"; +/// Trace-level input payload. +pub const TRACE_INPUT: &str = "langfuse.trace.input"; +/// Trace-level output payload. +pub const TRACE_OUTPUT: &str = "langfuse.trace.output"; +/// Trace-level structured metadata. +pub const TRACE_METADATA: &str = "langfuse.trace.metadata"; +/// Whether the trace is publicly viewable. +pub const TRACE_PUBLIC: &str = "langfuse.trace.public"; + +// ── Langfuse: observation ─────────────────────────────────────────────────── + +/// Observation taxonomy slot (`GENERATION`, `TOOL`, `AGENT`, ...). +pub const OBSERVATION_TYPE: &str = "langfuse.observation.type"; +/// Observation severity. +pub const OBSERVATION_LEVEL: &str = "langfuse.observation.level"; +/// Error or warning detail. +pub const OBSERVATION_STATUS_MESSAGE: &str = "langfuse.observation.status_message"; +/// Observation input payload. +pub const OBSERVATION_INPUT: &str = "langfuse.observation.input"; +/// Observation output payload. +pub const OBSERVATION_OUTPUT: &str = "langfuse.observation.output"; +/// Observation structured metadata. +pub const OBSERVATION_METADATA: &str = "langfuse.observation.metadata"; +/// Time the first output token arrived. +pub const OBSERVATION_COMPLETION_START_TIME: &str = "langfuse.observation.completion_start_time"; +/// Model identifier. +pub const OBSERVATION_MODEL: &str = "langfuse.observation.model.name"; +/// Sampling parameters. +pub const OBSERVATION_MODEL_PARAMETERS: &str = "langfuse.observation.model.parameters"; +/// Token usage breakdown. +pub const OBSERVATION_USAGE_DETAILS: &str = "langfuse.observation.usage_details"; +/// Cost breakdown in USD. +pub const OBSERVATION_COST_DETAILS: &str = "langfuse.observation.cost_details"; +/// Managed-prompt name this generation was rendered from. +pub const OBSERVATION_PROMPT_NAME: &str = "langfuse.observation.prompt.name"; +/// Managed-prompt version this generation was rendered from. +pub const OBSERVATION_PROMPT_VERSION: &str = "langfuse.observation.prompt.version"; + +// ── Langfuse: general ─────────────────────────────────────────────────────── + +/// Deployment environment. +pub const ENVIRONMENT: &str = "langfuse.environment"; +/// Build release identifier. +pub const RELEASE: &str = "langfuse.release"; +/// Application-level version. +pub const VERSION: &str = "langfuse.version"; + +// ── OpenTelemetry GenAI semantic conventions ──────────────────────────────── + +/// Provider name, e.g. `anthropic`. +pub const GEN_AI_SYSTEM: &str = "gen_ai.system"; +/// Operation being performed, e.g. `chat`. +pub const GEN_AI_OPERATION_NAME: &str = "gen_ai.operation.name"; +/// Model requested. +pub const GEN_AI_REQUEST_MODEL: &str = "gen_ai.request.model"; +/// Model that actually served the request. +pub const GEN_AI_RESPONSE_MODEL: &str = "gen_ai.response.model"; +/// Requested sampling temperature. +pub const GEN_AI_REQUEST_TEMPERATURE: &str = "gen_ai.request.temperature"; +/// Requested output token ceiling. +pub const GEN_AI_REQUEST_MAX_TOKENS: &str = "gen_ai.request.max_tokens"; +/// Fresh input tokens consumed. +pub const GEN_AI_USAGE_INPUT_TOKENS: &str = "gen_ai.usage.input_tokens"; +/// Output tokens generated. +pub const GEN_AI_USAGE_OUTPUT_TOKENS: &str = "gen_ai.usage.output_tokens"; +/// Conversation grouping id. +pub const GEN_AI_CONVERSATION_ID: &str = "gen_ai.conversation.id"; +/// Tool name, for tool spans. +pub const GEN_AI_TOOL_NAME: &str = "gen_ai.tool.name"; + +// ── Resource attributes ───────────────────────────────────────────────────── + +/// Emitting service name. +pub const SERVICE_NAME: &str = "service.name"; +/// Emitting service version. +pub const SERVICE_VERSION: &str = "service.version"; +/// Deployment environment, OTel's own spelling. +pub const DEPLOYMENT_ENVIRONMENT: &str = "deployment.environment.name"; diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs new file mode 100644 index 000000000..3506372c0 --- /dev/null +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -0,0 +1,998 @@ +//! Projection of the internal trace model onto OTLP spans. +//! +//! Every emission decision is gated on the sink's [`ExportProfile`], so the +//! same agent run yields a content-rich payload for Langfuse and a lean +//! operational span for Datadog or Grafana. See `crate::profile` for why. + +use time::OffsetDateTime; + +use { + super::{attributes as attr, wire}, + crate::{ + model::{Event, Level, ObservationRecord, TraceRecord, TraceScope}, + profile::ExportProfile, + }, +}; + +/// Instrumentation scope name reported to backends. +pub const SCOPE_NAME: &str = "moltis"; + +/// Identity and policy applied to one export batch. +#[derive(Debug, Clone)] +pub struct ExportContext { + /// Value reported as `service.name`. + pub service_name: String, + /// Value reported as `service.version`. + pub service_version: String, + /// Deployment environment. + pub environment: Option, + /// What this backend is allowed to receive. + pub profile: ExportProfile, +} + +/// Convert an RFC3339 instant to nanoseconds since the Unix epoch. +fn unix_nanos(at: OffsetDateTime) -> String { + let nanos = at.unix_timestamp_nanos(); + // OTLP treats the field as unsigned; pre-epoch instants cannot occur here + // but clamping is cheaper than reasoning about a negative timestamp + // silently wrapping in a collector. + if nanos < 0 { + return "0".to_string(); + } + nanos.to_string() +} + +/// Truncate an attribute value to the profile's ceiling on a char boundary. +fn clamp(text: String, max: usize) -> String { + if max == 0 || text.len() <= max { + return text; + } + let mut end = max; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + format!("{}…", &text[..end]) +} + +/// Serialize a value for an attribute slot that expects JSON text. +fn json_attr(key: &str, value: &serde_json::Value, max: usize) -> Option { + match value { + serde_json::Value::Null => None, + serde_json::Value::String(s) => Some(wire::KeyValue::string(key, clamp(s.clone(), max))), + other => serde_json::to_string(other) + .ok() + .map(|s| wire::KeyValue::string(key, clamp(s, max))), + } +} + +/// Approximate size of a payload, for backends that get counts but not content. +fn payload_size(value: &serde_json::Value) -> i64 { + match value { + serde_json::Value::String(s) => s.len() as i64, + other => serde_json::to_string(other) + .map(|s| s.len() as i64) + .unwrap_or(0), + } +} + +/// Append attributes carried by every span in a trace. +fn push_scope_attrs(out: &mut Vec, scope: &TraceScope, profile: &ExportProfile) { + if let Some(session) = &scope.session_id + && profile.emit_session_id + { + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string(attr::SESSION_ID, session.clone())); + } + // GenAI semconv's own conversation grouping, understood everywhere. + out.push(wire::KeyValue::string( + attr::GEN_AI_CONVERSATION_ID, + session.clone(), + )); + } + if let Some(user) = &scope.user_id + && profile.emit_user_id + { + out.push(wire::KeyValue::string(attr::USER_ID, user.clone())); + } + if !scope.tags.is_empty() && profile.emit_tags && profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string_array(attr::TRACE_TAGS, &scope.tags)); + } + if let Some(env) = &scope.environment { + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string(attr::ENVIRONMENT, env.clone())); + } + out.push(wire::KeyValue::string( + attr::DEPLOYMENT_ENVIRONMENT, + env.clone(), + )); + } + if !profile.emits_langfuse_attrs() { + return; + } + if let Some(release) = &scope.release { + out.push(wire::KeyValue::string(attr::RELEASE, release.clone())); + } + if let Some(version) = &scope.version { + out.push(wire::KeyValue::string(attr::VERSION, version.clone())); + } +} + +/// Build the span representing a trace's root. +/// +/// Langfuse reconstructs the trace record from its spans, so trace-level input, +/// output and metadata ride on a zero-duration root span. +#[must_use] +pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { + let profile = &ctx.profile; + let mut attributes = Vec::new(); + + if profile.emits_langfuse_attrs() { + attributes.push(wire::KeyValue::string(attr::TRACE_NAME, trace.name.clone())); + attributes.push(wire::KeyValue::string(attr::OBSERVATION_TYPE, "SPAN")); + } + push_scope_attrs(&mut attributes, &trace.scope, profile); + + if profile.emits_bodies() { + if let Some(input) = &trace.input + && let Some(kv) = json_attr(attr::TRACE_INPUT, input, profile.max_attribute_bytes) + { + attributes.push(kv); + } + if let Some(output) = &trace.output + && let Some(kv) = json_attr(attr::TRACE_OUTPUT, output, profile.max_attribute_bytes) + { + attributes.push(kv); + } + } else if profile.emits_content_metadata() { + // Size without content: enough to spot a runaway prompt in an APM + // without copying the prompt itself into it. + if let Some(input) = &trace.input { + attributes.push(wire::KeyValue::int( + "moltis.input.bytes", + payload_size(input), + )); + } + if let Some(output) = &trace.output { + attributes.push(wire::KeyValue::int( + "moltis.output.bytes", + payload_size(output), + )); + } + } + + if profile.emits_langfuse_attrs() { + if !trace.metadata.is_empty() + && let Ok(meta) = serde_json::to_string(&trace.metadata) + { + attributes.push(wire::KeyValue::string( + attr::TRACE_METADATA, + clamp(meta, profile.max_attribute_bytes), + )); + } + if trace.public { + attributes.push(wire::KeyValue::bool(attr::TRACE_PUBLIC, true)); + } + } + + let start = unix_nanos(trace.timestamp); + wire::Span { + trace_id: trace.id.as_otel_hex(), + // Deriving the root span id from the trace id keeps it stable across + // repeated trace updates, so the backend upserts instead of creating + // a fresh root span each time. + span_id: crate::model::ObservationId(trace.id.0.clone()).as_otel_hex(), + parent_span_id: None, + name: trace.name.clone(), + kind: wire::SPAN_KIND_INTERNAL, + end_time_unix_nano: start.clone(), + start_time_unix_nano: start, + attributes, + status: wire::Status::unset(), + } +} + +/// Build the span representing an observation. +#[must_use] +pub fn observation_to_span(obs: &ObservationRecord, ctx: &ExportContext) -> wire::Span { + let profile = &ctx.profile; + let mut attributes = Vec::new(); + + if profile.emits_langfuse_attrs() { + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_TYPE, + obs.kind.as_str(), + )); + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_LEVEL, + obs.level.as_str(), + )); + } else { + // Ops backends still need to group by step kind; a lowercase + // operation name is the GenAI-conventional way to express it. + attributes.push(wire::KeyValue::string( + attr::GEN_AI_OPERATION_NAME, + obs.kind.as_str().to_lowercase(), + )); + } + + push_scope_attrs(&mut attributes, &obs.scope, profile); + + // Error detail is operational, not conversational: every backend gets it. + if let Some(message) = &obs.status_message { + let key = if profile.emits_langfuse_attrs() { + attr::OBSERVATION_STATUS_MESSAGE + } else { + "error.message" + }; + attributes.push(wire::KeyValue::string( + key, + clamp(message.clone(), profile.max_attribute_bytes), + )); + } + + push_payload_attrs(&mut attributes, obs, profile); + push_model_attrs(&mut attributes, obs, profile); + push_usage_attrs(&mut attributes, obs, profile); + + if profile.emits_langfuse_attrs() { + push_prompt_attrs(&mut attributes, obs); + if !obs.metadata.is_empty() + && let Ok(meta) = serde_json::to_string(&obs.metadata) + { + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_METADATA, + clamp(meta, profile.max_attribute_bytes), + )); + } + } + + if obs.kind == crate::model::ObservationKind::Tool { + attributes.push(wire::KeyValue::string( + attr::GEN_AI_TOOL_NAME, + obs.name.clone(), + )); + } + + let status = if obs.level == Level::Error { + wire::Status::error(obs.status_message.clone()) + } else { + wire::Status::unset() + }; + + wire::Span { + trace_id: obs.trace_id.as_otel_hex(), + span_id: obs.id.as_otel_hex(), + parent_span_id: Some( + obs.parent_id + .as_ref() + .map_or_else( + || crate::model::ObservationId(obs.trace_id.0.clone()), + Clone::clone, + ) + .as_otel_hex(), + ), + name: obs.name.clone(), + kind: wire::SPAN_KIND_INTERNAL, + start_time_unix_nano: unix_nanos(obs.start_time), + // A still-running observation reports its start as its end; the update + // that arrives when it finishes carries the real end time. + end_time_unix_nano: unix_nanos(obs.end_time.unwrap_or(obs.start_time)), + attributes, + status, + } +} + +/// Input and output payloads, or their sizes. +fn push_payload_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + if profile.emits_bodies() { + if let Some(input) = &obs.input + && let Some(kv) = json_attr(attr::OBSERVATION_INPUT, input, profile.max_attribute_bytes) + { + out.push(kv); + } + if let Some(output) = &obs.output + && let Some(kv) = json_attr( + attr::OBSERVATION_OUTPUT, + output, + profile.max_attribute_bytes, + ) + { + out.push(kv); + } + return; + } + if !profile.emits_content_metadata() { + return; + } + if let Some(input) = &obs.input { + out.push(wire::KeyValue::int( + "moltis.input.bytes", + payload_size(input), + )); + } + if let Some(output) = &obs.output { + out.push(wire::KeyValue::int( + "moltis.output.bytes", + payload_size(output), + )); + } +} + +/// Model identity and sampling parameters. +fn push_model_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + let Some(model) = &obs.model else { + return; + }; + if profile.emits_langfuse_attrs() { + out.push(wire::KeyValue::string( + attr::OBSERVATION_MODEL, + model.clone(), + )); + } + // Model identity is low-cardinality and operationally essential, so it + // goes to every backend. + out.push(wire::KeyValue::string( + attr::GEN_AI_REQUEST_MODEL, + model.clone(), + )); + out.push(wire::KeyValue::string( + attr::GEN_AI_RESPONSE_MODEL, + model.clone(), + )); + + if obs.model_parameters.is_empty() { + return; + } + if profile.emits_langfuse_attrs() + && let Ok(params) = serde_json::to_string(&obs.model_parameters) + { + out.push(wire::KeyValue::string( + attr::OBSERVATION_MODEL_PARAMETERS, + clamp(params, profile.max_attribute_bytes), + )); + } + // Promote the two parameters GenAI consumers actually chart. + if let Some(temp) = obs + .model_parameters + .get("temperature") + .and_then(serde_json::Value::as_f64) + { + out.push(wire::KeyValue::double( + attr::GEN_AI_REQUEST_TEMPERATURE, + temp, + )); + } + if let Some(max) = obs + .model_parameters + .get("max_tokens") + .and_then(serde_json::Value::as_i64) + { + out.push(wire::KeyValue::int(attr::GEN_AI_REQUEST_MAX_TOKENS, max)); + } +} + +/// Token usage and cost. +fn push_usage_attrs( + out: &mut Vec, + obs: &ObservationRecord, + profile: &ExportProfile, +) { + if !profile.emit_usage { + return; + } + + if let Some(completion_start) = obs.completion_start_time + && profile.emits_langfuse_attrs() + { + // Langfuse parses this as an ISO-8601 instant, not a nanosecond count. + if let Ok(text) = completion_start.format(&time::format_description::well_known::Rfc3339) { + out.push(wire::KeyValue::string( + attr::OBSERVATION_COMPLETION_START_TIME, + text, + )); + } + } + + if let Some(usage) = obs.usage.filter(|u| !u.is_empty()) { + if profile.emits_langfuse_attrs() { + // Only Langfuse prices the cache buckets; elsewhere this is an + // opaque JSON blob nothing can chart. + let details = usage.to_usage_details(); + if let Ok(json) = serde_json::to_string(&details) { + out.push(wire::KeyValue::string( + attr::OBSERVATION_USAGE_DETAILS, + json, + )); + } + } + out.push(wire::KeyValue::int( + attr::GEN_AI_USAGE_INPUT_TOKENS, + i64::from(usage.input), + )); + out.push(wire::KeyValue::int( + attr::GEN_AI_USAGE_OUTPUT_TOKENS, + i64::from(usage.output), + )); + } + + if profile.emits_langfuse_attrs() + && !obs.cost_details.is_empty() + && let Ok(json) = serde_json::to_string(&obs.cost_details) + { + out.push(wire::KeyValue::string(attr::OBSERVATION_COST_DETAILS, json)); + } +} + +/// Managed-prompt linkage. Langfuse-only: no other backend models it. +fn push_prompt_attrs(out: &mut Vec, obs: &ObservationRecord) { + if let Some(name) = &obs.prompt_name { + out.push(wire::KeyValue::string( + attr::OBSERVATION_PROMPT_NAME, + name.clone(), + )); + } + if let Some(version) = obs.prompt_version { + out.push(wire::KeyValue::int( + attr::OBSERVATION_PROMPT_VERSION, + i64::from(version), + )); + } +} + +/// Resource attributes describing this Moltis instance. +#[must_use] +pub fn resource_attributes(ctx: &ExportContext) -> Vec { + let mut attrs = vec![ + wire::KeyValue::string(attr::SERVICE_NAME, ctx.service_name.clone()), + wire::KeyValue::string(attr::SERVICE_VERSION, ctx.service_version.clone()), + ]; + if let Some(env) = &ctx.environment { + attrs.push(wire::KeyValue::string( + attr::DEPLOYMENT_ENVIRONMENT, + env.clone(), + )); + if ctx.profile.emits_langfuse_attrs() { + attrs.push(wire::KeyValue::string(attr::ENVIRONMENT, env.clone())); + } + } + attrs +} + +/// Convert a batch of events into an OTLP export request. +/// +/// Score events carry no span representation and are skipped here; they are +/// delivered through the backend's scoring API instead. +#[must_use] +pub fn batch_to_request(events: &[Event], ctx: &ExportContext) -> wire::ExportTraceServiceRequest { + let spans: Vec = events + .iter() + .filter_map(|event| match event { + Event::Trace(trace) => Some(trace_to_span(trace, ctx)), + Event::ObservationStart(obs) | Event::ObservationEnd(obs) => { + Some(observation_to_span(obs, ctx)) + }, + Event::Score(_) => None, + }) + .collect(); + + wire::ExportTraceServiceRequest { + resource_spans: vec![wire::ResourceSpans { + resource: wire::Resource { + attributes: resource_attributes(ctx), + }, + scope_spans: vec![wire::ScopeSpans { + scope: wire::InstrumentationScope { + name: SCOPE_NAME.to_string(), + version: ctx.service_version.clone(), + }, + spans, + }], + }], + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use { + super::*, + crate::model::{ + ObservationId, ObservationKind, ScoreRecord, ScoreValue, TokenUsage, TraceId, + }, + }; + + fn ctx(profile: ExportProfile) -> ExportContext { + ExportContext { + service_name: "moltis".into(), + service_version: "20260726.01".into(), + environment: Some("production".into()), + profile, + } + } + + fn langfuse_ctx() -> ExportContext { + ctx(ExportProfile::langfuse()) + } + + fn otel_ctx() -> ExportContext { + ctx(ExportProfile::otel_generic()) + } + + fn attr_value(span: &wire::Span, key: &str) -> Option { + let json = serde_json::to_value(span).ok()?; + json["attributes"] + .as_array()? + .iter() + .find(|kv| kv["key"] == key) + .map(|kv| kv["value"].clone()) + } + + fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:main".into()), + user_id: Some("telegram:42".into()), + tags: vec!["telegram".into(), "agent:main".into()], + environment: Some("production".into()), + release: Some("20260726.01".into()), + version: Some("preset-v3".into()), + } + } + + fn generation() -> ObservationRecord { + let mut obs = ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "anthropic/claude", + ) + .with_scope(scope()); + obs.model = Some("claude-opus-4".into()); + obs.usage = Some(TokenUsage::from_provider_totals(100, 50, 900, 20)); + obs.input = Some(json!({ "messages": [{ "role": "user", "content": "secret plan" }] })); + obs.output = Some(json!("here is the answer")); + obs.finish(); + obs + } + + // ── Profile separation: the core of the Langfuse/APM split ────────── + + #[test] + fn langfuse_receives_conversation_bodies() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_some()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_some()); + } + + #[test] + fn generic_otel_never_receives_conversation_bodies() { + // The guarantee that makes it safe to point this at a shared APM. + let span = observation_to_span(&generation(), &otel_ctx()); + let rendered = serde_json::to_string(&span).expect("serializable"); + + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_none()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_none()); + assert!( + !rendered.contains("secret plan"), + "prompt content leaked into an APM payload" + ); + assert!(!rendered.contains("here is the answer")); + } + + #[test] + fn generic_otel_receives_payload_sizes_instead_of_payloads() { + // Enough to alert on a runaway prompt without copying the prompt. + let span = observation_to_span(&generation(), &otel_ctx()); + assert!(matches!( + attr_value(&span, "moltis.input.bytes"), + Some(v) if v["intValue"].is_string() + )); + } + + #[test] + fn content_none_emits_neither_bodies_nor_sizes() { + let profile = ExportProfile { + content: crate::profile::ContentCapture::None, + ..ExportProfile::otel_generic() + }; + let span = observation_to_span(&generation(), &ctx(profile)); + + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_none()); + assert!(attr_value(&span, "moltis.input.bytes").is_none()); + } + + #[test] + fn generic_otel_omits_langfuse_vocabulary() { + let span = observation_to_span(&generation(), &otel_ctx()); + let rendered = serde_json::to_string(&span).expect("serializable"); + assert!( + !rendered.contains("langfuse."), + "langfuse-specific attributes are noise in an APM" + ); + } + + #[test] + fn generic_otel_omits_user_id_but_keeps_session() { + let span = observation_to_span(&generation(), &otel_ctx()); + + // Per-user cardinality is a billing and compliance problem in an APM. + assert!(attr_value(&span, attr::USER_ID).is_none()); + // Session grouping stays: it is bounded and operationally useful. + assert!(attr_value(&span, attr::GEN_AI_CONVERSATION_ID).is_some()); + } + + #[test] + fn datadog_profile_drops_tags() { + let span = observation_to_span(&generation(), &ctx(ExportProfile::datadog())); + assert!(attr_value(&span, attr::TRACE_TAGS).is_none()); + } + + #[test] + fn generic_otel_expresses_step_kind_as_gen_ai_operation() { + let mut obs = generation(); + obs.kind = ObservationKind::Retriever; + let span = observation_to_span(&obs, &otel_ctx()); + + assert_eq!( + attr_value(&span, attr::GEN_AI_OPERATION_NAME), + Some(json!({ "stringValue": "retriever" })) + ); + assert!(attr_value(&span, attr::OBSERVATION_TYPE).is_none()); + } + + #[test] + fn model_and_error_reach_every_backend() { + // Operationally essential and low-cardinality, so never gated. + let mut obs = generation(); + obs.fail("provider returned 500"); + + for context in [langfuse_ctx(), otel_ctx(), ctx(ExportProfile::datadog())] { + let span = observation_to_span(&obs, &context); + assert!(attr_value(&span, attr::GEN_AI_REQUEST_MODEL).is_some()); + assert_eq!(span.status.code, wire::STATUS_ERROR); + } + } + + #[test] + fn token_counts_reach_every_backend_but_cost_detail_does_not() { + let mut obs = generation(); + obs.cost_details.insert("total".into(), 0.42); + + let otel = observation_to_span(&obs, &otel_ctx()); + assert!(attr_value(&otel, attr::GEN_AI_USAGE_INPUT_TOKENS).is_some()); + // No APM has a model price table, so the breakdown is dead weight. + assert!(attr_value(&otel, attr::OBSERVATION_COST_DETAILS).is_none()); + assert!(attr_value(&otel, attr::OBSERVATION_USAGE_DETAILS).is_none()); + + let lf = observation_to_span(&obs, &langfuse_ctx()); + assert!(attr_value(&lf, attr::OBSERVATION_COST_DETAILS).is_some()); + } + + #[test] + fn usage_can_be_disabled_entirely() { + let profile = ExportProfile { + emit_usage: false, + ..ExportProfile::otel_generic() + }; + let span = observation_to_span(&generation(), &ctx(profile)); + assert!(attr_value(&span, attr::GEN_AI_USAGE_INPUT_TOKENS).is_none()); + } + + #[test] + fn prompt_linkage_is_langfuse_only() { + let mut obs = generation(); + obs.prompt_name = Some("system-prompt".into()); + obs.prompt_version = Some(7); + + assert!( + attr_value( + &observation_to_span(&obs, &langfuse_ctx()), + attr::OBSERVATION_PROMPT_NAME + ) + .is_some() + ); + assert!( + attr_value( + &observation_to_span(&obs, &otel_ctx()), + attr::OBSERVATION_PROMPT_NAME + ) + .is_none() + ); + } + + #[test] + fn trace_bodies_are_withheld_from_an_apm() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!("private question")); + trace.output = Some(json!("private answer")); + + let rendered = + serde_json::to_string(&trace_to_span(&trace, &otel_ctx())).expect("serializable"); + assert!(!rendered.contains("private question")); + assert!(!rendered.contains("private answer")); + } + + #[test] + fn attribute_values_are_clamped_to_the_profile_ceiling() { + let mut obs = generation(); + obs.output = Some(json!("x".repeat(100_000))); + let profile = ExportProfile { + max_attribute_bytes: 100, + ..ExportProfile::langfuse() + }; + let span = observation_to_span(&obs, &ctx(profile)); + let value = attr_value(&span, attr::OBSERVATION_OUTPUT) + .and_then(|v| v["stringValue"].as_str().map(str::to_string)) + .expect("output present"); + + assert!( + value.len() <= 110, + "clamped value was {} bytes", + value.len() + ); + } + + #[test] + fn clamping_does_not_split_multibyte_characters() { + // Slicing blindly at a byte offset would panic on this input. + let clamped = clamp("ααααααααα".to_string(), 5); + assert!(clamped.ends_with('…')); + } + + // ── Shared structural behaviour ───────────────────────────────────── + + #[test] + fn observation_type_uses_langfuse_taxonomy() { + let mut obs = generation(); + obs.kind = ObservationKind::Tool; + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::OBSERVATION_TYPE), + Some(json!({ "stringValue": "TOOL" })) + ); + } + + #[test] + fn tool_spans_carry_gen_ai_tool_name_on_every_backend() { + let mut obs = generation(); + obs.kind = ObservationKind::Tool; + obs.name = "exec".into(); + + for context in [langfuse_ctx(), otel_ctx()] { + assert_eq!( + attr_value(&observation_to_span(&obs, &context), attr::GEN_AI_TOOL_NAME), + Some(json!({ "stringValue": "exec" })) + ); + } + } + + #[test] + fn scope_attributes_ride_on_every_span() { + // OTLP has no trace-level attribute concept, so a backend filtering by + // session sees only what each span itself carries. + let span = observation_to_span(&generation(), &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::SESSION_ID), + Some(json!({ "stringValue": "agent:main:main" })) + ); + assert_eq!( + attr_value(&span, attr::USER_ID), + Some(json!({ "stringValue": "telegram:42" })) + ); + assert_eq!( + attr_value(&span, attr::RELEASE), + Some(json!({ "stringValue": "20260726.01" })) + ); + } + + #[test] + fn usage_details_carry_cache_buckets_separately() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + let raw = attr_value(&span, attr::OBSERVATION_USAGE_DETAILS) + .and_then(|v| v["stringValue"].as_str().map(str::to_string)) + .expect("usage details present"); + let parsed: serde_json::Value = serde_json::from_str(&raw).expect("valid json"); + + assert_eq!(parsed["input"], 100); + assert_eq!(parsed["input_cache_read"], 900); + assert_eq!(parsed["input_cache_write"], 20); + } + + #[test] + fn empty_usage_is_omitted_rather_than_reported_as_zero() { + let mut obs = generation(); + obs.usage = Some(TokenUsage::default()); + let span = observation_to_span(&obs, &langfuse_ctx()); + + // A zero-token generation distorts the backend's cost charts. + assert!(attr_value(&span, attr::OBSERVATION_USAGE_DETAILS).is_none()); + assert!(attr_value(&span, attr::GEN_AI_USAGE_INPUT_TOKENS).is_none()); + } + + #[test] + fn error_level_sets_otel_status_code_two() { + let mut obs = generation(); + obs.fail("provider returned 500"); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!(span.status.code, wire::STATUS_ERROR); + assert_eq!( + span.status.message.as_deref(), + Some("provider returned 500") + ); + } + + #[test] + fn apm_gets_error_message_under_the_conventional_key() { + let mut obs = generation(); + obs.fail("provider returned 500"); + let span = observation_to_span(&obs, &otel_ctx()); + + assert_eq!( + attr_value(&span, "error.message"), + Some(json!({ "stringValue": "provider returned 500" })) + ); + } + + #[test] + fn non_error_levels_leave_status_unset() { + let span = observation_to_span(&generation(), &langfuse_ctx()); + assert_eq!(span.status.code, wire::STATUS_UNSET); + } + + #[test] + fn orphan_observations_are_parented_to_the_trace_root() { + // Otherwise a top-level observation becomes a second root span and the + // backend renders two disconnected traces. + let obs = generation(); + let span = observation_to_span(&obs, &langfuse_ctx()); + let expected = ObservationId(obs.trace_id.0.clone()).as_otel_hex(); + + assert_eq!(span.parent_span_id, Some(expected)); + } + + #[test] + fn nested_observations_point_at_their_parent() { + let parent = generation(); + let child = + ObservationRecord::start(parent.trace_id.clone(), ObservationKind::Tool, "exec") + .with_parent(Some(parent.id.clone())); + + let span = observation_to_span(&child, &langfuse_ctx()); + assert_eq!(span.parent_span_id, Some(parent.id.as_otel_hex())); + } + + #[test] + fn trace_root_span_id_is_derived_from_the_trace_id() { + // A fresh random root id per update would create duplicate roots. + let trace = TraceRecord::new("turn"); + let first = trace_to_span(&trace, &langfuse_ctx()); + let second = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!(first.span_id, second.span_id); + assert!(first.parent_span_id.is_none()); + } + + #[test] + fn string_inputs_are_not_double_encoded() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!("what is the weather")); + let span = trace_to_span(&trace, &langfuse_ctx()); + + // Wrapping a plain string in JSON quotes renders as `"\"text\""` in the + // backend UI, on every single trace. + assert_eq!( + attr_value(&span, attr::TRACE_INPUT), + Some(json!({ "stringValue": "what is the weather" })) + ); + } + + #[test] + fn structured_inputs_are_json_encoded() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(json!({ "role": "user" })); + let span = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::TRACE_INPUT), + Some(json!({ "stringValue": "{\"role\":\"user\"}" })) + ); + } + + #[test] + fn null_payloads_are_dropped() { + let mut trace = TraceRecord::new("turn"); + trace.input = Some(serde_json::Value::Null); + let span = trace_to_span(&trace, &langfuse_ctx()); + assert!(attr_value(&span, attr::TRACE_INPUT).is_none()); + } + + #[test] + fn completion_start_time_is_rfc3339_not_nanos() { + let mut obs = generation(); + obs.completion_start_time = Some(OffsetDateTime::UNIX_EPOCH); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::OBSERVATION_COMPLETION_START_TIME), + Some(json!({ "stringValue": "1970-01-01T00:00:00Z" })) + ); + } + + #[test] + fn running_observations_report_end_equal_to_start() { + let mut obs = generation(); + obs.end_time = None; + let span = observation_to_span(&obs, &langfuse_ctx()); + assert_eq!(span.start_time_unix_nano, span.end_time_unix_nano); + } + + #[test] + fn model_parameters_promote_temperature_and_max_tokens() { + let mut obs = generation(); + obs.model_parameters + .insert("temperature".into(), json!(0.7)); + obs.model_parameters + .insert("max_tokens".into(), json!(4096)); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::GEN_AI_REQUEST_TEMPERATURE), + Some(json!({ "doubleValue": 0.7 })) + ); + assert_eq!( + attr_value(&span, attr::GEN_AI_REQUEST_MAX_TOKENS), + Some(json!({ "intValue": "4096" })) + ); + } + + #[test] + fn scores_produce_no_spans() { + let events = vec![Event::Score(Box::new(ScoreRecord::new( + TraceId::generate(), + "user-feedback", + ScoreValue::Numeric(1.0), + )))]; + let request = batch_to_request(&events, &langfuse_ctx()); + + assert!(request.resource_spans[0].scope_spans[0].spans.is_empty()); + } + + #[test] + fn batch_carries_resource_and_scope_identity() { + let events = vec![Event::Trace(Box::new(TraceRecord::new("turn")))]; + let request = batch_to_request(&events, &langfuse_ctx()); + let json = serde_json::to_value(&request).expect("serializable"); + + assert_eq!( + json["resourceSpans"][0]["scopeSpans"][0]["scope"]["name"], + SCOPE_NAME + ); + let names: Vec<&str> = json["resourceSpans"][0]["resource"]["attributes"] + .as_array() + .map(|a| a.iter().filter_map(|kv| kv["key"].as_str()).collect()) + .unwrap_or_default(); + assert!(names.contains(&attr::SERVICE_NAME)); + assert!(names.contains(&attr::DEPLOYMENT_ENVIRONMENT)); + } + + #[test] + fn pre_epoch_timestamps_clamp_to_zero() { + let mut trace = TraceRecord::new("turn"); + trace.timestamp = OffsetDateTime::UNIX_EPOCH - time::Duration::days(1); + let span = trace_to_span(&trace, &langfuse_ctx()); + + // OTLP treats the field as unsigned; a negative value wraps to a + // far-future timestamp in collectors. + assert_eq!(span.start_time_unix_nano, "0"); + } +} diff --git a/crates/observability/src/exporters/otlp/mod.rs b/crates/observability/src/exporters/otlp/mod.rs new file mode 100644 index 000000000..579d5144f --- /dev/null +++ b/crates/observability/src/exporters/otlp/mod.rs @@ -0,0 +1,386 @@ +//! OTLP/HTTP JSON exporter. +//! +//! One transport serves every backend we target: Langfuse (which accepts OTLP +//! on `/api/public/otel/v1/traces`), Grafana Tempo/Alloy, Datadog's OTLP +//! intake, Honeycomb, and any other OTLP-compatible collector. The difference +//! between them is the endpoint and the auth header, not the payload. + +pub mod attributes; +pub mod mapping; +pub mod wire; + +use std::{collections::BTreeMap, time::Duration}; + +use { + async_trait::async_trait, + reqwest::{ + StatusCode, + header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue}, + }, + tracing::{debug, warn}, +}; + +use crate::{ + model::Event, + profile::ExportProfile, + runtime::{Transport, TransportError}, +}; + +/// Configuration for an OTLP exporter. +#[derive(Debug, Clone)] +pub struct OtlpConfig { + /// Display name for logs and status reporting. + pub name: String, + /// Full traces endpoint, e.g. `https://cloud.langfuse.com/api/public/otel/v1/traces`. + pub endpoint: String, + /// Additional headers, typically carrying authentication. + pub headers: BTreeMap, + /// Per-request timeout. + pub timeout: Duration, + /// Value reported as `service.name`. + pub service_name: String, + /// Value reported as `service.version`. + pub service_version: String, + /// Deployment environment, reported on the resource and every span. + pub environment: Option, + /// What this backend is allowed to receive. Defaults to the conservative + /// profile so a misconfigured backend cannot silently receive prompts. + pub profile: ExportProfile, +} + +impl Default for OtlpConfig { + fn default() -> Self { + Self { + name: "otlp".to_string(), + endpoint: String::new(), + headers: BTreeMap::new(), + timeout: Duration::from_secs(10), + service_name: "moltis".to_string(), + service_version: env!("CARGO_PKG_VERSION").to_string(), + environment: None, + profile: ExportProfile::default(), + } + } +} + +/// OTLP/HTTP JSON transport. +pub struct OtlpTransport { + config: OtlpConfig, + client: reqwest::Client, + headers: HeaderMap, +} + +impl OtlpTransport { + /// Build a transport, reusing the workspace HTTP client so the configured + /// upstream proxy and default headers apply. + /// + /// Header values that cannot be encoded are dropped with a warning rather + /// than failing construction: one malformed custom header should not + /// disable instrumentation entirely. + #[must_use] + pub fn new(config: OtlpConfig) -> Self { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json")); + + for (key, value) in &config.headers { + let Ok(name) = key.parse::() else { + warn!(header = %key, "ignoring invalid OTLP header name"); + continue; + }; + let Ok(mut val) = HeaderValue::from_str(value) else { + warn!(header = %key, "ignoring OTLP header with unencodable value"); + continue; + }; + // Credentials must never surface in a debug dump of the headers. + val.set_sensitive(true); + headers.insert(name, val); + } + + Self { + client: moltis_common::http_client::build_default_http_client(), + headers, + config, + } + } + + /// The configured endpoint. + #[must_use] + pub fn endpoint(&self) -> &str { + &self.config.endpoint + } + + /// Identity and policy applied to each exported batch. + fn export_context(&self) -> mapping::ExportContext { + mapping::ExportContext { + service_name: self.config.service_name.clone(), + service_version: self.config.service_version.clone(), + environment: self.config.environment.clone(), + profile: self.config.profile.clone(), + } + } + + /// Classify an HTTP status into a retry decision. + /// + /// 4xx other than 408/429 are fatal: retrying a rejected payload or bad + /// credentials cannot succeed and only burns rate-limit budget. + fn classify(status: StatusCode, body: String) -> TransportError { + if status == StatusCode::TOO_MANY_REQUESTS + || status == StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + TransportError::Retryable(format!("HTTP {status}: {body}")) + } else { + TransportError::Fatal(format!("HTTP {status}: {body}")) + } + } +} + +#[async_trait] +impl Transport for OtlpTransport { + fn name(&self) -> &str { + &self.config.name + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + let request = mapping::batch_to_request(batch, &self.export_context()); + + // A batch of only score events produces no spans; sending an empty + // resourceSpans payload is a wasted round trip. + if request + .resource_spans + .iter() + .all(|rs| rs.scope_spans.iter().all(|ss| ss.spans.is_empty())) + { + return Ok(()); + } + + let body = serde_json::to_vec(&request) + .map_err(|e| TransportError::Fatal(format!("failed to encode OTLP payload: {e}")))?; + + let response = self + .client + .post(&self.config.endpoint) + .headers(self.headers.clone()) + .timeout(self.config.timeout) + .body(body) + .send() + .await + .map_err(|e| { + // Connection-level failures are always worth retrying: the + // collector may simply be restarting. + TransportError::Retryable(format!( + "request to {} failed: {e}", + self.config.endpoint + )) + })?; + + let status = response.status(); + if status.is_success() { + debug!( + sink = %self.config.name, + events = batch.len(), + "OTLP batch accepted" + ); + return Ok(()); + } + + let body = response.text().await.unwrap_or_default(); + // Truncate: some collectors echo the whole rejected payload back. + let body = body.chars().take(512).collect::(); + Err(Self::classify(status, body)) + } +} + +#[cfg(test)] +mod tests { + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId}, + }; + + fn config(endpoint: String) -> OtlpConfig { + let mut headers = BTreeMap::new(); + headers.insert("Authorization".to_string(), "Basic dGVzdA==".to_string()); + OtlpConfig { + name: "test".into(), + endpoint, + headers, + timeout: Duration::from_secs(5), + service_name: "moltis".into(), + service_version: "test".into(), + environment: Some("staging".into()), + profile: ExportProfile::langfuse(), + } + } + + fn events() -> Vec { + let mut obs = + ObservationRecord::start(TraceId::generate(), ObservationKind::Generation, "llm-call"); + obs.finish(); + vec![Event::ObservationEnd(Box::new(obs))] + } + + #[tokio::test] + async fn posts_json_with_configured_headers() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/traces")) + .and(header("content-type", "application/json")) + .and(header("authorization", "Basic dGVzdA==")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + transport + .send(&events()) + .await + .expect("send should succeed"); + } + + #[tokio::test] + async fn empty_span_batches_skip_the_request() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + // Scores map to no spans, so this batch has nothing to send. + let score = crate::model::ScoreRecord::new( + TraceId::generate(), + "feedback", + crate::model::ScoreValue::Numeric(1.0), + ); + transport + .send(&[Event::Score(Box::new(score))]) + .await + .expect("empty batch should be a no-op"); + } + + #[tokio::test] + async fn server_errors_are_retryable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("503 should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[tokio::test] + async fn rate_limiting_is_retryable() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(429)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("429 should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[tokio::test] + async fn unauthorized_is_fatal_not_retried() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("401 should fail"); + + // Retrying bad credentials cannot succeed and wastes rate-limit budget. + assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + } + + #[tokio::test] + async fn bad_request_is_fatal() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400)) + .mount(&server) + .await; + + let transport = OtlpTransport::new(config(format!("{}/v1/traces", server.uri()))); + let error = transport + .send(&events()) + .await + .expect_err("400 should fail"); + + assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + } + + #[tokio::test] + async fn connection_failure_is_retryable() { + // Port 1 is reserved and refuses connections immediately. + let transport = OtlpTransport::new(config("http://127.0.0.1:1/v1/traces".into())); + let error = transport + .send(&events()) + .await + .expect_err("connection refused should fail"); + + assert!(matches!(error, TransportError::Retryable(_)), "{error:?}"); + } + + #[test] + fn invalid_headers_are_skipped_without_disabling_the_exporter() { + let mut headers = BTreeMap::new(); + headers.insert("valid-header".to_string(), "ok".to_string()); + // Newlines cannot be encoded in a header value. + headers.insert("bad".to_string(), "line\nbreak".to_string()); + // Spaces are not legal in a header name. + headers.insert("in valid".to_string(), "x".to_string()); + + let transport = OtlpTransport::new(OtlpConfig { + headers, + ..Default::default() + }); + + assert!(transport.headers.contains_key("valid-header")); + assert!(!transport.headers.contains_key("bad")); + assert_eq!(transport.headers.len(), 2, "content-type plus valid-header"); + } + + #[test] + fn credential_headers_are_marked_sensitive() { + let mut headers = BTreeMap::new(); + headers.insert("authorization".to_string(), "Basic secret".to_string()); + let transport = OtlpTransport::new(OtlpConfig { + headers, + ..Default::default() + }); + + let value = transport + .headers + .get("authorization") + .expect("header present"); + // Sensitive headers are redacted by reqwest's own Debug output. + assert!(value.is_sensitive()); + assert!(!format!("{:?}", transport.headers).contains("Basic secret")); + } +} diff --git a/crates/observability/src/exporters/otlp/wire.rs b/crates/observability/src/exporters/otlp/wire.rs new file mode 100644 index 000000000..66862fb83 --- /dev/null +++ b/crates/observability/src/exporters/otlp/wire.rs @@ -0,0 +1,311 @@ +//! OTLP/HTTP JSON wire types. +//! +//! Hand-written rather than generated from protobuf: the JSON encoding is a +//! small, stable subset and hand-rolling it avoids pulling `prost`, `tonic` and +//! a build-time protoc dependency into the workspace for one request shape. +//! +//! The one sharp edge in OTLP/JSON is that 64-bit integers are encoded as +//! **decimal strings**, not JSON numbers, because JSON numbers cannot carry the +//! full int64 range. Timestamps and `intValue` therefore serialize as strings. + +use serde::Serialize; + +/// Top-level OTLP trace export request. +#[derive(Debug, Clone, Serialize)] +pub struct ExportTraceServiceRequest { + /// Spans grouped by emitting resource. + #[serde(rename = "resourceSpans")] + pub resource_spans: Vec, +} + +/// Spans from one resource (one service instance). +#[derive(Debug, Clone, Serialize)] +pub struct ResourceSpans { + /// Attributes describing the emitting service. + pub resource: Resource, + /// Spans grouped by instrumentation scope. + #[serde(rename = "scopeSpans")] + pub scope_spans: Vec, +} + +/// The emitting service. +#[derive(Debug, Clone, Serialize)] +pub struct Resource { + /// Resource-level attributes. + pub attributes: Vec, +} + +/// Spans from one instrumentation scope. +#[derive(Debug, Clone, Serialize)] +pub struct ScopeSpans { + /// The instrumentation library. + pub scope: InstrumentationScope, + /// The spans themselves. + pub spans: Vec, +} + +/// Instrumentation library identity. +#[derive(Debug, Clone, Serialize)] +pub struct InstrumentationScope { + /// Library name. + pub name: String, + /// Library version. + pub version: String, +} + +/// A single span. +#[derive(Debug, Clone, Serialize)] +pub struct Span { + /// 32-character lowercase hex trace id. + #[serde(rename = "traceId")] + pub trace_id: String, + /// 16-character lowercase hex span id. + #[serde(rename = "spanId")] + pub span_id: String, + /// Parent span id, omitted for a root span. + #[serde(rename = "parentSpanId", skip_serializing_if = "Option::is_none")] + pub parent_span_id: Option, + /// Span name. + pub name: String, + /// Span kind. `1` is INTERNAL, which is correct for in-process work. + pub kind: i32, + /// Start time in nanoseconds since the Unix epoch, as a decimal string. + #[serde(rename = "startTimeUnixNano")] + pub start_time_unix_nano: String, + /// End time in nanoseconds since the Unix epoch, as a decimal string. + #[serde(rename = "endTimeUnixNano")] + pub end_time_unix_nano: String, + /// Span attributes. + pub attributes: Vec, + /// Span status. + pub status: Status, +} + +/// Span kind: in-process work. +pub const SPAN_KIND_INTERNAL: i32 = 1; + +/// Span completion status. +#[derive(Debug, Clone, Serialize)] +pub struct Status { + /// `0` unset, `1` OK, `2` ERROR. + pub code: i32, + /// Human-readable detail, present only on error. + #[serde(skip_serializing_if = "Option::is_none")] + pub message: Option, +} + +/// Status code: no explicit status. +pub const STATUS_UNSET: i32 = 0; +/// Status code: failed. +pub const STATUS_ERROR: i32 = 2; + +impl Status { + /// A status carrying no signal. + #[must_use] + pub const fn unset() -> Self { + Self { + code: STATUS_UNSET, + message: None, + } + } + + /// A failure status with detail. + #[must_use] + pub fn error(message: Option) -> Self { + Self { + code: STATUS_ERROR, + message, + } + } +} + +/// One attribute. +#[derive(Debug, Clone, Serialize)] +pub struct KeyValue { + /// Attribute name. + pub key: String, + /// Attribute value. + pub value: AnyValue, +} + +impl KeyValue { + /// String-valued attribute. + #[must_use] + pub fn string(key: impl Into, value: impl Into) -> Self { + Self { + key: key.into(), + value: AnyValue::String { + string_value: value.into(), + }, + } + } + + /// Integer-valued attribute. + #[must_use] + pub fn int(key: impl Into, value: i64) -> Self { + Self { + key: key.into(), + value: AnyValue::Int { + int_value: value.to_string(), + }, + } + } + + /// Float-valued attribute. + #[must_use] + pub fn double(key: impl Into, value: f64) -> Self { + Self { + key: key.into(), + value: AnyValue::Double { + double_value: value, + }, + } + } + + /// Boolean-valued attribute. + #[must_use] + pub fn bool(key: impl Into, value: bool) -> Self { + Self { + key: key.into(), + value: AnyValue::Bool { bool_value: value }, + } + } + + /// String-array-valued attribute. + #[must_use] + pub fn string_array(key: impl Into, values: &[String]) -> Self { + Self { + key: key.into(), + value: AnyValue::Array { + array_value: ArrayValue { + values: values + .iter() + .map(|v| AnyValue::String { + string_value: v.clone(), + }) + .collect(), + }, + }, + } + } +} + +/// OTLP `AnyValue`. +#[derive(Debug, Clone, Serialize)] +#[serde(untagged)] +pub enum AnyValue { + /// UTF-8 string. + String { + /// The string. + #[serde(rename = "stringValue")] + string_value: String, + }, + /// 64-bit integer, encoded as a decimal string per OTLP/JSON. + Int { + /// The integer, decimal-encoded. + #[serde(rename = "intValue")] + int_value: String, + }, + /// Double-precision float. + Double { + /// The float. + #[serde(rename = "doubleValue")] + double_value: f64, + }, + /// Boolean. + Bool { + /// The boolean. + #[serde(rename = "boolValue")] + bool_value: bool, + }, + /// Homogeneous array. + Array { + /// The array. + #[serde(rename = "arrayValue")] + array_value: ArrayValue, + }, +} + +/// OTLP array value wrapper. +#[derive(Debug, Clone, Serialize)] +pub struct ArrayValue { + /// Array elements. + pub values: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn int_values_serialize_as_decimal_strings() { + // OTLP/JSON requires int64 as a string; emitting a JSON number here + // makes collectors reject or silently truncate large values. + let kv = KeyValue::int("gen_ai.usage.input_tokens", 9_007_199_254_740_993); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["value"]["intValue"], "9007199254740993"); + assert!(json["value"]["intValue"].is_string()); + } + + #[test] + fn string_values_serialize_under_string_value() { + let kv = KeyValue::string("service.name", "moltis"); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["key"], "service.name"); + assert_eq!(json["value"]["stringValue"], "moltis"); + } + + #[test] + fn bool_and_double_use_their_own_slots() { + let b = serde_json::to_value(KeyValue::bool("k", true)).expect("serializable"); + let d = serde_json::to_value(KeyValue::double("k", 0.5)).expect("serializable"); + + assert_eq!(b["value"]["boolValue"], true); + assert_eq!(d["value"]["doubleValue"], 0.5); + } + + #[test] + fn string_arrays_nest_under_array_value() { + let kv = KeyValue::string_array("langfuse.trace.tags", &["a".into(), "b".into()]); + let json = serde_json::to_value(&kv).expect("serializable"); + + assert_eq!(json["value"]["arrayValue"]["values"][0]["stringValue"], "a"); + assert_eq!(json["value"]["arrayValue"]["values"][1]["stringValue"], "b"); + } + + #[test] + fn unset_status_omits_the_message_field() { + let json = serde_json::to_value(Status::unset()).expect("serializable"); + assert_eq!(json["code"], 0); + assert!(json.get("message").is_none()); + } + + #[test] + fn error_status_carries_code_two_and_detail() { + let json = serde_json::to_value(Status::error(Some("boom".into()))).expect("serializable"); + assert_eq!(json["code"], 2); + assert_eq!(json["message"], "boom"); + } + + #[test] + fn root_span_omits_parent_span_id() { + let span = Span { + trace_id: "0".repeat(32), + span_id: "0".repeat(16), + parent_span_id: None, + name: "root".into(), + kind: SPAN_KIND_INTERNAL, + start_time_unix_nano: "1".into(), + end_time_unix_nano: "2".into(), + attributes: Vec::new(), + status: Status::unset(), + }; + let json = serde_json::to_value(&span).expect("serializable"); + + // A present-but-null parentSpanId makes some collectors treat the span + // as a child of a missing parent, orphaning the trace. + assert!(json.get("parentSpanId").is_none()); + } +} diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs new file mode 100644 index 000000000..950963a45 --- /dev/null +++ b/crates/observability/src/lib.rs @@ -0,0 +1,40 @@ +//! Agent instrumentation with pluggable backends. +//! +//! One instrumentation pass in the agent runtime feeds any number of backends +//! through [`sink::ObservationSink`]. What each backend actually receives is +//! governed by its [`profile::ExportProfile`], because an LLM observability +//! product and an infrastructure APM want very different things: +//! +//! * **Langfuse** gets the full conversation — prompts, completions, tool +//! arguments and results — plus the observation taxonomy, cache-aware token +//! usage, cost, and managed-prompt linkage. +//! * **Grafana, Datadog, Honeycomb** get operational shape only: latency, +//! errors, model, token counts. No conversation content, no per-user +//! cardinality. +//! +//! See `docs/src/instrumentation.md` for the operator-facing guide. + +pub mod builder; +pub mod exporters; +pub mod model; +pub mod profile; +pub mod recorder; +pub mod redact; +pub mod runtime; +pub mod sink; + +pub use { + builder::{BuildOutcome, BuiltInstrumentation, SkippedBackend, build}, + model::{ + Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, + TokenUsage, TraceId, TraceRecord, TraceScope, + }, + profile::{ContentCapture, ExportProfile, Vocabulary}, + recorder::{RecorderSettings, StepGuard, TurnRecorder}, + redact::RedactionPolicy, + runtime::{BatchConfig, BatchSink, SinkStatsSnapshot, Transport, TransportError}, + sink::{ + ObservationSink, SinkFanout, clear_global_sink, global_sink, is_enabled, record, + set_global_sink, + }, +}; diff --git a/crates/observability/src/model.rs b/crates/observability/src/model.rs new file mode 100644 index 000000000..5add8237e --- /dev/null +++ b/crates/observability/src/model.rs @@ -0,0 +1,590 @@ +//! Backend-agnostic trace model. +//! +//! The shape follows Langfuse's observation taxonomy because it is the richest +//! of the backends we target: OTLP exporters can always project a richer model +//! down onto spans, but the reverse loses tool/generation/retriever semantics. + +use std::collections::BTreeMap; + +use { + serde::{Deserialize, Serialize}, + time::OffsetDateTime, + uuid::Uuid, +}; + +// ── Identifiers ───────────────────────────────────────────────────────────── + +/// Trace identifier. One trace per agent run (one user turn). +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct TraceId(pub String); + +/// Observation identifier, unique within a trace. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct ObservationId(pub String); + +impl TraceId { + /// Generate a fresh random trace id. + #[must_use] + pub fn generate() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Lowercase 32-char hex form required by the OTLP wire format. + #[must_use] + pub fn as_otel_hex(&self) -> String { + otel_hex(&self.0, 32) + } +} + +impl ObservationId { + /// Generate a fresh random observation id. + #[must_use] + pub fn generate() -> Self { + Self(Uuid::new_v4().to_string()) + } + + /// Lowercase 16-char hex form required by the OTLP wire format. + #[must_use] + pub fn as_otel_hex(&self) -> String { + otel_hex(&self.0, 16) + } +} + +/// Project an arbitrary id string onto a fixed-width lowercase hex string. +/// +/// UUIDs already carry 32 hex digits once dashes are stripped, so the common +/// case is a pure filter. Non-UUID ids (or ids shorter than `width`) are padded +/// deterministically so the same input always yields the same OTLP id. +fn otel_hex(raw: &str, width: usize) -> String { + let mut hex: String = raw + .chars() + .filter(char::is_ascii_hexdigit) + .map(|c| c.to_ascii_lowercase()) + .take(width) + .collect(); + if hex.len() < width { + // Deterministic padding: repeat the digest of the raw id. + let filler = format!("{:016x}", fnv1a64(raw.as_bytes())); + while hex.len() < width { + let need = width - hex.len(); + let take = need.min(filler.len()); + hex.push_str(&filler[..take]); + } + } + hex +} + +/// FNV-1a 64-bit. Used only to derive deterministic padding for OTLP ids, never +/// for anything security-relevant. +fn fnv1a64(bytes: &[u8]) -> u64 { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + hash ^= u64::from(*b); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + hash +} + +// ── Taxonomy ──────────────────────────────────────────────────────────────── + +/// Observation kind. Mirrors Langfuse's `ObservationType` enum verbatim so the +/// exporter never has to guess at a mapping. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum ObservationKind { + /// Generic unit of work. + Span, + /// Point-in-time occurrence with no duration. + Event, + /// An LLM completion call. + Generation, + /// An agent run, possibly containing nested agents. + Agent, + /// A tool invocation. + Tool, + /// A fixed multi-step sequence. + Chain, + /// A retrieval/search step (memory, code index). + Retriever, + /// A quality-evaluation step. + Evaluator, + /// An embedding computation. + Embedding, + /// A policy check that may block execution. + Guardrail, +} + +impl ObservationKind { + /// Langfuse wire representation. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Span => "SPAN", + Self::Event => "EVENT", + Self::Generation => "GENERATION", + Self::Agent => "AGENT", + Self::Tool => "TOOL", + Self::Chain => "CHAIN", + Self::Retriever => "RETRIEVER", + Self::Evaluator => "EVALUATOR", + Self::Embedding => "EMBEDDING", + Self::Guardrail => "GUARDRAIL", + } + } + + /// Whether Langfuse treats this kind as generation-like, i.e. carrying + /// model, usage and cost fields. + #[must_use] + pub const fn is_generation_like(self) -> bool { + matches!( + self, + Self::Generation + | Self::Agent + | Self::Tool + | Self::Chain + | Self::Retriever + | Self::Evaluator + | Self::Embedding + | Self::Guardrail + ) + } +} + +/// Severity attached to an observation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "UPPERCASE")] +pub enum Level { + /// Verbose diagnostic detail. + Debug, + /// Normal operation. + #[default] + Default, + /// Recovered from something unexpected. + Warning, + /// Failed. + Error, +} + +impl Level { + /// Langfuse wire representation. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Debug => "DEBUG", + Self::Default => "DEFAULT", + Self::Warning => "WARNING", + Self::Error => "ERROR", + } + } +} + +// ── Usage and cost ────────────────────────────────────────────────────────── + +/// Token usage for a generation. +/// +/// `input` is the count of *fresh* (non-cached) input tokens. Callers that +/// receive a provider total which already includes cache reads must subtract +/// before constructing this type — see [`TokenUsage::from_provider_totals`]. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TokenUsage { + /// Fresh input tokens. + pub input: u32, + /// Generated output tokens. + pub output: u32, + /// Tokens served from the provider's prompt cache. + pub cache_read: u32, + /// Tokens written into the provider's prompt cache. + pub cache_write: u32, + /// Reasoning tokens, when the provider reports them separately. + pub reasoning: u32, +} + +impl TokenUsage { + /// Build usage from provider counters where `input` is already exclusive of + /// cached tokens (Anthropic's `input_tokens` behaves this way). + #[must_use] + pub const fn from_provider_totals( + input: u32, + output: u32, + cache_read: u32, + cache_write: u32, + ) -> Self { + Self { + input, + output, + cache_read, + cache_write, + reasoning: 0, + } + } + + /// Total tokens across every bucket. + #[must_use] + pub const fn total(&self) -> u32 { + self.input + .saturating_add(self.output) + .saturating_add(self.cache_read) + .saturating_add(self.cache_write) + } + + /// Whether every counter is zero, in which case the field is omitted from + /// the payload rather than reporting a misleading zero-token generation. + #[must_use] + pub const fn is_empty(&self) -> bool { + self.input == 0 + && self.output == 0 + && self.cache_read == 0 + && self.cache_write == 0 + && self.reasoning == 0 + } + + /// Langfuse `usageDetails` map. + /// + /// Key names match what Langfuse's ingestion recognises for cache-aware + /// cost attribution. + #[must_use] + pub fn to_usage_details(&self) -> BTreeMap { + let mut map = BTreeMap::new(); + map.insert("input".to_string(), self.input); + map.insert("output".to_string(), self.output); + if self.cache_read > 0 { + map.insert("input_cache_read".to_string(), self.cache_read); + } + if self.cache_write > 0 { + map.insert("input_cache_write".to_string(), self.cache_write); + } + if self.reasoning > 0 { + map.insert("output_reasoning_tokens".to_string(), self.reasoning); + } + map.insert("total".to_string(), self.total()); + map + } +} + +// ── Records ───────────────────────────────────────────────────────────────── + +/// Free-form metadata attached to traces and observations. +/// +/// `BTreeMap` keeps key order stable, which makes payload assertions in tests +/// deterministic. +pub type Metadata = BTreeMap; + +/// Trace-scoped attributes. +/// +/// These are duplicated onto every observation rather than held only on the +/// trace. OTLP has no trace-level attribute concept — backends reconstruct +/// trace metadata from the spans — so an exporter that kept this only on the +/// trace record would lose user and session attribution whenever a trace event +/// was dropped, batched separately, or evicted from a cache. Duplication costs +/// a few bytes per span and makes exporters stateless. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct TraceScope { + /// Groups related traces in the backend's session view. + pub session_id: Option, + /// End-user identity, when known. + pub user_id: Option, + /// Filterable labels. + pub tags: Vec, + /// Deployment environment (`production`, `staging`, ...). + pub environment: Option, + /// Build release identifier. + pub release: Option, + /// Application-level version, e.g. an agent preset version. + pub version: Option, +} + +/// A trace: one agent run, corresponding to one user turn. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TraceRecord { + /// Unique trace id. + pub id: TraceId, + /// Human-readable trace name. + pub name: String, + /// Wall-clock start. + #[serde(with = "time::serde::rfc3339")] + pub timestamp: OffsetDateTime, + /// Trace-scoped attributes. + pub scope: TraceScope, + /// Structured context. + pub metadata: Metadata, + /// Turn input, subject to redaction and capture settings. + pub input: Option, + /// Turn output, subject to redaction and capture settings. + pub output: Option, + /// Whether the backend should expose this trace publicly. + pub public: bool, +} + +impl TraceRecord { + /// Start a new trace with the given name at the current instant. + #[must_use] + pub fn new(name: impl Into) -> Self { + Self { + id: TraceId::generate(), + name: name.into(), + timestamp: OffsetDateTime::now_utc(), + scope: TraceScope::default(), + metadata: Metadata::new(), + input: None, + output: None, + public: false, + } + } +} + +/// A single observation within a trace. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ObservationRecord { + /// Unique observation id. + pub id: ObservationId, + /// Owning trace. + pub trace_id: TraceId, + /// Enclosing observation, when nested. + pub parent_id: Option, + /// What kind of work this represents. + pub kind: ObservationKind, + /// Human-readable name. + pub name: String, + /// Trace-scoped attributes, copied from the owning trace. + pub scope: TraceScope, + /// Wall-clock start. + #[serde(with = "time::serde::rfc3339")] + pub start_time: OffsetDateTime, + /// Wall-clock end; `None` while still running. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, + /// Time the first output token arrived, for streaming generations. + #[serde(with = "time::serde::rfc3339::option")] + pub completion_start_time: Option, + /// Severity. + pub level: Level, + /// Error or warning detail. + pub status_message: Option, + /// Step input, subject to redaction and capture settings. + pub input: Option, + /// Step output, subject to redaction and capture settings. + pub output: Option, + /// Structured context. + pub metadata: Metadata, + /// Model identifier, for generation-like observations. + pub model: Option, + /// Sampling parameters, for generation-like observations. + pub model_parameters: Metadata, + /// Token usage, for generation-like observations. + pub usage: Option, + /// Cost breakdown in USD, when locally known. + pub cost_details: BTreeMap, + /// Managed-prompt name this generation was rendered from. + pub prompt_name: Option, + /// Managed-prompt version this generation was rendered from. + pub prompt_version: Option, +} + +impl ObservationRecord { + /// Start an observation of `kind` in `trace_id` at the current instant. + #[must_use] + pub fn start(trace_id: TraceId, kind: ObservationKind, name: impl Into) -> Self { + Self { + id: ObservationId::generate(), + trace_id, + parent_id: None, + kind, + name: name.into(), + scope: TraceScope::default(), + start_time: OffsetDateTime::now_utc(), + end_time: None, + completion_start_time: None, + level: Level::Default, + status_message: None, + input: None, + output: None, + metadata: Metadata::new(), + model: None, + model_parameters: Metadata::new(), + usage: None, + cost_details: BTreeMap::new(), + prompt_name: None, + prompt_version: None, + } + } + + /// Set the parent observation, producing a nested observation. + #[must_use] + pub fn with_parent(mut self, parent: Option) -> Self { + self.parent_id = parent; + self + } + + /// Copy trace-scoped attributes onto this observation. + #[must_use] + pub fn with_scope(mut self, scope: TraceScope) -> Self { + self.scope = scope; + self + } + + /// Stamp the end time as now. + pub fn finish(&mut self) { + self.end_time = Some(OffsetDateTime::now_utc()); + } + + /// Mark this observation as failed with the given message. + pub fn fail(&mut self, message: impl Into) { + self.level = Level::Error; + self.status_message = Some(message.into()); + } +} + +/// Kinds of score value a backend accepts. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ScoreValue { + /// Numeric score. Also carries boolean scores as 0.0/1.0. + Numeric(f64), + /// Categorical score, e.g. "helpful". + Categorical(String), +} + +/// A quality score attached to a trace or observation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ScoreRecord { + /// Unique score id. + pub id: String, + /// Trace the score applies to. + pub trace_id: TraceId, + /// Specific observation, when the score is narrower than the whole trace. + pub observation_id: Option, + /// Score name, e.g. `user-feedback`. + pub name: String, + /// The score itself. + pub value: ScoreValue, + /// Optional rationale. + pub comment: Option, + /// Deployment environment. + pub environment: Option, +} + +impl ScoreRecord { + /// Build a score with a generated id. + #[must_use] + pub fn new(trace_id: TraceId, name: impl Into, value: ScoreValue) -> Self { + Self { + id: Uuid::new_v4().to_string(), + trace_id, + observation_id: None, + name: name.into(), + value, + comment: None, + environment: None, + } + } +} + +/// A single unit of work handed to a sink. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum Event { + /// Create or update a trace. + Trace(Box), + /// An observation began. Emitted eagerly so long runs are visible live. + ObservationStart(Box), + /// An observation finished. + ObservationEnd(Box), + /// A score was produced. + Score(Box), +} + +impl Event { + /// The trace this event belongs to. + #[must_use] + pub fn trace_id(&self) -> &TraceId { + match self { + Self::Trace(t) => &t.id, + Self::ObservationStart(o) | Self::ObservationEnd(o) => &o.trace_id, + Self::Score(s) => &s.trace_id, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn usage_details_split_cache_buckets() { + let usage = TokenUsage::from_provider_totals(100, 50, 900, 20); + let details = usage.to_usage_details(); + + // `input` must stay exclusive of cache buckets: Langfuse prices + // cache reads and writes at different rates than fresh input, so + // folding them together silently inflates the reported cost. + assert_eq!(details.get("input"), Some(&100)); + assert_eq!(details.get("output"), Some(&50)); + assert_eq!(details.get("input_cache_read"), Some(&900)); + assert_eq!(details.get("input_cache_write"), Some(&20)); + assert_eq!(details.get("total"), Some(&1070)); + } + + #[test] + fn usage_details_omit_zero_cache_buckets() { + let usage = TokenUsage::from_provider_totals(10, 5, 0, 0); + let details = usage.to_usage_details(); + + assert!(!details.contains_key("input_cache_read")); + assert!(!details.contains_key("input_cache_write")); + assert_eq!(details.get("total"), Some(&15)); + } + + #[test] + fn usage_total_saturates_instead_of_overflowing() { + let usage = TokenUsage::from_provider_totals(u32::MAX, u32::MAX, 0, 0); + assert_eq!(usage.total(), u32::MAX); + } + + #[test] + fn trace_id_renders_as_32_hex_digits() { + let hex = TraceId::generate().as_otel_hex(); + assert_eq!(hex.len(), 32); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn observation_id_renders_as_16_hex_digits() { + let hex = ObservationId::generate().as_otel_hex(); + assert_eq!(hex.len(), 16); + assert!(hex.chars().all(|c| c.is_ascii_hexdigit())); + } + + #[test] + fn otel_hex_is_deterministic_for_non_uuid_ids() { + let id = ObservationId("not-a-uuid".to_string()); + assert_eq!(id.as_otel_hex(), id.as_otel_hex()); + assert_eq!(id.as_otel_hex().len(), 16); + } + + #[test] + fn otel_hex_pads_short_ids_to_full_width() { + // "ab" contributes two hex digits; the rest must be filled. + let id = TraceId("ab".to_string()); + let hex = id.as_otel_hex(); + assert_eq!(hex.len(), 32); + assert!(hex.starts_with("ab")); + } + + #[test] + fn generation_like_kinds_match_langfuse() { + assert!(ObservationKind::Generation.is_generation_like()); + assert!(ObservationKind::Tool.is_generation_like()); + assert!(ObservationKind::Agent.is_generation_like()); + assert!(!ObservationKind::Span.is_generation_like()); + assert!(!ObservationKind::Event.is_generation_like()); + } + + #[test] + fn observation_fail_sets_error_level_and_message() { + let mut obs = + ObservationRecord::start(TraceId::generate(), ObservationKind::Generation, "llm-call"); + obs.fail("rate limited"); + + assert_eq!(obs.level, Level::Error); + assert_eq!(obs.status_message.as_deref(), Some("rate limited")); + } +} diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs new file mode 100644 index 000000000..126d1e20f --- /dev/null +++ b/crates/observability/src/profile.rs @@ -0,0 +1,213 @@ +//! Per-backend export profiles. +//! +//! Langfuse and an infrastructure APM want fundamentally different things from +//! the same agent run, so they must not receive the same payload: +//! +//! * **Langfuse** is an LLM-native product. It wants the whole conversation — +//! prompts, completions, tool arguments and results — plus the observation +//! taxonomy, token usage split by cache bucket, per-turn cost, managed-prompt +//! versions, and session/user attribution. Content *is* the product. +//! +//! * **Datadog, Grafana Tempo, Honeycomb** are operational tools. They want +//! latency, error rates and throughput. Shipping prompt bodies to them is +//! actively harmful: it explodes span size and cardinality, most vendors bill +//! per ingested span-byte or custom metric, and it copies user conversation +//! content into a system that was never scoped to hold it. For these backends +//! the useful signal is the *shape* of the run — how long, how many +//! iterations, which model, did it error — not what was said. +//! +//! A profile is therefore attached to each sink, and the mapping layer consults +//! it before emitting anything. + +use serde::{Deserialize, Serialize}; + +/// How much conversation content an exporter may carry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ContentCapture { + /// Export inputs, outputs, tool arguments and tool results. + Full, + /// Export structural metadata only: sizes, counts, model, timings. + #[default] + MetadataOnly, + /// Export neither content nor content-derived metadata. + None, +} + +impl ContentCapture { + /// Whether payload bodies may be exported. + #[must_use] + pub const fn allows_bodies(self) -> bool { + matches!(self, Self::Full) + } +} + +/// Which attribute vocabularies an exporter emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Vocabulary { + /// Langfuse's own `langfuse.*` mapping, plus `gen_ai.*`. + Langfuse, + /// OpenTelemetry GenAI semantic conventions only. + GenAi, +} + +/// What a given backend receives. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct ExportProfile { + /// Content policy. + pub content: ContentCapture, + /// Attribute vocabulary. + pub vocabulary: Vocabulary, + /// Whether to attach end-user identity. + /// + /// High-cardinality in an APM's index and often a compliance question; + /// essential in Langfuse, where per-user analysis is a core view. + pub emit_user_id: bool, + /// Whether to attach session identity. + pub emit_session_id: bool, + /// Whether to attach free-form tags. + pub emit_tags: bool, + /// Whether to attach token usage and cost. + /// + /// Langfuse prices these; an APM generally has no model price table, so + /// the same numbers are better served from the Prometheus endpoint. + pub emit_usage: bool, + /// Ceiling on any single exported attribute value, in bytes. + pub max_attribute_bytes: usize, +} + +impl ExportProfile { + /// Profile for Langfuse: everything, because the product is built on it. + #[must_use] + pub const fn langfuse() -> Self { + Self { + content: ContentCapture::Full, + vocabulary: Vocabulary::Langfuse, + emit_user_id: true, + emit_session_id: true, + emit_tags: true, + emit_usage: true, + max_attribute_bytes: 32_768, + } + } + + /// Profile for a generic OTel backend (Grafana Tempo, Honeycomb, an + /// OpenTelemetry Collector): operational shape only, no conversation + /// content, no per-user cardinality. + #[must_use] + pub const fn otel_generic() -> Self { + Self { + content: ContentCapture::MetadataOnly, + vocabulary: Vocabulary::GenAi, + emit_user_id: false, + emit_session_id: true, + emit_tags: true, + emit_usage: true, + max_attribute_bytes: 4_096, + } + } + + /// Profile for Datadog APM. As [`Self::otel_generic`], but tags are dropped + /// too: Datadog indexes span tags and bills on custom metric cardinality, + /// so an unbounded tag set is a billing surprise waiting to happen. + #[must_use] + pub const fn datadog() -> Self { + Self { + content: ContentCapture::MetadataOnly, + vocabulary: Vocabulary::GenAi, + emit_user_id: false, + emit_session_id: true, + emit_tags: false, + emit_usage: true, + max_attribute_bytes: 4_096, + } + } + + /// Whether `langfuse.*` attributes should be written. + #[must_use] + pub const fn emits_langfuse_attrs(&self) -> bool { + matches!(self.vocabulary, Vocabulary::Langfuse) + } + + /// Whether payload bodies may be written. + #[must_use] + pub const fn emits_bodies(&self) -> bool { + self.content.allows_bodies() + } + + /// Whether content-derived structural metadata (lengths, counts) may be + /// written. Suppressed only under [`ContentCapture::None`]. + #[must_use] + pub const fn emits_content_metadata(&self) -> bool { + !matches!(self.content, ContentCapture::None) + } +} + +impl Default for ExportProfile { + fn default() -> Self { + Self::otel_generic() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn langfuse_profile_carries_conversation_content() { + let profile = ExportProfile::langfuse(); + assert!(profile.emits_bodies()); + assert!(profile.emits_langfuse_attrs()); + assert!(profile.emit_user_id); + assert!(profile.emit_usage); + } + + #[test] + fn generic_otel_profile_withholds_conversation_content() { + // The whole point of the split: prompt bodies must not reach an APM. + let profile = ExportProfile::otel_generic(); + assert!(!profile.emits_bodies()); + assert!(!profile.emits_langfuse_attrs()); + assert!( + !profile.emit_user_id, + "user id is high-cardinality in an APM" + ); + // Structural metadata is still useful for latency and error analysis. + assert!(profile.emits_content_metadata()); + } + + #[test] + fn datadog_profile_also_drops_tags() { + let profile = ExportProfile::datadog(); + assert!(!profile.emits_bodies()); + assert!(!profile.emit_tags, "Datadog bills on tag cardinality"); + assert!(profile.emit_session_id); + } + + #[test] + fn content_none_suppresses_even_derived_metadata() { + let profile = ExportProfile { + content: ContentCapture::None, + ..ExportProfile::otel_generic() + }; + assert!(!profile.emits_bodies()); + assert!(!profile.emits_content_metadata()); + } + + #[test] + fn default_profile_is_the_conservative_one() { + // Defaulting to Full would leak conversation content to any newly + // configured backend that forgot to set a profile. + assert_eq!(ExportProfile::default(), ExportProfile::otel_generic()); + assert!(!ExportProfile::default().emits_bodies()); + } + + #[test] + fn langfuse_allows_larger_attributes_than_an_apm() { + assert!( + ExportProfile::langfuse().max_attribute_bytes + > ExportProfile::otel_generic().max_attribute_bytes + ); + } +} diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs new file mode 100644 index 000000000..e55f49cd2 --- /dev/null +++ b/crates/observability/src/recorder.rs @@ -0,0 +1,754 @@ +//! The instrumentation API used by the agent runtime. +//! +//! Callers hold a [`TurnRecorder`] for the duration of an agent run and open a +//! [`StepGuard`] around each LLM call, tool invocation or retrieval. The +//! recorder owns id generation, parent/child nesting, trace-scope propagation +//! and redaction, so instrumentation at the call site stays to a few lines. +//! +//! Every entry point is a no-op when no sink is installed, and the constructor +//! returns `None` in that case so callers pay nothing beyond one atomic read. + +use std::sync::{Arc, Mutex}; + +use time::OffsetDateTime; + +use crate::{ + model::{ + Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, + TokenUsage, TraceId, TraceRecord, TraceScope, + }, + redact::RedactionPolicy, + sink::{self, ObservationSink}, +}; + +/// Settings that shape what a recorder emits. +#[derive(Debug, Clone)] +pub struct RecorderSettings { + /// Redaction applied to every payload before it is handed to a sink. + pub redaction: RedactionPolicy, + /// Whether to attach turn and step inputs. + pub capture_input: bool, + /// Whether to attach turn and step outputs. + pub capture_output: bool, + /// Whether to attach tool arguments and results. + pub capture_tool_io: bool, + /// Fraction of turns to trace, in `0.0..=1.0`. + pub sample_rate: f64, +} + +impl Default for RecorderSettings { + fn default() -> Self { + Self { + redaction: RedactionPolicy::default(), + capture_input: true, + capture_output: true, + capture_tool_io: true, + sample_rate: 1.0, + } + } +} + +impl RecorderSettings { + /// Whether this turn should be traced, given the sample rate. + #[must_use] + fn sampled(&self) -> bool { + if self.sample_rate >= 1.0 { + return true; + } + if self.sample_rate <= 0.0 { + return false; + } + rand::random::() < self.sample_rate + } +} + +/// Records one agent run. +pub struct TurnRecorder { + sink: Arc, + settings: RecorderSettings, + trace_id: TraceId, + scope: TraceScope, + /// The run's root observation, parent of every step. + root_id: ObservationId, + /// Trace record, retained so the closing update carries the final output. + trace: Mutex, +} + +impl TurnRecorder { + /// Begin recording a turn, or return `None` when instrumentation is off or + /// the turn was not sampled. + #[must_use] + pub fn begin( + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + ) -> Option { + let sink = sink::global_sink()?; + if !settings.sampled() { + return None; + } + + let mut trace = TraceRecord::new(name); + trace.scope = scope.clone(); + let trace_id = trace.id.clone(); + + // The root observation shares the trace id so exporters can parent + // orphan steps onto it without extra bookkeeping. + let root_id = ObservationId(trace_id.0.clone()); + + let recorder = Self { + sink, + settings, + trace_id: trace_id.clone(), + scope, + root_id, + trace: Mutex::new(trace.clone()), + }; + recorder.sink.record(Event::Trace(Box::new(trace))); + Some(recorder) + } + + /// The trace being recorded, for correlating scores later. + #[must_use] + pub fn trace_id(&self) -> &TraceId { + &self.trace_id + } + + /// Attach the turn's input. + pub fn set_input(&self, input: serde_json::Value) { + if !self.settings.capture_input { + return; + } + let redacted = self.settings.redaction.redact(&input); + self.with_trace(|trace| trace.input = Some(redacted)); + } + + /// Attach the turn's output. + pub fn set_output(&self, output: serde_json::Value) { + if !self.settings.capture_output { + return; + } + let redacted = self.settings.redaction.redact(&output); + self.with_trace(|trace| trace.output = Some(redacted)); + } + + /// Attach a metadata entry to the trace. + pub fn set_metadata(&self, key: impl Into, value: serde_json::Value) { + self.with_trace(|trace| { + trace.metadata.insert(key.into(), value); + }); + } + + /// Open a step nested under the run's root. + #[must_use] + pub fn step(&self, kind: ObservationKind, name: impl Into) -> StepGuard { + self.step_under(kind, name, Some(self.root_id.clone())) + } + + /// Open a step nested under an explicit parent, for sub-agents and tools + /// invoked from within another step. + #[must_use] + pub fn step_under( + &self, + kind: ObservationKind, + name: impl Into, + parent: Option, + ) -> StepGuard { + let record = ObservationRecord::start(self.trace_id.clone(), kind, name) + .with_parent(parent.or_else(|| Some(self.root_id.clone()))) + .with_scope(self.scope.clone()); + + // Emit the start eagerly so a long-running turn is visible in the + // backend while it is still executing, rather than only at the end. + self.sink + .record(Event::ObservationStart(Box::new(record.clone()))); + + StepGuard { + sink: Arc::clone(&self.sink), + settings: self.settings.clone(), + record: Some(record), + } + } + + /// Record a score against this turn. + pub fn score(&self, name: impl Into, value: ScoreValue, comment: Option) { + let mut score = ScoreRecord::new(self.trace_id.clone(), name, value); + score.comment = comment; + score.environment = self.scope.environment.clone(); + self.sink.record(Event::Score(Box::new(score))); + } + + /// Close the turn, emitting the final trace state. + pub fn finish(self) { + let trace = self.snapshot_trace(); + self.sink.record(Event::Trace(Box::new(trace))); + } + + /// Close the turn as failed. + pub fn finish_with_error(self, message: impl Into) { + self.set_metadata("error", serde_json::Value::String(message.into())); + self.finish(); + } + + fn with_trace(&self, f: impl FnOnce(&mut TraceRecord)) { + let mut guard = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f(&mut guard); + } + + fn snapshot_trace(&self) -> TraceRecord { + self.trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } +} + +/// An open observation. Emits its completion on drop, so an early `?` return +/// still produces a closed span rather than one that hangs open forever. +pub struct StepGuard { + sink: Arc, + settings: RecorderSettings, + /// `None` once the guard has emitted, so drop does not emit twice. + record: Option, +} + +impl StepGuard { + /// This step's id, for nesting children beneath it. + #[must_use] + pub fn id(&self) -> Option { + self.record.as_ref().map(|r| r.id.clone()) + } + + /// Attach the step's input. + pub fn set_input(&mut self, input: serde_json::Value) { + if !self.capture_input_allowed() { + return; + } + let redacted = self.settings.redaction.redact(&input); + if let Some(record) = self.record.as_mut() { + record.input = Some(redacted); + } + } + + /// Attach the step's output. + pub fn set_output(&mut self, output: serde_json::Value) { + if !self.capture_output_allowed() { + return; + } + let redacted = self.settings.redaction.redact(&output); + if let Some(record) = self.record.as_mut() { + record.output = Some(redacted); + } + } + + /// Set the model this step used. + pub fn set_model(&mut self, model: impl Into) { + if let Some(record) = self.record.as_mut() { + record.model = Some(model.into()); + } + } + + /// Set a sampling parameter. + pub fn set_model_parameter(&mut self, key: impl Into, value: serde_json::Value) { + if let Some(record) = self.record.as_mut() { + record.model_parameters.insert(key.into(), value); + } + } + + /// Set token usage. + pub fn set_usage(&mut self, usage: TokenUsage) { + if let Some(record) = self.record.as_mut() { + record.usage = Some(usage); + } + } + + /// Mark the instant the first output token arrived. + /// + /// Ignored if called twice: only the first token defines time-to-first-token. + pub fn mark_first_token(&mut self) { + if let Some(record) = self.record.as_mut() + && record.completion_start_time.is_none() + { + record.completion_start_time = Some(OffsetDateTime::now_utc()); + } + } + + /// Link this generation to a managed prompt version. + pub fn set_prompt(&mut self, name: impl Into, version: i32) { + if let Some(record) = self.record.as_mut() { + record.prompt_name = Some(name.into()); + record.prompt_version = Some(version); + } + } + + /// Attach a metadata entry. + pub fn set_metadata(&mut self, key: impl Into, value: serde_json::Value) { + if let Some(record) = self.record.as_mut() { + record.metadata.insert(key.into(), value); + } + } + + /// Raise the severity of this step. + pub fn set_level(&mut self, level: Level, message: Option) { + if let Some(record) = self.record.as_mut() { + record.level = level; + record.status_message = message; + } + } + + /// Mark the step failed. The span is still emitted on drop. + pub fn fail(&mut self, message: impl Into) { + if let Some(record) = self.record.as_mut() { + record.fail(message); + } + } + + /// Emit the completed observation now rather than at drop. + pub fn finish(mut self) { + self.emit(); + } + + fn emit(&mut self) { + let Some(mut record) = self.record.take() else { + return; + }; + record.finish(); + self.sink.record(Event::ObservationEnd(Box::new(record))); + } + + /// Tool arguments are gated by their own switch as well as the input + /// switch, since they are the most likely place for credentials to appear. + fn capture_input_allowed(&self) -> bool { + match self.record.as_ref().map(|r| r.kind) { + Some(ObservationKind::Tool) => self.settings.capture_tool_io, + _ => self.settings.capture_input, + } + } + + fn capture_output_allowed(&self) -> bool { + match self.record.as_ref().map(|r| r.kind) { + Some(ObservationKind::Tool) => self.settings.capture_tool_io, + _ => self.settings.capture_output, + } + } +} + +impl Drop for StepGuard { + fn drop(&mut self) { + // An early return through `?` must still close the span; otherwise the + // backend shows a step that never ended. + self.emit(); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex as StdMutex; + + use {async_trait::async_trait, serde_json::json}; + + use super::*; + + struct CollectingSink { + events: StdMutex>, + } + + impl CollectingSink { + fn new() -> Arc { + Arc::new(Self { + events: StdMutex::new(Vec::new()), + }) + } + + fn events(&self) -> Vec { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + fn observations(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|e| match e { + Event::ObservationEnd(o) => Some(*o), + _ => None, + }) + .collect() + } + + fn traces(&self) -> Vec { + self.events() + .into_iter() + .filter_map(|e| match e { + Event::Trace(t) => Some(*t), + _ => None, + }) + .collect() + } + } + + #[async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + "collecting" + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: std::time::Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + /// Serialises tests that touch the process-wide sink registry. + static GLOBAL_LOCK: StdMutex<()> = StdMutex::new(()); + + fn with_sink(f: impl FnOnce(Arc) -> R) -> R { + let _guard = GLOBAL_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let collected = CollectingSink::new(); + sink::set_global_sink(collected.clone()); + let out = f(collected); + sink::clear_global_sink(); + out + } + + fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:main".into()), + user_id: Some("telegram:42".into()), + tags: vec!["telegram".into()], + environment: Some("production".into()), + release: None, + version: None, + } + } + + #[test] + fn returns_none_when_no_sink_is_installed() { + let _guard = GLOBAL_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + sink::clear_global_sink(); + + // Instrumentation must cost nothing when switched off. + assert!(TurnRecorder::begin("turn", scope(), RecorderSettings::default()).is_none()); + } + + #[test] + fn zero_sample_rate_skips_the_turn_entirely() { + with_sink(|collected| { + let settings = RecorderSettings { + sample_rate: 0.0, + ..Default::default() + }; + assert!(TurnRecorder::begin("turn", scope(), settings).is_none()); + assert!(collected.events().is_empty()); + }); + } + + #[test] + fn steps_are_parented_to_the_run_root() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let root = ObservationId(recorder.trace_id().0.clone()); + + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + + let observations = collected.observations(); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].parent_id, Some(root)); + }); + } + + #[test] + fn nested_steps_are_parented_to_their_enclosing_step() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let outer = recorder.step(ObservationKind::Agent, "sub-agent"); + let outer_id = outer.id().expect("open step has an id"); + recorder + .step_under(ObservationKind::Tool, "exec", Some(outer_id.clone())) + .finish(); + outer.finish(); + recorder.finish(); + + let tool = collected + .observations() + .into_iter() + .find(|o| o.kind == ObservationKind::Tool) + .expect("tool observation emitted"); + assert_eq!(tool.parent_id, Some(outer_id)); + }); + } + + #[test] + fn steps_carry_the_trace_scope() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert_eq!( + observed.scope.session_id.as_deref(), + Some("agent:main:main") + ); + assert_eq!(observed.scope.user_id.as_deref(), Some("telegram:42")); + }); + } + + #[test] + fn a_start_event_is_emitted_before_the_step_completes() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let step = recorder.step(ObservationKind::Generation, "llm"); + + // A multi-minute turn should be visible while it is still running. + let starts = collected + .events() + .into_iter() + .filter(|e| matches!(e, Event::ObservationStart(_))) + .count(); + assert_eq!(starts, 1); + + step.finish(); + recorder.finish(); + }); + } + + #[test] + fn dropping_a_step_still_closes_it() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + // Simulates an early `?` return out of the agent loop. + drop(recorder.step(ObservationKind::Tool, "exec")); + recorder.finish(); + + let observations = collected.observations(); + assert_eq!(observations.len(), 1, "dropped step must still be emitted"); + assert!(observations[0].end_time.is_some()); + }); + } + + #[test] + fn finishing_a_step_does_not_emit_it_twice() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.step(ObservationKind::Tool, "exec").finish(); + recorder.finish(); + + assert_eq!(collected.observations().len(), 1); + }); + } + + #[test] + fn payloads_are_redacted_before_reaching_the_sink() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_input(json!({ "api_key": "abc123", "cmd": "ls" })); + step.finish(); + recorder.set_input(json!({ "password": "hunter2" })); + recorder.finish(); + + let tool_input = collected.observations()[0] + .input + .clone() + .expect("input captured"); + assert_eq!(tool_input["api_key"], json!("[REDACTED]")); + assert_eq!(tool_input["cmd"], json!("ls")); + + let trace = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!( + trace.input.expect("trace input")["password"], + json!("[REDACTED]") + ); + }); + } + + #[test] + fn capture_switches_suppress_payloads() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_input: false, + capture_output: false, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_input(json!({ "cmd": "ls" })); + step.set_output(json!("file list")); + step.finish(); + recorder.set_input(json!("hello")); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert!(observed.input.is_none()); + assert!(observed.output.is_none()); + assert!(collected.traces().last().expect("trace").input.is_none()); + }); + } + + #[test] + fn tool_io_switch_is_independent_of_the_generation_switches() { + with_sink(|collected| { + // Tool arguments are the likeliest place for credentials, so they + // must be suppressible without losing LLM inputs. + let settings = RecorderSettings { + capture_input: true, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + + let mut tool = recorder.step(ObservationKind::Tool, "exec"); + tool.set_input(json!({ "cmd": "ls" })); + tool.finish(); + + let mut generation = recorder.step(ObservationKind::Generation, "llm"); + generation.set_input(json!("hello")); + generation.finish(); + recorder.finish(); + + let observations = collected.observations(); + let tool = observations + .iter() + .find(|o| o.kind == ObservationKind::Tool) + .expect("tool emitted"); + let generation = observations + .iter() + .find(|o| o.kind == ObservationKind::Generation) + .expect("generation emitted"); + + assert!(tool.input.is_none()); + assert!(generation.input.is_some()); + }); + } + + #[test] + fn first_token_marker_records_only_the_first_call() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.mark_first_token(); + let first = step + .record + .as_ref() + .and_then(|r| r.completion_start_time) + .expect("first token recorded"); + step.mark_first_token(); + let second = step + .record + .as_ref() + .and_then(|r| r.completion_start_time) + .expect("still recorded"); + step.finish(); + recorder.finish(); + + assert_eq!(first, second, "time-to-first-token must not drift"); + assert!(collected.observations()[0].completion_start_time.is_some()); + }); + } + + #[test] + fn failed_steps_carry_error_level_and_message() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.fail("provider returned 500"); + step.finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert_eq!(observed.level, Level::Error); + assert_eq!( + observed.status_message.as_deref(), + Some("provider returned 500") + ); + }); + } + + #[test] + fn scores_are_emitted_against_the_turn() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + let trace_id = recorder.trace_id().clone(); + + recorder.score( + "user-feedback", + ScoreValue::Numeric(1.0), + Some("helpful".into()), + ); + recorder.finish(); + + let score = collected + .events() + .into_iter() + .find_map(|e| match e { + Event::Score(s) => Some(*s), + _ => None, + }) + .expect("score emitted"); + + assert_eq!(score.trace_id, trace_id); + assert_eq!(score.environment.as_deref(), Some("production")); + }); + } + + #[test] + fn the_closing_trace_carries_output_set_during_the_turn() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_output(json!("final answer")); + recorder.finish(); + + let last = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!(last.output, Some(json!("final answer"))); + }); + } + + #[test] + fn finish_with_error_records_the_failure_on_the_trace() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.finish_with_error("context window exceeded"); + + let last = collected.traces().last().cloned().expect("trace emitted"); + assert_eq!( + last.metadata.get("error"), + Some(&json!("context window exceeded")) + ); + }); + } +} diff --git a/crates/observability/src/redact.rs b/crates/observability/src/redact.rs new file mode 100644 index 000000000..eda83ee15 --- /dev/null +++ b/crates/observability/src/redact.rs @@ -0,0 +1,319 @@ +//! Redaction of sensitive values before export. +//! +//! Instrumentation ships conversation content, tool arguments and tool results +//! to a third party, so redaction is a security control, not a nicety. Two +//! independent passes run over every payload: +//! +//! 1. **Key-based**: any object key containing a configured needle (`password`, +//! `token`, ...) has its value replaced wholesale. +//! 2. **Shape-based**: string values that look like credentials (provider key +//! prefixes, bearer tokens, PEM private keys) are replaced even when the key +//! name is innocuous, because tool results are unstructured. +//! +//! Both passes are applied to keys and values at every depth. Redaction failing +//! open would be a silent data leak, so unknown structures are traversed rather +//! than skipped. + +use serde_json::Value; + +/// Replacement text substituted for redacted values. +pub const REDACTED: &str = "[REDACTED]"; + +/// Marker appended to truncated strings. +const TRUNCATION_SUFFIX: &str = "…[truncated]"; + +/// Default key needles. Matched case-insensitively as substrings. +pub const DEFAULT_KEY_NEEDLES: &[&str] = &[ + "password", + "passwd", + "secret", + "token", + "api_key", + "apikey", + "authorization", + "credential", + "private_key", + "session_key", + "access_key", +]; + +/// Credential-looking string prefixes, matched case-sensitively because real +/// key formats are case-sensitive and lowering would cause false positives. +const SECRET_PREFIXES: &[&str] = &[ + "sk-", + "sk_live_", + "sk_test_", + "pk_live_", + "rk_live_", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "github_pat_", + "xoxb-", + "xoxp-", + "xoxa-", + "xapp-", + "AIza", + "ya29.", + "AKIA", + "ASIA", + "hf_", + "lf_", + "Bearer ", + "bearer ", + "-----BEGIN", +]; + +/// Policy controlling what leaves the process. +#[derive(Debug, Clone)] +pub struct RedactionPolicy { + /// Object keys whose values are always replaced. + key_needles: Vec, + /// Whether to also scan string values for credential shapes. + scan_values: bool, + /// Maximum length of any single string before truncation. `0` disables. + max_string_len: usize, +} + +impl Default for RedactionPolicy { + fn default() -> Self { + Self { + key_needles: DEFAULT_KEY_NEEDLES + .iter() + .map(|s| (*s).to_string()) + .collect(), + scan_values: true, + max_string_len: 32_768, + } + } +} + +impl RedactionPolicy { + /// Build a policy from configured needles, merging in the defaults so an + /// operator cannot accidentally disable baseline protection by supplying a + /// narrower list. + #[must_use] + pub fn from_needles(needles: &[String]) -> Self { + let mut key_needles: Vec = DEFAULT_KEY_NEEDLES + .iter() + .map(|s| (*s).to_string()) + .collect(); + for needle in needles { + let lowered = needle.to_lowercase(); + if !lowered.is_empty() && !key_needles.contains(&lowered) { + key_needles.push(lowered); + } + } + Self { + key_needles, + ..Self::default() + } + } + + /// Set the per-string truncation limit. `0` disables truncation. + #[must_use] + pub const fn with_max_string_len(mut self, max: usize) -> Self { + self.max_string_len = max; + self + } + + /// Enable or disable credential-shape scanning of string values. + #[must_use] + pub const fn with_value_scanning(mut self, scan: bool) -> Self { + self.scan_values = scan; + self + } + + /// Whether `key` names something that must never be exported. + #[must_use] + pub fn is_sensitive_key(&self, key: &str) -> bool { + let lowered = key.to_lowercase(); + self.key_needles + .iter() + .any(|n| lowered.contains(n.as_str())) + } + + /// Whether `value` looks like a credential regardless of its key. + #[must_use] + pub fn looks_like_secret(&self, value: &str) -> bool { + if !self.scan_values { + return false; + } + let trimmed = value.trim(); + // Very short strings cannot be meaningful credentials and matching them + // produces false positives on ordinary prose. + if trimmed.len() < 8 { + return false; + } + SECRET_PREFIXES.iter().any(|p| trimmed.starts_with(p)) + } + + /// Redact a JSON value in place-equivalent fashion, returning a clean copy. + #[must_use] + pub fn redact(&self, value: &Value) -> Value { + match value { + Value::Object(map) => { + let cleaned = map + .iter() + .map(|(key, val)| { + if self.is_sensitive_key(key) { + (key.clone(), Value::String(REDACTED.to_string())) + } else { + (key.clone(), self.redact(val)) + } + }) + .collect(); + Value::Object(cleaned) + }, + Value::Array(items) => { + Value::Array(items.iter().map(|item| self.redact(item)).collect()) + }, + Value::String(text) => Value::String(self.redact_str(text)), + other => other.clone(), + } + } + + /// Redact and truncate a single string. + #[must_use] + pub fn redact_str(&self, text: &str) -> String { + if self.looks_like_secret(text) { + return REDACTED.to_string(); + } + self.truncate(text) + } + + /// Truncate `text` to the configured limit on a character boundary. + fn truncate(&self, text: &str) -> String { + if self.max_string_len == 0 || text.len() <= self.max_string_len { + return text.to_string(); + } + // `floor_char_boundary` is unstable, so walk back to a boundary by hand + // rather than slicing blindly and panicking on multi-byte input. + let mut end = self.max_string_len; + while end > 0 && !text.is_char_boundary(end) { + end -= 1; + } + format!("{}{TRUNCATION_SUFFIX}", &text[..end]) + } +} + +#[cfg(test)] +mod tests { + use {super::*, serde_json::json}; + + #[test] + fn redacts_sensitive_keys_at_any_depth() { + let policy = RedactionPolicy::default(); + let input = json!({ + "safe": "keep me", + "nested": { + "api_key": "abc123", + "deeper": [{ "password": "hunter2" }] + } + }); + + let out = policy.redact(&input); + + assert_eq!(out["safe"], json!("keep me")); + assert_eq!(out["nested"]["api_key"], json!(REDACTED)); + assert_eq!(out["nested"]["deeper"][0]["password"], json!(REDACTED)); + } + + #[test] + fn key_matching_is_case_insensitive_and_substring_based() { + let policy = RedactionPolicy::default(); + let input = json!({ + "ANTHROPIC_API_KEY": "x", + "userToken": "y", + "Authorization": "z", + }); + + let out = policy.redact(&input); + + assert_eq!(out["ANTHROPIC_API_KEY"], json!(REDACTED)); + assert_eq!(out["userToken"], json!(REDACTED)); + assert_eq!(out["Authorization"], json!(REDACTED)); + } + + #[test] + fn redacts_credential_shaped_values_under_innocuous_keys() { + let policy = RedactionPolicy::default(); + // Tool results are unstructured: a leaked key often lands under a key + // name that no needle list would ever catch. + let input = json!({ + "stdout": "sk-ant-api03-abcdefghijklmnop", + "header": "Bearer eyJhbGciOiJIUzI1NiJ9", + "note": "the deploy finished successfully", + }); + + let out = policy.redact(&input); + + assert_eq!(out["stdout"], json!(REDACTED)); + assert_eq!(out["header"], json!(REDACTED)); + assert_eq!(out["note"], json!("the deploy finished successfully")); + } + + #[test] + fn redacts_pem_private_key_blocks() { + let policy = RedactionPolicy::default(); + let input = json!("-----BEGIN RSA PRIVATE KEY-----\nMIIEow==\n"); + assert_eq!(policy.redact(&input), json!(REDACTED)); + } + + #[test] + fn short_strings_are_not_treated_as_secrets() { + let policy = RedactionPolicy::default(); + // "sk-1" is too short to be a real key; matching it would mangle prose. + assert!(!policy.looks_like_secret("sk-1")); + assert!(policy.looks_like_secret("sk-ant-api03-abcdefgh")); + } + + #[test] + fn value_scanning_can_be_disabled() { + let policy = RedactionPolicy::default().with_value_scanning(false); + assert!(!policy.looks_like_secret("sk-ant-api03-abcdefghijkl")); + } + + #[test] + fn custom_needles_extend_rather_than_replace_defaults() { + let policy = RedactionPolicy::from_needles(&["internal_id".to_string()]); + + assert!(policy.is_sensitive_key("internal_id")); + // The baseline must survive a narrower operator-supplied list. + assert!(policy.is_sensitive_key("password")); + assert!(policy.is_sensitive_key("api_key")); + } + + #[test] + fn empty_needles_are_ignored() { + let policy = RedactionPolicy::from_needles(&[String::new()]); + // An empty needle is a substring of every key and would redact + // everything, so it must be dropped. + assert!(!policy.is_sensitive_key("harmless")); + } + + #[test] + fn truncates_long_strings_on_a_char_boundary() { + let policy = RedactionPolicy::default().with_max_string_len(8); + // Multi-byte characters straddling the limit must not panic. + let out = policy.redact_str("ααααααααααα"); + + assert!(out.ends_with(TRUNCATION_SUFFIX)); + assert!(out.len() < 40); + } + + #[test] + fn truncation_can_be_disabled() { + let policy = RedactionPolicy::default().with_max_string_len(0); + let long = "a".repeat(100_000); + assert_eq!(policy.redact_str(&long).len(), 100_000); + } + + #[test] + fn non_string_scalars_pass_through_unchanged() { + let policy = RedactionPolicy::default(); + let input = json!({ "n": 42, "b": true, "nil": null, "f": 1.5 }); + assert_eq!(policy.redact(&input), input); + } +} diff --git a/crates/observability/src/runtime.rs b/crates/observability/src/runtime.rs new file mode 100644 index 000000000..a0df56146 --- /dev/null +++ b/crates/observability/src/runtime.rs @@ -0,0 +1,606 @@ +//! Batching export runtime shared by every backend. +//! +//! A backend supplies a [`Transport`]; this module supplies everything around +//! it — a bounded queue, size- and time-based batching, retry with exponential +//! backoff and jitter, and drop accounting. +//! +//! The central invariant: **the agent loop is never blocked by telemetry**. +//! [`BatchSink::record`] uses `try_send` and drops on a full queue. Losing +//! traces is an acceptable failure; stalling a user's turn is not. + +use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; + +use { + async_trait::async_trait, + tokio::sync::{mpsc, oneshot}, + tracing::{debug, trace, warn}, +}; + +use crate::{model::Event, sink::ObservationSink}; + +/// Why a transport send failed. +#[derive(Debug, thiserror::Error)] +pub enum TransportError { + /// Transient: worth retrying (5xx, 429, connection reset). + #[error("retryable transport failure: {0}")] + Retryable(String), + /// Permanent: retrying cannot help (401, 400, malformed payload). + #[error("fatal transport failure: {0}")] + Fatal(String), +} + +/// Backend-specific delivery mechanism. +#[async_trait] +pub trait Transport: Send + Sync + 'static { + /// Transport name, used in logs and status reporting. + fn name(&self) -> &str; + + /// Deliver a batch. Called from the background task only. + async fn send(&self, batch: &[Event]) -> Result<(), TransportError>; + + /// Approximate serialized size of `event`, used for batch sizing. + fn estimate_bytes(&self, event: &Event) -> usize { + serde_json::to_vec(event).map(|v| v.len()).unwrap_or(1024) + } +} + +/// Batching and retry parameters. +#[derive(Debug, Clone)] +pub struct BatchConfig { + /// Maximum events held before a forced flush. + pub max_batch_events: usize, + /// Maximum estimated batch size in bytes before a forced flush. + pub max_batch_bytes: usize, + /// Maximum time an event waits before being flushed. + pub flush_interval: Duration, + /// Bounded queue depth. Events are dropped once this fills. + pub queue_capacity: usize, + /// Retry attempts for retryable failures. + pub max_retries: u32, + /// Base delay for exponential backoff. + pub initial_backoff: Duration, + /// Ceiling for exponential backoff. + pub max_backoff: Duration, +} + +impl Default for BatchConfig { + fn default() -> Self { + Self { + max_batch_events: 128, + // Comfortably under Langfuse's per-request ceiling, leaving room + // for the estimate to be wrong without the request being rejected. + max_batch_bytes: 3_000_000, + flush_interval: Duration::from_secs(5), + queue_capacity: 10_000, + max_retries: 3, + initial_backoff: Duration::from_millis(500), + max_backoff: Duration::from_secs(30), + } + } +} + +/// Counters describing sink health, surfaced in the settings UI. +#[derive(Debug, Default)] +pub struct SinkStats { + /// Events accepted into the queue. + pub accepted: AtomicU64, + /// Events dropped because the queue was full. + pub dropped_queue_full: AtomicU64, + /// Events dropped after exhausting retries or hitting a fatal error. + pub dropped_failed: AtomicU64, + /// Batches delivered successfully. + pub batches_sent: AtomicU64, + /// Batches that failed permanently. + pub batches_failed: AtomicU64, +} + +/// Point-in-time snapshot of [`SinkStats`]. +#[derive(Debug, Clone, Copy, serde::Serialize)] +pub struct SinkStatsSnapshot { + /// Events accepted into the queue. + pub accepted: u64, + /// Events dropped because the queue was full. + pub dropped_queue_full: u64, + /// Events dropped after exhausting retries or hitting a fatal error. + pub dropped_failed: u64, + /// Batches delivered successfully. + pub batches_sent: u64, + /// Batches that failed permanently. + pub batches_failed: u64, +} + +impl SinkStats { + /// Read every counter. + #[must_use] + pub fn snapshot(&self) -> SinkStatsSnapshot { + SinkStatsSnapshot { + accepted: self.accepted.load(Ordering::Relaxed), + dropped_queue_full: self.dropped_queue_full.load(Ordering::Relaxed), + dropped_failed: self.dropped_failed.load(Ordering::Relaxed), + batches_sent: self.batches_sent.load(Ordering::Relaxed), + batches_failed: self.batches_failed.load(Ordering::Relaxed), + } + } +} + +/// Queue message: either an event or a flush barrier. +enum Message { + Event(Box), + Flush(oneshot::Sender<()>), +} + +/// An [`ObservationSink`] that batches into a [`Transport`]. +pub struct BatchSink { + name: String, + tx: mpsc::Sender, + stats: Arc, +} + +impl BatchSink { + /// Spawn the background export task and return a sink feeding it. + #[must_use] + pub fn spawn(transport: Arc, config: BatchConfig) -> Self { + let (tx, rx) = mpsc::channel(config.queue_capacity); + let stats = Arc::new(SinkStats::default()); + let name = transport.name().to_string(); + + tokio::spawn(export_loop( + rx, + Arc::clone(&transport), + config, + Arc::clone(&stats), + )); + + Self { name, tx, stats } + } + + /// Health counters for this sink. + #[must_use] + pub fn stats(&self) -> SinkStatsSnapshot { + self.stats.snapshot() + } +} + +#[async_trait] +impl ObservationSink for BatchSink { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + // `try_send` never awaits: a saturated queue drops the event rather + // than applying backpressure to the caller's turn. + match self.tx.try_send(Message::Event(Box::new(event))) { + Ok(()) => { + self.stats.accepted.fetch_add(1, Ordering::Relaxed); + }, + Err(mpsc::error::TrySendError::Full(_)) => { + let dropped = self + .stats + .dropped_queue_full + .fetch_add(1, Ordering::Relaxed) + + 1; + // Log sparsely: a saturated queue would otherwise generate more + // log volume than the telemetry it is failing to send. + if dropped.is_power_of_two() { + warn!( + sink = %self.name, + dropped, + "observability queue full; dropping events" + ); + } + }, + Err(mpsc::error::TrySendError::Closed(_)) => { + self.stats.dropped_failed.fetch_add(1, Ordering::Relaxed); + }, + } + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + "moltis_observability_events_total", + "sink" => self.name.clone() + ) + .increment(1); + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + let (ack_tx, ack_rx) = oneshot::channel(); + if self.tx.send(Message::Flush(ack_tx)).await.is_err() { + return Err(anyhow::anyhow!("export task for {} has stopped", self.name)); + } + match tokio::time::timeout(timeout, ack_rx).await { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err(anyhow::anyhow!( + "export task for {} dropped the flush barrier", + self.name + )), + Err(_) => Err(anyhow::anyhow!("flush of {} timed out", self.name)), + } + } +} + +/// Drain the queue, batching by count, size and time. +async fn export_loop( + mut rx: mpsc::Receiver, + transport: Arc, + config: BatchConfig, + stats: Arc, +) { + let mut batch: Vec = Vec::with_capacity(config.max_batch_events.min(1024)); + let mut batch_bytes = 0usize; + let mut ticker = tokio::time::interval(config.flush_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + + loop { + tokio::select! { + message = rx.recv() => { + match message { + Some(Message::Event(event)) => { + batch_bytes += transport.estimate_bytes(&event); + batch.push(*event); + + if batch.len() >= config.max_batch_events + || batch_bytes >= config.max_batch_bytes + { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + } + }, + Some(Message::Flush(ack)) => { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + // The receiver may have timed out; that is not an error. + let _ = ack.send(()); + }, + None => { + // Sink dropped: make a final delivery attempt so a + // clean shutdown does not discard buffered spans. + deliver(&transport, &mut batch, &config, &stats).await; + return; + }, + } + }, + _ = ticker.tick() => { + if !batch.is_empty() { + deliver(&transport, &mut batch, &config, &stats).await; + batch_bytes = 0; + } + }, + } + } +} + +/// Send `batch` with retries, then clear it regardless of outcome. +async fn deliver( + transport: &Arc, + batch: &mut Vec, + config: &BatchConfig, + stats: &Arc, +) { + if batch.is_empty() { + return; + } + + let count = batch.len() as u64; + let mut attempt = 0u32; + + loop { + match transport.send(batch).await { + Ok(()) => { + stats.batches_sent.fetch_add(1, Ordering::Relaxed); + trace!(sink = transport.name(), events = count, "batch delivered"); + break; + }, + Err(TransportError::Fatal(reason)) => { + // Retrying a 401 or a malformed payload only burns quota. + warn!( + sink = transport.name(), + events = count, + %reason, + "dropping batch after fatal export failure" + ); + stats.batches_failed.fetch_add(1, Ordering::Relaxed); + stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + break; + }, + Err(TransportError::Retryable(reason)) if attempt < config.max_retries => { + let delay = backoff_delay(config, attempt); + debug!( + sink = transport.name(), + attempt, + delay_ms = delay.as_millis(), + %reason, + "retrying observability export" + ); + tokio::time::sleep(delay).await; + attempt += 1; + }, + Err(TransportError::Retryable(reason)) => { + warn!( + sink = transport.name(), + events = count, + attempts = attempt + 1, + %reason, + "dropping batch after exhausting export retries" + ); + stats.batches_failed.fetch_add(1, Ordering::Relaxed); + stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + break; + }, + } + } + + batch.clear(); +} + +/// Exponential backoff with full jitter, clamped to `max_backoff`. +/// +/// Jitter matters here: without it, every Moltis instance that hits the same +/// rate limit retries in lockstep and re-creates the overload. +fn backoff_delay(config: &BatchConfig, attempt: u32) -> Duration { + let exponent = attempt.min(16); + let base = config + .initial_backoff + .saturating_mul(2u32.saturating_pow(exponent)); + let capped = base.min(config.max_backoff); + let millis = capped.as_millis().min(u128::from(u64::MAX)) as u64; + if millis == 0 { + return Duration::ZERO; + } + let jittered = rand::random_range(0..=millis); + Duration::from_millis(jittered) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId}, + }; + + #[derive(Default)] + struct RecordingTransport { + batches: Mutex>, + /// Number of leading calls that fail retryably. + fail_retryable: AtomicU64, + /// Whether every call fails fatally. + fail_fatal: bool, + } + + #[async_trait] + impl Transport for RecordingTransport { + fn name(&self) -> &str { + "recording" + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + if self.fail_fatal { + return Err(TransportError::Fatal("unauthorized".into())); + } + if self.fail_retryable.load(Ordering::Relaxed) > 0 { + self.fail_retryable.fetch_sub(1, Ordering::Relaxed); + return Err(TransportError::Retryable("503".into())); + } + self.batches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(batch.len()); + Ok(()) + } + } + + impl RecordingTransport { + fn batches(&self) -> Vec { + self.batches + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + } + + fn event() -> Event { + Event::ObservationEnd(Box::new(ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "llm-call", + ))) + } + + fn fast_config() -> BatchConfig { + BatchConfig { + flush_interval: Duration::from_millis(50), + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(5), + ..Default::default() + } + } + + #[tokio::test] + async fn flush_delivers_pending_events() { + let transport = Arc::new(RecordingTransport::default()); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches(), vec![2]); + assert_eq!(sink.stats().accepted, 2); + } + + #[tokio::test] + async fn batch_is_forced_once_event_count_reached() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 3, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + for _ in 0..3 { + sink.record(event()); + } + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + // The size trigger must fire on its own, before the flush barrier. + assert_eq!(transport.batches().first(), Some(&3)); + } + + #[tokio::test] + async fn batch_is_forced_once_byte_budget_reached() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 10_000, + max_batch_bytes: 1, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches().first(), Some(&1)); + } + + #[tokio::test] + async fn retryable_failures_are_retried_then_succeed() { + let transport = Arc::new(RecordingTransport::default()); + transport.fail_retryable.store(2, Ordering::Relaxed); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + + assert_eq!(transport.batches(), vec![1]); + assert_eq!(sink.stats().batches_sent, 1); + assert_eq!(sink.stats().dropped_failed, 0); + } + + #[tokio::test] + async fn retries_are_bounded_and_batch_is_dropped() { + let transport = Arc::new(RecordingTransport::default()); + transport.fail_retryable.store(100, Ordering::Relaxed); + let config = BatchConfig { + max_retries: 2, + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should still complete"); + + assert!(transport.batches().is_empty()); + assert_eq!(sink.stats().dropped_failed, 1); + assert_eq!(sink.stats().batches_failed, 1); + } + + #[tokio::test] + async fn fatal_failures_are_not_retried() { + let transport = Arc::new(RecordingTransport { + fail_fatal: true, + ..Default::default() + }); + let sink = BatchSink::spawn(transport.clone(), fast_config()); + + sink.record(event()); + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should still complete"); + + assert_eq!(sink.stats().batches_failed, 1); + assert_eq!(sink.stats().dropped_failed, 1); + } + + #[tokio::test] + async fn full_queue_drops_events_without_blocking_caller() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + queue_capacity: 4, + max_batch_events: 100_000, + // Long interval so the drain never runs during the test. + flush_interval: Duration::from_secs(3600), + ..fast_config() + }; + let sink = BatchSink::spawn(transport, config); + + // Far more events than the queue can hold. `record` is synchronous, so + // if it ever blocked this test would hang rather than fail. + for _ in 0..500 { + sink.record(event()); + } + + let stats = sink.stats(); + assert!( + stats.dropped_queue_full > 0, + "expected drops on a full queue" + ); + assert!( + stats.accepted < 500, + "queue must not have absorbed every event" + ); + } + + #[tokio::test] + async fn time_based_flush_delivers_without_an_explicit_barrier() { + let transport = Arc::new(RecordingTransport::default()); + let config = BatchConfig { + max_batch_events: 10_000, + flush_interval: Duration::from_millis(20), + ..fast_config() + }; + let sink = BatchSink::spawn(transport.clone(), config); + + sink.record(event()); + // Deliberately no flush() call: the ticker must do the work. + for _ in 0..50 { + if !transport.batches().is_empty() { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + + assert_eq!(transport.batches(), vec![1]); + } + + #[test] + fn backoff_is_capped_and_jittered_within_bounds() { + let config = BatchConfig { + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_millis(1000), + ..Default::default() + }; + + for attempt in 0..8 { + let delay = backoff_delay(&config, attempt); + assert!( + delay <= config.max_backoff, + "attempt {attempt} exceeded the cap" + ); + } + } + + #[test] + fn backoff_does_not_overflow_on_large_attempt_counts() { + let config = BatchConfig::default(); + // A pathological attempt count must clamp, not panic on overflow. + assert!(backoff_delay(&config, u32::MAX) <= config.max_backoff); + } +} diff --git a/crates/observability/src/sink.rs b/crates/observability/src/sink.rs new file mode 100644 index 000000000..a9a6fad6d --- /dev/null +++ b/crates/observability/src/sink.rs @@ -0,0 +1,311 @@ +//! The [`ObservationSink`] boundary plus the process-wide registry. +//! +//! `record` is deliberately synchronous and non-blocking: instrumentation sits +//! on the agent's hot path, so a slow or unreachable backend must never stall a +//! turn. Implementations enqueue and return; dropping data is always preferable +//! to applying backpressure to the agent loop. + +use std::{ + sync::{Arc, RwLock}, + time::Duration, +}; + +use async_trait::async_trait; + +use crate::model::Event; + +/// Destination for observability events. +#[async_trait] +pub trait ObservationSink: Send + Sync { + /// Sink name, used in logs and status reporting. + fn name(&self) -> &str; + + /// Enqueue an event. Must not block and must not fail loudly. + fn record(&self, event: Event); + + /// Flush pending events, giving up after `timeout`. + async fn flush(&self, timeout: Duration) -> anyhow::Result<()>; +} + +/// Fans one event out to several sinks, so Langfuse and an OTLP collector can +/// run side by side from a single instrumentation pass. +pub struct SinkFanout { + sinks: Vec>, + name: String, +} + +impl SinkFanout { + /// Build a fanout over `sinks`. + #[must_use] + pub fn new(sinks: Vec>) -> Self { + let name = sinks + .iter() + .map(|s| s.name().to_string()) + .collect::>() + .join("+"); + Self { sinks, name } + } + + /// Whether any sink is attached. + #[must_use] + pub fn is_empty(&self) -> bool { + self.sinks.is_empty() + } + + /// Number of attached sinks. + #[must_use] + pub fn len(&self) -> usize { + self.sinks.len() + } + + /// Names of the attached sinks. + #[must_use] + pub fn sink_names(&self) -> Vec { + self.sinks.iter().map(|s| s.name().to_string()).collect() + } +} + +#[async_trait] +impl ObservationSink for SinkFanout { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + // Clone per sink except the last, which takes ownership. + let mut iter = self.sinks.iter().peekable(); + while let Some(sink) = iter.next() { + if iter.peek().is_some() { + sink.record(event.clone()); + } else { + sink.record(event); + return; + } + } + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + // Flush concurrently: a slow backend must not serialise the others. + let futures = self.sinks.iter().map(|s| s.flush(timeout)); + let results = futures::future::join_all(futures).await; + + let errors: Vec = results + .into_iter() + .filter_map(|r| r.err().map(|e| e.to_string())) + .collect(); + + if errors.is_empty() { + return Ok(()); + } + Err(anyhow::anyhow!("sink flush failed: {}", errors.join("; "))) + } +} + +// ── Process-wide registry ─────────────────────────────────────────────────── + +/// The active sink. `RwLock` rather than `OnceLock` because the settings UI can +/// reconfigure instrumentation without a restart. +static GLOBAL_SINK: RwLock>> = RwLock::new(None); + +/// Install `sink` as the process-wide destination, replacing any previous one. +pub fn set_global_sink(sink: Arc) { + match GLOBAL_SINK.write() { + Ok(mut guard) => *guard = Some(sink), + Err(poisoned) => *poisoned.into_inner() = Some(sink), + } +} + +/// Remove the process-wide sink, disabling instrumentation. +pub fn clear_global_sink() { + match GLOBAL_SINK.write() { + Ok(mut guard) => *guard = None, + Err(poisoned) => *poisoned.into_inner() = None, + } +} + +/// The active sink, if instrumentation is configured. +#[must_use] +pub fn global_sink() -> Option> { + match GLOBAL_SINK.read() { + Ok(guard) => guard.clone(), + Err(poisoned) => poisoned.into_inner().clone(), + } +} + +/// Whether instrumentation is active. +/// +/// Call this before doing any work to build an event: when no sink is +/// installed, instrumentation must cost nothing beyond this check. +#[must_use] +pub fn is_enabled() -> bool { + match GLOBAL_SINK.read() { + Ok(guard) => guard.is_some(), + Err(poisoned) => poisoned.into_inner().is_some(), + } +} + +/// Send `event` to the active sink, if any. +pub fn record(event: Event) { + if let Some(sink) = global_sink() { + sink.record(event); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use { + super::*, + crate::model::{ObservationKind, ObservationRecord, TraceId, TraceRecord}, + }; + + struct CollectingSink { + name: String, + events: Mutex>, + } + + impl CollectingSink { + fn new(name: &str) -> Arc { + Arc::new(Self { + name: name.to_string(), + events: Mutex::new(Vec::new()), + }) + } + + fn count(&self) -> usize { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + } + } + + #[async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + &self.name + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + struct FailingSink; + + #[async_trait] + impl ObservationSink for FailingSink { + fn name(&self) -> &str { + "failing" + } + + fn record(&self, _event: Event) {} + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Err(anyhow::anyhow!("backend unreachable")) + } + } + + fn sample_event() -> Event { + Event::ObservationEnd(Box::new(ObservationRecord::start( + TraceId::generate(), + ObservationKind::Generation, + "llm-call", + ))) + } + + #[test] + fn fanout_delivers_to_every_sink() { + let a = CollectingSink::new("a"); + let b = CollectingSink::new("b"); + let c = CollectingSink::new("c"); + let fanout = SinkFanout::new(vec![a.clone(), b.clone(), c.clone()]); + + fanout.record(sample_event()); + + assert_eq!(a.count(), 1); + assert_eq!(b.count(), 1); + assert_eq!(c.count(), 1); + } + + #[test] + fn fanout_name_lists_every_backend() { + let fanout = SinkFanout::new(vec![ + CollectingSink::new("langfuse"), + CollectingSink::new("otlp"), + ]); + assert_eq!(fanout.name(), "langfuse+otlp"); + assert_eq!(fanout.len(), 2); + assert!(!fanout.is_empty()); + } + + #[test] + fn empty_fanout_accepts_events_without_panicking() { + let fanout = SinkFanout::new(Vec::new()); + fanout.record(sample_event()); + assert!(fanout.is_empty()); + } + + #[tokio::test] + async fn fanout_flush_reports_every_failure() { + let fanout = SinkFanout::new(vec![ + CollectingSink::new("ok"), + Arc::new(FailingSink), + Arc::new(FailingSink), + ]); + + let error = fanout + .flush(Duration::from_millis(10)) + .await + .expect_err("failing sinks must surface an error"); + let rendered = error.to_string(); + + // Both failures must be reported, not just the first. + assert_eq!(rendered.matches("backend unreachable").count(), 2); + } + + #[tokio::test] + async fn fanout_flush_succeeds_when_all_sinks_succeed() { + let fanout = SinkFanout::new(vec![CollectingSink::new("a"), CollectingSink::new("b")]); + assert!(fanout.flush(Duration::from_millis(10)).await.is_ok()); + } + + // The global registry is process-wide state, so these assertions live in a + // single test to stay independent of test execution order. + #[test] + fn global_registry_install_record_and_clear() { + let sink = CollectingSink::new("global"); + + assert!(!is_enabled()); + record(sample_event()); + assert_eq!(sink.count(), 0, "events must drop while no sink installed"); + + set_global_sink(sink.clone()); + assert!(is_enabled()); + assert!(global_sink().is_some()); + + record(sample_event()); + assert_eq!(sink.count(), 1); + + clear_global_sink(); + assert!(!is_enabled()); + record(sample_event()); + assert_eq!(sink.count(), 1, "events must drop after clearing the sink"); + } + + #[test] + fn trace_event_reports_owning_trace() { + let trace = TraceRecord::new("turn"); + let id = trace.id.clone(); + let event = Event::Trace(Box::new(trace)); + assert_eq!(event.trace_id(), &id); + } +} From 6a65ba39d8fdaa0f81fa95e56975a70eb646f29e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 05:43:21 +0000 Subject: [PATCH 02/33] feat(observability): instrument the agent loop and wire backends at startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connects the instrumentation core to the places that generate data and to the gateway that configures it. The agent runner now records a trace per turn, a GENERATION observation per LLM call carrying model, cache-split token usage and time-to-first-token, and a TOOL observation per tool call. Retrieval tools are recorded as RETRIEVER so backends that special-case RAG steps light up rather than showing every tool as an opaque TOOL. Instrumentation reaches the runner through a process-wide sink rather than a new parameter on run_agent_loop_streaming and its siblings. Those signatures already carry ten arguments and are called from many places; threading an eleventh through for telemetry would have touched far more code than the feature warrants, and the runner already reads global config directly. Two details worth calling out. The generation span is closed explicitly on the stream-error path before the retry return, because relying on the drop guard there would emit a span with no error recorded on it — a failed provider call would look successful in the backend. And the turn recorder is shared as an Arc because tool futures execute concurrently; finish() takes &self for the same reason. Trace scope derives from the session key and channel binding: the session key becomes the backend session id so a Langfuse session matches a Moltis conversation one-to-one, and the sender becomes a user id namespaced by channel so telegram:42 and slack:42 are never merged into one person. Gateway startup applies config to GatewayState's own instrumentation instance, so the RPC handlers and the shutdown flush observe the same backends that were installed. Backends that fail to start are reported with a reason through instrumentation.status rather than only logged — a silently disabled exporter is indistinguishable from a broken one. instrumentation.status never returns the secret key, only whether one is set. instrumentation.test probes Langfuse's health and credential endpoints; for OTLP backends it declines rather than POSTing a probe span, which would put fake data in the operator's traces. Adds the config template section and docs/src/instrumentation.md. --- Cargo.lock | 2 + crates/agents/Cargo.toml | 3 +- crates/agents/src/runner/instrumentation.rs | 322 ++++++++++++++++++ crates/agents/src/runner/mod.rs | 1 + crates/agents/src/runner/streaming.rs | 83 ++++- crates/config/src/template.rs | 43 +++ crates/gateway/Cargo.toml | 1 + crates/gateway/src/methods/dispatch.rs | 2 + crates/gateway/src/methods/services.rs | 2 + .../src/methods/services/instrumentation.rs | 97 ++++++ crates/gateway/src/server/instrumentation.rs | 234 +++++++++++++ crates/gateway/src/server/mod.rs | 2 + .../src/server/prepare_core/post_state.rs | 20 ++ crates/gateway/src/state.rs | 6 + crates/observability/src/recorder.rs | 7 +- docs/src/SUMMARY.md | 1 + docs/src/instrumentation.md | 288 ++++++++++++++++ 17 files changed, 1110 insertions(+), 4 deletions(-) create mode 100644 crates/agents/src/runner/instrumentation.rs create mode 100644 crates/gateway/src/methods/services/instrumentation.rs create mode 100644 crates/gateway/src/server/instrumentation.rs create mode 100644 docs/src/instrumentation.md diff --git a/Cargo.lock b/Cargo.lock index 00ab9f938..4d4acf55e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7174,6 +7174,7 @@ dependencies = [ "moltis-common", "moltis-config", "moltis-metrics", + "moltis-observability", "moltis-sessions", "moltis-skills", "reqwest 0.13.2", @@ -7593,6 +7594,7 @@ dependencies = [ "moltis-network-filter", "moltis-nostr", "moltis-oauth", + "moltis-observability", "moltis-onboarding", "moltis-openclaw-import", "moltis-plugins", diff --git a/crates/agents/Cargo.toml b/crates/agents/Cargo.toml index 3108c93c3..6a2bb0e70 100644 --- a/crates/agents/Cargo.toml +++ b/crates/agents/Cargo.toml @@ -13,7 +13,8 @@ async-stream = { workspace = true } async-trait = { workspace = true } futures = { workspace = true } include_dir = { workspace = true } -moltis-common = { workspace = true } +moltis-common = { workspace = true } +moltis-observability = { workspace = true } moltis-config = { workspace = true } moltis-sessions = { workspace = true } moltis-skills = { workspace = true } diff --git a/crates/agents/src/runner/instrumentation.rs b/crates/agents/src/runner/instrumentation.rs new file mode 100644 index 000000000..74ece134a --- /dev/null +++ b/crates/agents/src/runner/instrumentation.rs @@ -0,0 +1,322 @@ +//! Bridge between the agent loop and `moltis-observability`. +//! +//! Keeps trace-scope derivation out of the runner bodies: the loops call +//! [`begin_turn`] once and then open steps. Every function here degrades to a +//! no-op when instrumentation is disabled, so the runner needs no `cfg` gates. + +use { + moltis_common::hooks::ChannelBinding, + moltis_observability::{ + ObservationKind, RecorderSettings, TokenUsage, TraceScope, TurnRecorder, + }, +}; + +use crate::model::{Usage, UserContent}; + +/// Derive the trace scope from the session and channel context. +/// +/// The session key becomes the backend's session id, so a Langfuse session +/// view lines up one-to-one with a Moltis conversation. Channel provenance +/// becomes tags, and the sender becomes the user id — namespaced by channel so +/// `telegram:42` and `slack:42` are never conflated into one person. +#[must_use] +pub fn trace_scope( + session_key: &str, + channel: Option<&ChannelBinding>, + environment: String, + release: String, +) -> TraceScope { + let mut tags = Vec::new(); + let mut user_id = None; + + if let Some(binding) = channel { + if let Some(channel_type) = &binding.channel_type { + tags.push(format!("channel:{channel_type}")); + } + if let Some(surface) = &binding.surface { + tags.push(format!("surface:{surface}")); + } + if let Some(chat_type) = &binding.chat_type { + tags.push(format!("chat:{chat_type}")); + } + if let Some(sender) = &binding.sender_id { + user_id = Some(binding.channel_type.as_ref().map_or_else( + || sender.clone(), + |channel_type| format!("{channel_type}:{sender}"), + )); + } + } + + // The agent id is the leading segment of `agent::...`. + if let Some(agent_id) = session_key + .strip_prefix("agent:") + .and_then(|rest| rest.split(':').next()) + && !agent_id.is_empty() + { + tags.push(format!("agent:{agent_id}")); + } + + TraceScope { + session_id: (!session_key.is_empty()).then(|| session_key.to_string()), + user_id, + tags, + environment: Some(environment), + release: Some(release), + version: None, + } +} + +/// Recorder settings derived from `[instrumentation]`. +/// +/// Capture switches come from the Langfuse sub-table because it is the only +/// backend that receives payload bodies at all; the APM profiles gate content +/// independently through their own `content` mode. +#[must_use] +pub fn recorder_settings(config: &moltis_config::InstrumentationConfig) -> RecorderSettings { + RecorderSettings { + redaction: moltis_observability::RedactionPolicy::from_needles(&config.redact), + capture_input: config.langfuse.capture_input, + capture_output: config.langfuse.capture_output, + capture_tool_io: config.langfuse.capture_tool_io, + sample_rate: config.sample_rate.clamp(0.0, 1.0), + } +} + +/// Release identifier reported to backends, defaulting to the running version. +#[must_use] +pub fn release(config: &moltis_config::InstrumentationConfig) -> String { + config + .release + .clone() + .unwrap_or_else(|| moltis_config::VERSION.to_string()) +} + +/// Begin recording an agent run, if instrumentation is enabled. +/// +/// Returns `None` when no sink is installed or the turn was not sampled, in +/// which case every downstream call is skipped by the caller's `Option` checks. +#[must_use] +pub fn begin_turn( + session_key: &str, + channel: Option<&ChannelBinding>, + provider: &str, + model: &str, + user_content: &UserContent, + settings: RecorderSettings, + environment: String, + release: String, +) -> Option { + let scope = trace_scope(session_key, channel, environment, release); + let recorder = TurnRecorder::begin("agent-run", scope, settings)?; + + recorder.set_metadata("provider", serde_json::Value::String(provider.to_string())); + recorder.set_metadata("model", serde_json::Value::String(model.to_string())); + recorder.set_input(user_content_to_json(user_content)); + Some(recorder) +} + +/// Render turn input for the trace. +/// +/// Multimodal turns report only their shape: the image bytes are already +/// carried elsewhere and inlining base64 here would dwarf every other +/// attribute in the payload. +#[must_use] +pub fn user_content_to_json(content: &UserContent) -> serde_json::Value { + match content { + UserContent::Text(text) => serde_json::Value::String(text.clone()), + UserContent::Multimodal(parts) => serde_json::json!({ + "type": "multimodal", + "parts": parts.len(), + }), + } +} + +/// Convert provider usage counters into the observability model. +/// +/// Provider `input_tokens` is already exclusive of cached tokens, so the +/// buckets are carried across as-is rather than being summed: Langfuse prices +/// cache reads and writes differently from fresh input, and folding them +/// together silently inflates reported cost. +#[must_use] +pub const fn to_token_usage(usage: &Usage) -> TokenUsage { + TokenUsage::from_provider_totals( + usage.input_tokens, + usage.output_tokens, + usage.cache_read_tokens, + usage.cache_write_tokens, + ) +} + +/// Name for the generation step of a given iteration. +#[must_use] +pub fn generation_name(provider: &str, model: &str) -> String { + format!("{provider}/{model}") +} + +/// The observation kind a tool call should be recorded as. +/// +/// Retrieval tools are reported as `RETRIEVER` so backends that special-case +/// RAG steps light up, rather than showing every tool as an opaque `TOOL`. +#[must_use] +pub fn tool_observation_kind(tool_name: &str) -> ObservationKind { + const RETRIEVAL_TOOLS: &[&str] = &[ + "memory_search", + "memory_query", + "code_search", + "code_index_search", + "web_search", + "search", + ]; + if RETRIEVAL_TOOLS.contains(&tool_name) { + ObservationKind::Retriever + } else { + ObservationKind::Tool + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn binding() -> ChannelBinding { + ChannelBinding { + surface: Some("chat".into()), + session_kind: None, + channel_type: Some("telegram".into()), + account_id: Some("acct-1".into()), + chat_id: Some("chat-1".into()), + chat_type: Some("dm".into()), + sender_id: Some("42".into()), + } + } + + #[test] + fn session_key_becomes_the_backend_session_id() { + let scope = trace_scope( + "agent:main:main", + None, + "production".into(), + "20260726.01".into(), + ); + assert_eq!(scope.session_id.as_deref(), Some("agent:main:main")); + } + + #[test] + fn empty_session_keys_are_omitted_rather_than_grouped() { + // An empty string would collapse every unattributed turn into one + // giant session in the backend's session view. + let scope = trace_scope("", None, "production".into(), "1.0".into()); + assert!(scope.session_id.is_none()); + } + + #[test] + fn user_id_is_namespaced_by_channel() { + // Otherwise Telegram user 42 and Slack user 42 merge into one person. + let scope = trace_scope( + "agent:main:main", + Some(&binding()), + "production".into(), + "1.0".into(), + ); + assert_eq!(scope.user_id.as_deref(), Some("telegram:42")); + } + + #[test] + fn sender_without_a_channel_type_is_used_verbatim() { + let unnamed = ChannelBinding { + channel_type: None, + ..binding() + }; + let scope = trace_scope( + "agent:main:main", + Some(&unnamed), + "prod".into(), + "1.0".into(), + ); + assert_eq!(scope.user_id.as_deref(), Some("42")); + } + + #[test] + fn channel_provenance_and_agent_become_tags() { + let scope = trace_scope( + "agent:support:channel:telegram:account:a:peer:user:42", + Some(&binding()), + "production".into(), + "1.0".into(), + ); + + assert!(scope.tags.contains(&"channel:telegram".to_string())); + assert!(scope.tags.contains(&"surface:chat".to_string())); + assert!(scope.tags.contains(&"chat:dm".to_string())); + assert!(scope.tags.contains(&"agent:support".to_string())); + } + + #[test] + fn a_turn_with_no_channel_still_produces_a_usable_scope() { + let scope = trace_scope("agent:main:main", None, "staging".into(), "1.0".into()); + + assert!(scope.user_id.is_none()); + assert_eq!(scope.tags, vec!["agent:main".to_string()]); + assert_eq!(scope.environment.as_deref(), Some("staging")); + } + + #[test] + fn malformed_session_keys_do_not_produce_an_empty_agent_tag() { + let scope = trace_scope("agent:", None, "prod".into(), "1.0".into()); + assert!( + !scope.tags.iter().any(|t| t == "agent:"), + "empty agent tag is noise in the backend's tag filter" + ); + } + + #[test] + fn usage_conversion_preserves_cache_buckets_separately() { + let usage = Usage { + input_tokens: 100, + output_tokens: 50, + cache_read_tokens: 900, + cache_write_tokens: 20, + }; + let converted = to_token_usage(&usage); + + assert_eq!(converted.input, 100); + assert_eq!(converted.cache_read, 900); + assert_eq!(converted.cache_write, 20); + // Summing cache into input would inflate the priced fresh-input count. + assert_ne!(converted.input, 1020); + } + + #[test] + fn multimodal_input_reports_shape_not_image_bytes() { + let json = user_content_to_json(&UserContent::Multimodal(Vec::new())); + assert_eq!(json["type"], "multimodal"); + assert_eq!(json["parts"], 0); + } + + #[test] + fn text_input_is_carried_verbatim() { + let json = user_content_to_json(&UserContent::Text("hello".into())); + assert_eq!(json, serde_json::Value::String("hello".into())); + } + + #[test] + fn retrieval_tools_are_recorded_as_retriever_observations() { + assert_eq!( + tool_observation_kind("memory_search"), + ObservationKind::Retriever + ); + assert_eq!( + tool_observation_kind("code_search"), + ObservationKind::Retriever + ); + assert_eq!(tool_observation_kind("exec"), ObservationKind::Tool); + } + + #[test] + fn generation_name_identifies_provider_and_model() { + assert_eq!( + generation_name("anthropic", "claude-opus-4"), + "anthropic/claude-opus-4" + ); + } +} diff --git a/crates/agents/src/runner/mod.rs b/crates/agents/src/runner/mod.rs index 31f47fb61..1f9ef0e67 100644 --- a/crates/agents/src/runner/mod.rs +++ b/crates/agents/src/runner/mod.rs @@ -1,6 +1,7 @@ //! Agent runner: LLM call loop with tool execution, retry, and streaming support. mod helpers; +pub mod instrumentation; mod non_streaming; pub mod retry; mod streaming; diff --git a/crates/agents/src/runner/streaming.rs b/crates/agents/src/runner/streaming.rs index 941be50f5..478de87fb 100644 --- a/crates/agents/src/runner/streaming.rs +++ b/crates/agents/src/runner/streaming.rs @@ -35,7 +35,7 @@ use super::{ channel_binding_from_tool_context, dispatch_after_llm_call_hook, dispatch_before_agent_start_hook, empty_tool_name_retry_prompt, explicit_shell_command_from_user_content, find_empty_tool_name_call, finish_agent_run, - has_named_tool_call, is_substantive_answer_text, log_tool_argument_diagnostic, + has_named_tool_call, instrumentation, is_substantive_answer_text, log_tool_argument_diagnostic, resolve_tool_lookup, retry::{ RATE_LIMIT_MAX_RETRIES, is_context_window_error, next_retry_delay_ms, @@ -147,6 +147,19 @@ pub async fn run_agent_loop_streaming_with_limits( let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); + // Instrumentation. `None` when no backend is configured or the turn was + // not sampled, in which case every call below is skipped. + let turn_recorder = Arc::new(instrumentation::begin_turn( + &session_key_for_hooks, + channel_for_hooks.as_ref(), + provider.name(), + provider.id(), + user_content, + instrumentation::recorder_settings(&config.instrumentation), + config.instrumentation.environment.clone(), + instrumentation::release(&config.instrumentation), + )); + dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -289,6 +302,20 @@ pub async fn run_agent_loop_streaming_with_limits( // Use streaming API. #[cfg(feature = "metrics")] let iter_start = std::time::Instant::now(); + let mut generation_step = turn_recorder.as_ref().as_ref().map(|recorder| { + let mut step = recorder.step( + moltis_observability::ObservationKind::Generation, + instrumentation::generation_name(provider.name(), provider.id()), + ); + step.set_model(provider.id()); + step.set_metadata("iteration", serde_json::json!(iterations)); + step.set_metadata("tool_count", serde_json::json!(schemas_for_api.len())); + step.set_input(serde_json::Value::Array( + messages.iter().map(ChatMessage::to_openai_value).collect(), + )); + step + }); + let mut stream = provider.stream_with_tools_and_options( messages.clone(), schemas_for_api.clone(), @@ -314,6 +341,9 @@ pub async fn run_agent_loop_streaming_with_limits( while let Some(event) = stream.next().await { match event { StreamEvent::Delta(text) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } accumulated_text.push_str(&text); if let Some(cb) = on_event { cb(RunnerEvent::TextDelta(text.clone())); @@ -423,6 +453,14 @@ pub async fn run_agent_loop_streaming_with_limits( // Handle stream errors — retry on transient failures/rate limits. if let Some(err) = stream_error { + // Close the generation as failed before any early return, so a + // failed call is visible in the backend rather than silently + // ending as a span with no error on it. + if let Some(mut step) = generation_step.take() { + step.fail(err.clone()); + step.set_usage(instrumentation::to_token_usage(&request_usage)); + step.finish(); + } if is_context_window_error(&err) { return Err(AgentRunError::ContextWindowExceeded(err)); } @@ -453,6 +491,17 @@ pub async fn run_agent_loop_streaming_with_limits( return Err(AgentRunError::Other(anyhow::anyhow!(err))); } + if let Some(mut step) = generation_step.take() { + step.set_usage(instrumentation::to_token_usage(&request_usage)); + if !accumulated_text.is_empty() { + step.set_output(serde_json::Value::String(accumulated_text.clone())); + } + if !tool_calls.is_empty() { + step.set_metadata("tool_calls", serde_json::json!(tool_calls.len())); + } + step.finish(); + } + usage_accumulator.record_request(request_usage.clone()); // Finalize tool call arguments from accumulated strings. @@ -636,6 +685,12 @@ pub async fn run_agent_loop_streaming_with_limits( tool_calls = total_tool_calls, "streaming agent loop complete — returning text" ); + if let Some(recorder) = turn_recorder.as_ref() { + recorder.set_output(serde_json::Value::String(final_text.clone())); + recorder.set_metadata("iterations", serde_json::json!(iterations)); + recorder.set_metadata("tool_calls", serde_json::json!(total_tool_calls)); + recorder.finish(); + } return Ok(finish_agent_run( final_text, iterations, @@ -689,6 +744,7 @@ pub async fn run_agent_loop_streaming_with_limits( let tool_futures: Vec<_> = tool_calls .iter() .map(|tc| { + let tool_turn_recorder = Arc::clone(&turn_recorder); let sanitized = sanitize_tool_name(&tc.name); if *sanitized != tc.name { debug!(original = %tc.name, sanitized = %sanitized, "sanitized mangled tool name"); @@ -820,6 +876,15 @@ pub async fn run_agent_loop_streaming_with_limits( } } + let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { + let mut step = recorder.step( + instrumentation::tool_observation_kind(&tc_name), + tc_name.clone(), + ); + step.set_input(args.clone()); + step + }); + if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { @@ -848,6 +913,14 @@ pub async fn run_agent_loop_streaming_with_limits( } } + if let Some(mut step) = tool_step.take() { + step.set_output(val.clone()); + if let Some(message) = error_msg.clone() { + step.fail(message); + } + step.finish(); + } + if has_error { (false, serde_json::json!({ "result": val }), error_msg, false) } else { @@ -856,6 +929,10 @@ pub async fn run_agent_loop_streaming_with_limits( } Err(e) => { let err_str = e.to_string(); + if let Some(mut step) = tool_step.take() { + step.fail(err_str.clone()); + step.finish(); + } if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), @@ -878,6 +955,10 @@ pub async fn run_agent_loop_streaming_with_limits( } } else { let err_str = format!("unknown tool: {tc_name}"); + if let Some(mut step) = tool_step.take() { + step.fail(err_str.clone()); + step.finish(); + } ( false, serde_json::json!({ "error": err_str }), diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index 93e3f46ec..f3a50779c 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -569,6 +569,49 @@ port = {port} # Port number (auto-generated for this i # enabled = true # Enable metrics collection # prometheus_endpoint = true # Expose /metrics endpoint +# ══════════════════════════════════════════════════════════════════════════════ +# INSTRUMENTATION (Langfuse / OpenTelemetry / Datadog) +# ══════════════════════════════════════════════════════════════════════════════ +# +# Exports agent runs — LLM calls, tool calls, retrievals — to an external +# backend. Disabled by default: enabling it sends data about your +# conversations to a third party. See docs/src/instrumentation.md. +# +# Backends deliberately receive different data. Langfuse gets the full +# conversation because its cost, session and evaluation features are built on +# it. OTLP and Datadog get operational shape only — latency, errors, model, +# token counts — because prompt bodies in an APM mean unbounded span size, +# cardinality pressure, per-byte billing, and conversation content in a system +# nobody scoped for it. + +# [instrumentation] +# enabled = false # Master switch, gates every backend +# environment = "production" # Reported to every backend +# sample_rate = 1.0 # Fraction of turns traced (0.0-1.0) +# redact = ["customer_ref"] # Extra keys to redact; extends the defaults +# queue_capacity = 10000 # Events are dropped, never blocking a turn + +# [instrumentation.langfuse] +# enabled = false +# host = "https://cloud.langfuse.com" # Or a self-hosted URL +# public_key = "pk-lf-..." +# secret_key = "sk-lf-..." +# capture_input = true # Turn and LLM inputs +# capture_output = true # Turn and LLM outputs +# capture_tool_io = true # Tool arguments and results + +# [instrumentation.otlp] # Grafana Tempo/Alloy, Honeycomb, a collector +# enabled = false +# endpoint = "http://localhost:4318/v1/traces" +# content = "metadata_only" # "full" | "metadata_only" | "none" +# emit_user_id = false # High-cardinality in an APM index + +# [instrumentation.datadog] # Via the Datadog Agent's OTLP intake +# enabled = false +# endpoint = "http://localhost:4318/v1/traces" +# service = "moltis" +# content = "metadata_only" + # ══════════════════════════════════════════════════════════════════════════════ # CRON # ══════════════════════════════════════════════════════════════════════════════ diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 21da6a5d6..5c4816a0f 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -47,6 +47,7 @@ moltis-msteams = { optional = true, workspace = true } moltis-netbird = { optional = true, workspace = true } moltis-network-filter = { features = ["proxy", "service"], optional = true, workspace = true } moltis-nostr = { optional = true, workspace = true } +moltis-observability = { workspace = true } moltis-oauth = { workspace = true } moltis-onboarding = { workspace = true } moltis-openclaw-import = { optional = true, workspace = true } diff --git a/crates/gateway/src/methods/dispatch.rs b/crates/gateway/src/methods/dispatch.rs index 4d0c8f95b..e1c57d02c 100644 --- a/crates/gateway/src/methods/dispatch.rs +++ b/crates/gateway/src/methods/dispatch.rs @@ -95,6 +95,7 @@ const READ_METHODS: &[&str] = &[ "webhooks.delivery.get", "webhooks.delivery.payload", "webhooks.delivery.actions", + "instrumentation.status", "heartbeat.status", "heartbeat.runs", "system-presence", @@ -255,6 +256,7 @@ const WRITE_METHODS: &[&str] = &[ "webhooks.create", "webhooks.update", "webhooks.delete", + "instrumentation.test", "heartbeat.update", "heartbeat.run", "voice.config.save_key", diff --git a/crates/gateway/src/methods/services.rs b/crates/gateway/src/methods/services.rs index 9b6abde09..a519ebf22 100644 --- a/crates/gateway/src/methods/services.rs +++ b/crates/gateway/src/methods/services.rs @@ -506,6 +506,7 @@ mod admin; mod agents; mod channels; mod core; +mod instrumentation; mod modes; mod sessions; mod system; @@ -520,6 +521,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { core::register(reg); system::register(reg); admin::register(reg); + instrumentation::register(reg); voice_personas::register(reg); voicecall::register(reg); } diff --git a/crates/gateway/src/methods/services/instrumentation.rs b/crates/gateway/src/methods/services/instrumentation.rs new file mode 100644 index 000000000..0313277a4 --- /dev/null +++ b/crates/gateway/src/methods/services/instrumentation.rs @@ -0,0 +1,97 @@ +//! `instrumentation.*` RPC methods backing the Instrumentation settings page. + +use moltis_protocol::{ErrorShape, error_codes}; + +use super::MethodRegistry; + +/// Register the `instrumentation.*` namespace. +pub(super) fn register(reg: &mut MethodRegistry) { + reg.register( + "instrumentation.status", + Box::new(|ctx| { + Box::pin(async move { + let status = ctx.state.instrumentation.status(); + let config = &ctx.state.config.instrumentation; + + // The secret key is deliberately absent: the UI shows whether + // one is configured, never its value. + serde_json::to_value(serde_json::json!({ + "active": status.active, + "backends": status.backends, + "skipped": status.skipped, + "config": { + "enabled": config.enabled, + "environment": config.environment, + "sample_rate": config.sample_rate, + "queue_capacity": config.queue_capacity, + "langfuse": { + "enabled": config.langfuse.enabled, + "host": config.langfuse.host, + "public_key": config.langfuse.public_key, + "secret_key_set": config.langfuse.secret_key.is_some(), + "capture_input": config.langfuse.capture_input, + "capture_output": config.langfuse.capture_output, + "capture_tool_io": config.langfuse.capture_tool_io, + }, + "otlp": { + "enabled": config.otlp.enabled, + "endpoint": config.otlp.endpoint, + "content": config.otlp.content, + "emit_user_id": config.otlp.emit_user_id, + }, + "datadog": { + "enabled": config.datadog.enabled, + "endpoint": config.datadog.endpoint, + "service": config.datadog.service, + "api_key_set": config.datadog.api_key.is_some(), + "content": config.datadog.content, + }, + }, + })) + .map_err(|e| ErrorShape::new(error_codes::INTERNAL, e.to_string())) + }) + }), + ); + + reg.register( + "instrumentation.test", + Box::new(|ctx| { + Box::pin(async move { + let backend = ctx + .params + .get("backend") + .and_then(|v| v.as_str()) + .unwrap_or("langfuse"); + + match backend { + "langfuse" => { + let Some(client) = ctx.state.instrumentation.langfuse() else { + return Ok(serde_json::json!({ + "ok": false, + "error": "Langfuse is not enabled, or it failed to start. \ + Save valid credentials first.", + })); + }; + match client.test_connection().await { + Ok(()) => Ok(serde_json::json!({ "ok": true })), + Err(error) => Ok(serde_json::json!({ + "ok": false, + "error": error.to_string(), + })), + } + }, + // OTLP collectors have no standard health endpoint, and + // POSTing a probe span would pollute the operator's traces + // with fake data. Sink counters are the honest signal. + other => Ok(serde_json::json!({ + "ok": false, + "error": format!( + "no connection test available for `{other}`; check the \ + delivery counters after the next agent run instead" + ), + })), + } + }) + }), + ); +} diff --git a/crates/gateway/src/server/instrumentation.rs b/crates/gateway/src/server/instrumentation.rs new file mode 100644 index 000000000..6bf5e07a6 --- /dev/null +++ b/crates/gateway/src/server/instrumentation.rs @@ -0,0 +1,234 @@ +//! Startup wiring for agent instrumentation. +//! +//! Builds every configured backend and installs the resulting fanout as the +//! process-wide sink, which the agent runner then discovers without any +//! plumbing through its call signatures. + +use std::sync::{Arc, RwLock}; + +use { + moltis_config::InstrumentationConfig, + moltis_observability::{ + BuiltInstrumentation, SkippedBackend, exporters::langfuse::LangfuseClient, + }, + tracing::{info, warn}, +}; + +/// Live instrumentation state, surfaced by the `instrumentation.*` RPC methods. +#[derive(Default)] +pub struct InstrumentationState { + inner: RwLock>, +} + +/// What is currently running. +struct ActiveInstrumentation { + backends: Vec, + skipped: Vec, + langfuse: Option>, +} + +/// Status reported to the settings UI. +#[derive(Debug, Clone, serde::Serialize)] +pub struct InstrumentationStatus { + /// Whether any backend is actively exporting. + pub active: bool, + /// Backends that are running. + pub backends: Vec, + /// Backends that were enabled in config but could not start, with reasons. + /// Surfaced rather than only logged: a silently disabled exporter is + /// indistinguishable from a broken one. + pub skipped: Vec, +} + +impl InstrumentationState { + /// Build from config and install the sink. Replaces any previous setup, so + /// the settings UI can reconfigure without a restart. + pub fn apply(&self, config: &InstrumentationConfig, release: &str) -> InstrumentationStatus { + let outcome = moltis_observability::build(config, release); + + let status = match &outcome.built { + Some(built) => InstrumentationStatus { + active: true, + backends: built.backends.clone(), + skipped: outcome.skipped.clone(), + }, + None => InstrumentationStatus { + active: false, + backends: Vec::new(), + skipped: outcome.skipped.clone(), + }, + }; + + match outcome.built { + Some(BuiltInstrumentation { + sink, + langfuse, + backends, + .. + }) => { + moltis_observability::set_global_sink(sink); + info!(backends = ?backends, "agent instrumentation active"); + self.store(Some(ActiveInstrumentation { + backends, + skipped: outcome.skipped, + langfuse, + })); + }, + None => { + // Tear down rather than leaving a stale sink installed, so + // disabling instrumentation in the UI takes effect immediately. + moltis_observability::clear_global_sink(); + if !outcome.skipped.is_empty() { + warn!( + skipped = ?outcome.skipped, + "instrumentation configured but no backend could start" + ); + } + self.store(None); + }, + } + + status + } + + /// Current status. + #[must_use] + pub fn status(&self) -> InstrumentationStatus { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.as_ref().map_or_else( + || InstrumentationStatus { + active: false, + backends: Vec::new(), + skipped: Vec::new(), + }, + |active| InstrumentationStatus { + active: true, + backends: active.backends.clone(), + skipped: active.skipped.clone(), + }, + ) + } + + /// The Langfuse client, when that backend is running. + #[must_use] + pub fn langfuse(&self) -> Option> { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.as_ref().and_then(|a| a.langfuse.clone()) + } + + /// Flush every backend, for a clean shutdown. + pub async fn flush(&self, timeout: std::time::Duration) { + let Some(sink) = moltis_observability::global_sink() else { + return; + }; + if let Err(error) = sink.flush(timeout).await { + warn!(%error, "instrumentation flush failed during shutdown"); + } + } + + fn store(&self, value: Option) { + match self.inner.write() { + Ok(mut guard) => *guard = value, + Err(poisoned) => *poisoned.into_inner() = value, + } + } +} + +#[cfg(test)] +mod tests { + use {moltis_config::LangfuseSettings, secrecy::Secret}; + + use super::*; + + fn valid_langfuse_config() -> InstrumentationConfig { + InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + } + } + + #[tokio::test] + async fn disabled_config_reports_inactive_and_installs_no_sink() { + let state = InstrumentationState::default(); + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + assert!(status.backends.is_empty()); + assert!(!moltis_observability::is_enabled()); + + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn applying_a_valid_config_installs_the_sink() { + let state = InstrumentationState::default(); + let status = state.apply(&valid_langfuse_config(), "20260726.01"); + + assert!(status.active); + assert_eq!(status.backends, vec!["langfuse"]); + assert!(moltis_observability::is_enabled()); + assert!(state.langfuse().is_some()); + + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn reapplying_a_disabled_config_tears_the_sink_down() { + let state = InstrumentationState::default(); + state.apply(&valid_langfuse_config(), "test"); + assert!(moltis_observability::is_enabled()); + + // Turning instrumentation off in the UI must take effect immediately + // rather than leaving the previous sink exporting. + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + assert!(!moltis_observability::is_enabled()); + assert!(state.langfuse().is_none()); + } + + #[tokio::test] + async fn skipped_backends_are_reported_not_just_logged() { + let mut config = valid_langfuse_config(); + config.langfuse.public_key = String::new(); + + let state = InstrumentationState::default(); + let status = state.apply(&config, "test"); + + assert!(!status.active); + assert_eq!(status.skipped.len(), 1); + assert_eq!(status.skipped[0].name, "langfuse"); + assert!(status.skipped[0].reason.contains("public_key")); + + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn status_before_any_apply_is_inactive() { + let state = InstrumentationState::default(); + let status = state.status(); + + assert!(!status.active); + assert!(status.backends.is_empty()); + } + + #[tokio::test] + async fn flush_without_a_sink_is_a_no_op() { + moltis_observability::clear_global_sink(); + let state = InstrumentationState::default(); + state.flush(std::time::Duration::from_millis(10)).await; + } +} diff --git a/crates/gateway/src/server/mod.rs b/crates/gateway/src/server/mod.rs index ff32319dc..26db8a852 100644 --- a/crates/gateway/src/server/mod.rs +++ b/crates/gateway/src/server/mod.rs @@ -32,6 +32,8 @@ mod tests_legacy; // ── Re-exports ─────────────────────────────────────────────────────────────── // Preserves the original public API surface of `crate::server::*`. +pub mod instrumentation; + pub(crate) use hooks::discover_and_build_hooks; pub use { helpers::approval_manager_from_config, diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index 28bf6d7fb..9ddb87471 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -405,6 +405,26 @@ pub(super) async fn complete_startup( vault.clone(), ); + // ── Agent instrumentation ───────────────────────────────────────────── + // Applied on the state's own instance so the RPC handlers and the shutdown + // flush observe the same backends that were installed here. Runs before + // any agent can be invoked, so the first turn is traced. + { + let status = state + .instrumentation + .apply(&state.config.instrumentation, &state.version); + if status.active { + tracing::info!(backends = ?status.backends, "agent instrumentation enabled"); + } + for skipped in &status.skipped { + tracing::warn!( + backend = %skipped.name, + reason = %skipped.reason, + "instrumentation backend could not start" + ); + } + } + { let (webhook_tx, webhook_rx) = tokio::sync::mpsc::channel::(256); let webhook_store: Arc = { diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index 857f9f56a..32ae25099 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -415,6 +415,9 @@ pub struct GatewayState { /// Code index for workspace codebase intelligence (discover, filter, status, peek). /// Always initialized in config-only mode; search is deferred to QMD backend. pub code_index: Arc, + /// Live agent instrumentation (Langfuse / OTLP / Datadog). + /// `Arc` because the RPC handlers and the shutdown path both reach it. + pub instrumentation: Arc, /// Whether the server is bound to a loopback address (localhost/127.0.0.1/::1). pub localhost_only: bool, /// Whether the server is known to be behind a reverse proxy. @@ -568,6 +571,9 @@ impl GatewayState { pairing_store, memory_manager, code_index, + // Constructed empty; `apply` runs later from `prepare_core`, which + // is inside a Tokio runtime (each backend spawns an export task). + instrumentation: Arc::default(), localhost_only, behind_proxy, tls_active, diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index e55f49cd2..9ff70c6a1 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -179,13 +179,16 @@ impl TurnRecorder { } /// Close the turn, emitting the final trace state. - pub fn finish(self) { + /// + /// Takes `&self` rather than `self` so the recorder can be shared across + /// the concurrently-executing tool futures in the agent loop. + pub fn finish(&self) { let trace = self.snapshot_trace(); self.sink.record(Event::Trace(Box::new(trace))); } /// Close the turn as failed. - pub fn finish_with_error(self, message: impl Into) { + pub fn finish_with_error(&self, message: impl Into) { self.set_metadata("error", serde_json::Value::String(message.into())); self.finish(); } diff --git a/docs/src/SUMMARY.md b/docs/src/SUMMARY.md index 3500f903c..4e2364b0e 100644 --- a/docs/src/SUMMARY.md +++ b/docs/src/SUMMARY.md @@ -91,6 +91,7 @@ - [Streaming](streaming.md) - [SQLite Migrations](sqlite-migration.md) - [Metrics & Tracing](metrics-and-tracing.md) +- [Instrumentation](instrumentation.md) - [Tool Registry](tool-registry.md) - [Tool Policy](tool-policy.md) - [Agent Presets](agent-presets.md) diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md new file mode 100644 index 000000000..ed8415dea --- /dev/null +++ b/docs/src/instrumentation.md @@ -0,0 +1,288 @@ +# Instrumentation + +Moltis can export what an agent run actually did — every LLM call, tool +invocation and retrieval, with timings, token usage and errors — to an external +backend. + +Three backends are supported, and they are configured together under one +`[instrumentation]` section because they are fed from a single instrumentation +pass in the agent runtime: + +| Backend | What it is for | +| --- | --- | +| **Langfuse** | LLM observability: prompts, completions, cost, sessions, prompt versions, evaluation | +| **OTLP** | Any OpenTelemetry backend — Grafana Tempo/Alloy, Honeycomb, an OpenTelemetry Collector | +| **Datadog** | Datadog APM, through the Datadog Agent's OTLP intake | + +> **Instrumentation is disabled by default.** Unlike every other Moltis feature +> flag, it stays off until you turn it on: enabling it sends data about your +> conversations to a third party. Read [What gets sent](#what-gets-sent) before +> switching it on. + +## Backends receive different data, on purpose + +This is the most important thing to understand about the design. + +Langfuse is an LLM-native product. Its cost attribution, session replay, +prompt-version comparison and evaluation features are all built on having the +actual conversation. Sending it prompts and completions is the entire point. + +Datadog and Grafana are infrastructure tools. Sending them prompt bodies is +actively harmful: + +- span size becomes unbounded, and most vendors bill per ingested byte; +- prompt text as a span attribute is a cardinality problem in a trace index; +- it copies user conversation content into a system that was never scoped, + reviewed, or access-controlled for it. + +So Moltis applies a different **export profile** per backend: + +| | Langfuse | OTLP / Datadog | +| --- | --- | --- | +| Prompts, completions, tool arguments and results | ✅ sent | ❌ **never sent** by default | +| Payload sizes (`moltis.input.bytes`) | — | ✅ sent | +| Observation type (`AGENT`, `TOOL`, `RETRIEVER`, …) | ✅ `langfuse.observation.type` | ✅ as `gen_ai.operation.name` | +| Model, latency, errors | ✅ | ✅ | +| Token counts | ✅ with cache split and cost | ✅ counts only | +| End-user id | ✅ | ❌ off by default (cardinality) | +| Session id | ✅ | ✅ | +| Tags | ✅ | ✅ OTLP · ❌ Datadog (billing) | +| Managed-prompt version | ✅ | — | + +You can raise an APM's content level deliberately with +`content = "full"`, or lower it to `none`. You cannot accidentally end up +sending prompts to Datadog: the default for any newly configured OTLP-family +backend is `metadata_only`. + +**For Grafana specifically:** most of what you want is already available +without this feature. Moltis exposes a Prometheus endpoint at `/metrics` with +`moltis_llm_*` counters for tokens, latency and time-to-first-token. Scrape +that for dashboards and alerts; use the OTLP exporter here when you want +per-run traces to correlate a latency spike with a specific agent run. + +## Quick start: Langfuse + +```toml +[instrumentation] +enabled = true +environment = "production" + +[instrumentation.langfuse] +enabled = true +host = "https://cloud.langfuse.com" # or your self-hosted URL +public_key = "pk-lf-..." +secret_key = "sk-lf-..." +``` + +Keys come from your Langfuse project settings. For a self-hosted deployment, +point `host` at your own instance — no data leaves your network. + +Use **Settings → Instrumentation → Test connection** in the web UI to verify +the host is reachable and the credentials are accepted before relying on it. + +## Quick start: Grafana / OpenTelemetry Collector + +```toml +[instrumentation] +enabled = true + +[instrumentation.otlp] +enabled = true +endpoint = "http://localhost:4318/v1/traces" +content = "metadata_only" # default; "full" or "none" also accepted + +[instrumentation.otlp.headers] +Authorization = "Basic ..." # if your collector requires auth +``` + +## Quick start: Datadog + +Run the Datadog Agent with OTLP ingest enabled and point Moltis at it. This is +the recommended setup — the Agent handles batching, retries and your API key, +so no Datadog credential needs to live in the Moltis config at all. + +```toml +[instrumentation] +enabled = true + +[instrumentation.datadog] +enabled = true +endpoint = "http://localhost:4318/v1/traces" +service = "moltis" +``` + +To post to Datadog's intake directly instead, set `endpoint` to the regional +OTLP intake URL and supply `api_key`. + +## Running several backends at once + +Backends are independent. Enabling all three sends full traces to Langfuse and +operational spans to Grafana and Datadog from the same run, each with its own +content policy. A backend that fails to start is reported in the settings UI +with a reason; the others keep running. + +## What gets sent + +With Langfuse enabled and default settings, each agent run produces: + +- a **trace** named `agent-run`, carrying the user's message and the final + answer; +- a **`GENERATION`** observation per LLM call, with the full message array sent + to the provider, the completion text, model, token usage split into fresh / + cache-read / cache-write buckets, and time-to-first-token; +- a **`TOOL`** (or **`RETRIEVER`**) observation per tool call, with arguments + and results; +- the session key as the session id, so a Langfuse session matches a Moltis + conversation; +- the channel sender as the user id, namespaced per channel (`telegram:42`). + +You can narrow this without turning the backend off: + +```toml +[instrumentation.langfuse] +capture_input = false # drop turn and LLM inputs +capture_output = false # drop turn and LLM outputs +capture_tool_io = false # drop tool arguments and results +``` + +`capture_tool_io` is separate from the others because tool arguments are the +likeliest place for a credential to appear — a `curl` command with a bearer +token, a connection string, an API key passed to a script. + +### Redaction + +Redaction runs before anything is serialized, in two passes: + +1. **By key name** — any object key containing `password`, `secret`, `token`, + `api_key`, `authorization`, `credential`, `private_key` and similar has its + value replaced with `[REDACTED]`, at any nesting depth. +2. **By value shape** — string values that look like credentials are replaced + even under an innocuous key, because tool output is unstructured. Covers + `sk-`, `ghp_`, `github_pat_`, `xoxb-`, `AKIA`, `AIza`, `Bearer …`, PEM + private key blocks and others. + +Add your own key patterns; they extend the built-in list rather than replacing +it, so you cannot accidentally weaken the baseline: + +```toml +[instrumentation] +redact = ["customer_ref", "internal_id"] +``` + +Redaction is a strong mitigation, not a guarantee. It cannot detect a secret +that looks like ordinary prose. If your agents routinely handle sensitive data, +prefer `capture_tool_io = false`, a self-hosted Langfuse, or both. + +### Transport security + +Plaintext `http://` endpoints are refused for non-loopback hosts. These +payloads carry conversation content and a credential in a header; shipping them +unencrypted across a network is not something you should be able to do by +mistyping a URL. `http://localhost` and `http://127.0.0.1` are allowed, for a +local Agent or collector. + +## Sampling and volume + +```toml +[instrumentation] +sample_rate = 1.0 # fraction of turns traced, 0.0-1.0 +queue_capacity = 10000 # bounded export queue +flush_interval_ms = 5000 +max_batch_bytes = 3000000 +``` + +Sampling is per-turn: a sampled turn is traced completely, so you never get a +trace with holes in it. + +Telemetry never blocks an agent. Events go into a bounded queue and are +**dropped** if it fills, rather than slowing a user's turn. Drops are counted +and shown in the settings UI — if you see them, raise `queue_capacity` or lower +`sample_rate`. + +## Configuration reference + +### `[instrumentation]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | Master switch. Off by default. | +| `environment` | `"production"` | Reported to every backend. | +| `release` | running version | Build identifier. | +| `sample_rate` | `1.0` | Fraction of turns traced. | +| `redact` | `[]` | Extra key patterns to redact. | +| `queue_capacity` | `10000` | Bounded export queue depth. | +| `flush_interval_ms` | `5000` | Maximum time an event waits before export. | +| `max_batch_bytes` | `3000000` | Forced flush threshold. | + +### `[instrumentation.langfuse]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `host` | `https://cloud.langfuse.com` | Set to your self-hosted URL to keep data on-premises. | +| `public_key` | `""` | | +| `secret_key` | unset | Held as a secret; never logged or shown in config dumps. | +| `capture_input` | `true` | | +| `capture_output` | `true` | | +| `capture_tool_io` | `true` | | +| `timeout_secs` | `10` | | + +### `[instrumentation.otlp]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `endpoint` | `""` | Full traces URL, e.g. `http://localhost:4318/v1/traces`. | +| `headers` | `{}` | Extra headers, typically auth. | +| `content` | `"metadata_only"` | `full`, `metadata_only`, or `none`. | +| `emit_user_id` | `false` | High-cardinality in an APM index. | +| `timeout_secs` | `10` | | + +### `[instrumentation.datadog]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `false` | | +| `endpoint` | `http://localhost:4318/v1/traces` | The Datadog Agent's OTLP intake. | +| `api_key` | unset | Only needed when posting to the intake directly. | +| `service` | `"moltis"` | Datadog service name. | +| `content` | `"metadata_only"` | | +| `timeout_secs` | `10` | | + +## How it works + +All three backends are fed over **OTLP/HTTP with a JSON payload**. + +For Langfuse this is deliberate rather than incidental. OTLP is Langfuse's own +modern ingest path — the one its v3+ Python and v4+ JavaScript SDKs use — and +the only one that carries the current observation taxonomy. Langfuse's native +`/api/public/ingestion` API still exists, but its `observation-create` event is +marked deprecated upstream and the supported `span-create` / `generation-create` +pair cannot express `AGENT`, `TOOL`, `RETRIEVER` and the other newer types at +all. The ingestion API is still used for **scores**, which OTLP has no way to +represent. + +Spans carry two attribute vocabularies depending on the profile: + +- `langfuse.*` — Langfuse's own mapping, for the observation taxonomy, usage + and cost detail, and prompt linkage. +- `gen_ai.*` — the OpenTelemetry GenAI semantic conventions, understood by + Grafana, Datadog, Honeycomb and other OTel consumers. + +## Troubleshooting + +**No traces appear.** Check `enabled = true` at both the `[instrumentation]` +level and the backend level — the master switch gates everything. Then use +Test connection in the settings UI. + +**Traces appear but token counts look wrong.** Moltis reports fresh input +tokens separately from cache reads and writes, because Langfuse prices them +differently. A run with heavy prompt caching will show a low `input` and a +large `input_cache_read`; that is correct, not a bug. + +**Events are being dropped.** The export queue is full — the backend is slower +than the agent is producing events. Raise `queue_capacity`, lower +`sample_rate`, or check backend latency. + +**A backend shows as skipped.** The settings UI gives the reason: missing +credentials, an empty endpoint, or a rejected plaintext URL. From ab11dcee96a03acfb165855b1f451b040a8dbf1f Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 05:47:05 +0000 Subject: [PATCH 03/33] feat(web): add combined Instrumentation settings page One page for Langfuse, OTLP and Datadog, since all three are fed from the same instrumentation pass and configuring them in separate places would misrepresent how they relate. The page leads with what each backend actually receives. That asymmetry is the design rather than an implementation detail, and an operator who does not know about it will either point Datadog at their prompts or wonder why their APM spans look empty compared to Langfuse. Backends that are enabled in config but failed to start are shown with the reason next to the backend they belong to, so a missing key or a rejected plaintext endpoint is visible where someone is already looking rather than only in the server log. instrumentation.status deliberately reports secret_key_set rather than the key: returning the value would place it in every browser devtools network log. An E2E test asserts that, alongside coverage for the backend cards, the disabled-by-default state, and the test-connection button being unavailable while Langfuse is off. --- .../web/ui/e2e/specs/instrumentation.spec.js | 71 ++++ crates/web/ui/package-lock.json | 30 -- crates/web/ui/src/pages/SettingsPage.tsx | 3 + .../pages/sections/InstrumentationSection.tsx | 305 ++++++++++++++++++ crates/web/ui/src/types/rpc-methods.ts | 4 + 5 files changed, 383 insertions(+), 30 deletions(-) create mode 100644 crates/web/ui/e2e/specs/instrumentation.spec.js create mode 100644 crates/web/ui/src/pages/sections/InstrumentationSection.tsx diff --git a/crates/web/ui/e2e/specs/instrumentation.spec.js b/crates/web/ui/e2e/specs/instrumentation.spec.js new file mode 100644 index 000000000..aac12cc35 --- /dev/null +++ b/crates/web/ui/e2e/specs/instrumentation.spec.js @@ -0,0 +1,71 @@ +const { expect, test } = require("../base-test"); +const { navigateAndWait, watchPageErrors, expectRpcOk } = require("../helpers"); + +test.describe("Instrumentation settings", () => { + test("page loads with the section heading", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/settings/instrumentation"); + + await expect(page.getByRole("heading", { name: "Instrumentation", exact: true })).toBeVisible(); + expect(pageErrors).toEqual([]); + }); + + test("shows a card for every backend", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // All three backends are always listed, including the disabled ones, so + // an operator can see what is available without reading the docs first. + await expect(page.getByText("Langfuse", { exact: true })).toBeVisible(); + await expect(page.getByText("Datadog", { exact: true })).toBeVisible(); + await expect( + page.getByText("OpenTelemetry (Grafana, Honeycomb, Collector)", { exact: true }), + ).toBeVisible(); + }); + + test("explains that backends receive different data", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // The Langfuse/APM content asymmetry is the single most important thing + // for an operator to understand before enabling anything, so it must be + // on the page itself rather than only in the docs. + await expect(page.getByText("What each backend receives", { exact: true })).toBeVisible(); + }); + + test("reports instrumentation as off by default", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + // Defaulting to on would ship conversation content to a third party + // without the operator having chosen to. + await expect(page.getByText(/Instrumentation is off/)).toBeVisible(); + }); + + test("test-connection button is disabled while Langfuse is off", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + const button = page.getByRole("button", { name: "Test connection" }); + await expect(button).toBeVisible(); + await expect(button).toBeDisabled(); + }); + + test("status RPC responds", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + await expectRpcOk(page, "instrumentation.status", {}); + }); + + test("status RPC never returns the secret key", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + const res = await expectRpcOk(page, "instrumentation.status", {}); + + // The UI only ever needs to know whether a key is configured. Returning + // the value would put it in every browser devtools network log. + const serialized = JSON.stringify(res); + expect(serialized).not.toContain("secret_key\""); + expect(serialized).toContain("secret_key_set"); + }); + + test("page has no JS errors", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/settings/instrumentation"); + expect(pageErrors).toEqual([]); + }); +}); diff --git a/crates/web/ui/package-lock.json b/crates/web/ui/package-lock.json index 796f7649d..bbc2436c3 100644 --- a/crates/web/ui/package-lock.json +++ b/crates/web/ui/package-lock.json @@ -1606,9 +1606,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1626,9 +1623,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1646,9 +1640,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1666,9 +1657,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1686,9 +1674,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -1706,9 +1691,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -4131,9 +4113,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4155,9 +4134,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4179,9 +4155,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4203,9 +4176,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/crates/web/ui/src/pages/SettingsPage.tsx b/crates/web/ui/src/pages/SettingsPage.tsx index 5f689e8c1..0bcf7267c 100644 --- a/crates/web/ui/src/pages/SettingsPage.tsx +++ b/crates/web/ui/src/pages/SettingsPage.tsx @@ -42,6 +42,7 @@ import { ConfigSection, GraphqlSection } from "./sections/ConfigSection"; import { EnvironmentSection } from "./sections/EnvironmentSection"; import { IdentitySection } from "./sections/IdentitySection"; import { ImportSection } from "./sections/ImportSection"; +import { InstrumentationSection } from "./sections/InstrumentationSection"; import { MemorySection } from "./sections/MemorySection"; import { NotificationsSection } from "./sections/NotificationsSection"; import { PhoneSection } from "./sections/PhoneSection"; @@ -200,6 +201,7 @@ const sections: SectionItem[] = [ { id: "terminal", label: "Terminal", page: true }, { id: "monitoring", label: "Monitoring", page: true }, { id: "logs", label: "Logs", page: true }, + { id: "instrumentation", label: "Instrumentation" }, { id: "graphql", label: "GraphQL" }, { id: "config", label: "Configuration" }, ]; @@ -410,6 +412,7 @@ function SettingsPage(): VNode { {section === "phone" ? : null} {section === "notifications" ? : null} {section === "import" ? : null} + {section === "instrumentation" ? : null} {section === "graphql" ? : null} {section === "config" ? : null} diff --git a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx new file mode 100644 index 000000000..a3790b3f6 --- /dev/null +++ b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx @@ -0,0 +1,305 @@ +// ── Instrumentation section ───────────────────────────────── +// +// One page for every tracing backend, because they are all fed from a single +// instrumentation pass in the agent runtime. The page is deliberately explicit +// about the fact that Langfuse and the APM backends receive *different* data: +// that asymmetry is the design, not an accident, and an operator who does not +// know about it will either send prompts somewhere they should not, or wonder +// why their Datadog spans look empty. + +import type { VNode } from "preact"; +import { useEffect, useState } from "preact/hooks"; +import { Badge, Loading, SectionHeading, StatusMessage, SubHeading } from "../../components/forms"; +import { sendRpc } from "../../helpers"; +import { rerender } from "./_shared"; + +type ContentMode = "full" | "metadata_only" | "none"; + +interface SkippedBackend { + name: string; + reason: string; +} + +interface LangfuseConfig { + enabled: boolean; + host: string; + public_key: string; + secret_key_set: boolean; + capture_input: boolean; + capture_output: boolean; + capture_tool_io: boolean; +} + +interface OtlpConfig { + enabled: boolean; + endpoint: string; + content: ContentMode; + emit_user_id: boolean; +} + +interface DatadogConfig { + enabled: boolean; + endpoint: string; + service: string; + api_key_set: boolean; + content: ContentMode; +} + +interface InstrumentationConfig { + enabled: boolean; + environment: string; + sample_rate: number; + queue_capacity: number; + langfuse: LangfuseConfig; + otlp: OtlpConfig; + datadog: DatadogConfig; +} + +interface InstrumentationStatus { + active: boolean; + backends: string[]; + skipped: SkippedBackend[]; + config: InstrumentationConfig; +} + +interface TestResult { + ok: boolean; + error?: string; +} + +const CONTENT_LABELS: Record = { + full: "Full conversation", + metadata_only: "Metadata only", + none: "Nothing", +}; + +// ── Read-only field rows ──────────────────────────────────── + +interface RowProps { + label: string; + value: string; + hint?: string; +} + +function Row({ label, value, hint }: RowProps): VNode { + return ( +
+ {label} + + {value} + {hint ? {hint} : null} + +
+ ); +} + +interface BackendCardProps { + title: string; + purpose: string; + enabled: boolean; + running: boolean; + skippedReason?: string; + children: VNode | VNode[]; +} + +function BackendCard({ title, purpose, enabled, running, skippedReason, children }: BackendCardProps): VNode { + return ( +
+
+ {title} + {running ? ( + + ) : enabled ? ( + + ) : ( + + )} +
+

{purpose}

+ {skippedReason ? ( +

Enabled in config but did not start: {skippedReason}

+ ) : null} + {children} +
+ ); +} + +// ── Section ───────────────────────────────────────────────── + +interface BackendsProps { + config: InstrumentationConfig; + status: InstrumentationStatus; + testing: boolean; + onTestLangfuse: () => void; +} + +/** The three backend cards, extracted so the section body stays readable. */ +function Backends({ config, status, testing, onTestLangfuse }: BackendsProps): VNode { + const skippedFor = (name: string): string | undefined => status.skipped.find((s) => s.name === name)?.reason; + const isRunning = (name: string): boolean => status.backends.includes(name); + + const captures = + [ + config.langfuse.capture_input ? "input" : null, + config.langfuse.capture_output ? "output" : null, + config.langfuse.capture_tool_io ? "tool I/O" : null, + ] + .filter(Boolean) + .join(", ") || "structure only"; + + return ( + <> + + + + + + + + + + + + + + + + + + + + + + ); +} + +/** Explains why backends receive different data. */ +function ProfileExplainer(): VNode { + return ( +
+ +

+ Langfuse gets the full conversation \u2014 its cost, session and evaluation features are built on it. OTLP and + Datadog get operational shape only: latency, errors, model and token counts, with payload sizes instead of + payloads. Prompt bodies in an APM mean unbounded span size, cardinality pressure, per-byte ingest billing, and + conversation content in a system nobody scoped for it. +

+
+ ); +} + +export function InstrumentationSection(): VNode { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + const [success, setSuccess] = useState(null); + const [testing, setTesting] = useState(false); + + useEffect(() => { + sendRpc("instrumentation.status", {}) + .then((res) => { + if (res.payload) setStatus(res.payload); + else setError(res.error?.message ?? "Failed to load instrumentation status"); + }) + .catch((e: Error) => setError(e.message)) + .finally(() => { + setLoading(false); + rerender(); + }); + }, []); + + function onTestLangfuse(): void { + setTesting(true); + setError(null); + setSuccess(null); + rerender(); + sendRpc("instrumentation.test", { backend: "langfuse" }) + .then((res) => { + if (res.payload?.ok) setSuccess("Langfuse reachable and credentials accepted."); + else setError(res.payload?.error ?? res.error?.message ?? "Connection test failed"); + }) + .catch((e: Error) => setError(e.message)) + .finally(() => { + setTesting(false); + rerender(); + }); + } + + if (loading) return ; + if (!status) { + return ( +
+ + +
+ ); + } + + const { config } = status; + + return ( +
+ + +

+ Exports what each agent run did \u2014 LLM calls, tool calls, retrievals \u2014 to an external backend. + Configured in moltis.toml under [instrumentation]; this page shows what is running and + lets you verify connectivity. +

+ + {config.enabled ? null : ( +

+ Instrumentation is off. It stays off until you enable it explicitly, because turning it on sends data about + your conversations to a third party. +

+ )} + + + + +
+ + + 0 ? status.backends.join(", ") : "none"} /> + + + +
+ + +
+ ); +} diff --git a/crates/web/ui/src/types/rpc-methods.ts b/crates/web/ui/src/types/rpc-methods.ts index 47956db19..b1fafc841 100644 --- a/crates/web/ui/src/types/rpc-methods.ts +++ b/crates/web/ui/src/types/rpc-methods.ts @@ -46,6 +46,10 @@ export interface RpcMethodMap { "chat.send": unknown; "chat.send_sync": unknown; + // ── Instrumentation ───────────────────────────────────────── + "instrumentation.status": unknown; + "instrumentation.test": unknown; + // ── Cron ──────────────────────────────────────────────────── "cron.list": unknown; "cron.remove": unknown; From ad6da5d792566a1cbb488efc242b80818a505da9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 05:49:43 +0000 Subject: [PATCH 04/33] fix(observability): stop duplicating spans on immutable-span backends MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An end-to-end test over the real HTTP path caught this: the eager in-progress span emitted when a step opens was being sent to every backend. Langfuse upserts observations by id, so the start span is exactly what makes a long-running turn visible while it is still executing, with the closing update filling in model, usage and output. A plain OTLP collector — Tempo, Datadog — treats spans as immutable, so it would render the start and the completion as two separate spans sharing one id, duplicating every LLM call and tool call in the trace. Partial spans are now gated on the export profile alongside the other per-backend policy. Langfuse keeps the live view; APM backends receive only completed spans. Adds tests/end_to_end.rs, which drives a recorder through a live BatchSink into a mock collector and asserts on the bytes that actually leave the process: that prompts and completions reach Langfuse, that neither they nor the user id reach a generic OTLP backend, that operational signal still does, that a tool-argument credential is redacted even on the backend that receives everything else, that the span tree nests tool under generation under root, that cache tokens survive unsummed, and that recording 200 steps against a dead backend does not stall the caller. --- .../src/exporters/otlp/mapping.rs | 35 +- crates/observability/src/profile.rs | 27 ++ crates/observability/tests/end_to_end.rs | 323 ++++++++++++++++++ 3 files changed, 382 insertions(+), 3 deletions(-) create mode 100644 crates/observability/tests/end_to_end.rs diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs index 3506372c0..af7a175fb 100644 --- a/crates/observability/src/exporters/otlp/mapping.rs +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -476,9 +476,13 @@ pub fn batch_to_request(events: &[Event], ctx: &ExportContext) -> wire::ExportTr .iter() .filter_map(|event| match event { Event::Trace(trace) => Some(trace_to_span(trace, ctx)), - Event::ObservationStart(obs) | Event::ObservationEnd(obs) => { - Some(observation_to_span(obs, ctx)) - }, + // Backends that treat spans as immutable get only the completion, + // or the same step would appear twice in the trace. + Event::ObservationStart(obs) => ctx + .profile + .emits_partial_spans() + .then(|| observation_to_span(obs, ctx)), + Event::ObservationEnd(obs) => Some(observation_to_span(obs, ctx)), Event::Score(_) => None, }) .collect(); @@ -955,6 +959,31 @@ mod tests { ); } + #[test] + fn in_progress_spans_are_dropped_for_immutable_span_backends() { + let mut obs = generation(); + obs.end_time = None; + let events = vec![Event::ObservationStart(Box::new(obs))]; + + // Langfuse upserts by id, so the live view is worth the extra span. + let lf = batch_to_request(&events, &langfuse_ctx()); + assert_eq!(lf.resource_spans[0].scope_spans[0].spans.len(), 1); + + // Tempo and Datadog would render this and the later completion as two + // separate spans with the same id. + let otel = batch_to_request(&events, &otel_ctx()); + assert!(otel.resource_spans[0].scope_spans[0].spans.is_empty()); + } + + #[test] + fn completed_spans_reach_every_backend() { + let events = vec![Event::ObservationEnd(Box::new(generation()))]; + for context in [langfuse_ctx(), otel_ctx()] { + let request = batch_to_request(&events, &context); + assert_eq!(request.resource_spans[0].scope_spans[0].spans.len(), 1); + } + } + #[test] fn scores_produce_no_spans() { let events = vec![Event::Score(Box::new(ScoreRecord::new( diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs index 126d1e20f..6929ebb07 100644 --- a/crates/observability/src/profile.rs +++ b/crates/observability/src/profile.rs @@ -73,6 +73,14 @@ pub struct ExportProfile { /// Langfuse prices these; an APM generally has no model price table, so /// the same numbers are better served from the Prometheus endpoint. pub emit_usage: bool, + /// Whether to export the in-progress span emitted when a step opens. + /// + /// Langfuse upserts observations by id, so the eager start makes a + /// long-running turn visible while it is still executing and the closing + /// update fills in model, usage and output. A plain OTLP collector treats + /// spans as immutable: it would render the start and the completion as two + /// separate spans with the same id, duplicating every step in the trace. + pub emit_partial_spans: bool, /// Ceiling on any single exported attribute value, in bytes. pub max_attribute_bytes: usize, } @@ -88,6 +96,7 @@ impl ExportProfile { emit_session_id: true, emit_tags: true, emit_usage: true, + emit_partial_spans: true, max_attribute_bytes: 32_768, } } @@ -104,6 +113,7 @@ impl ExportProfile { emit_session_id: true, emit_tags: true, emit_usage: true, + emit_partial_spans: false, max_attribute_bytes: 4_096, } } @@ -120,6 +130,7 @@ impl ExportProfile { emit_session_id: true, emit_tags: false, emit_usage: true, + emit_partial_spans: false, max_attribute_bytes: 4_096, } } @@ -136,6 +147,12 @@ impl ExportProfile { self.content.allows_bodies() } + /// Whether in-progress spans should be exported. + #[must_use] + pub const fn emits_partial_spans(&self) -> bool { + self.emit_partial_spans + } + /// Whether content-derived structural metadata (lengths, counts) may be /// written. Suppressed only under [`ContentCapture::None`]. #[must_use] @@ -203,6 +220,16 @@ mod tests { assert!(!ExportProfile::default().emits_bodies()); } + #[test] + fn only_langfuse_receives_in_progress_spans() { + // Langfuse upserts by observation id, so a start span becomes the + // live view. A plain OTLP collector treats spans as immutable and + // would render start and completion as two duplicate spans. + assert!(ExportProfile::langfuse().emits_partial_spans()); + assert!(!ExportProfile::otel_generic().emits_partial_spans()); + assert!(!ExportProfile::datadog().emits_partial_spans()); + } + #[test] fn langfuse_allows_larger_attributes_than_an_apm() { assert!( diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs new file mode 100644 index 000000000..4dfcdbde8 --- /dev/null +++ b/crates/observability/tests/end_to_end.rs @@ -0,0 +1,323 @@ +//! End-to-end tests over the real export path. +//! +//! The unit tests check the mapping in isolation; these drive a recorder +//! through a live `BatchSink` into an HTTP server and assert on the bytes that +//! actually leave the process. The content-separation guarantee is only worth +//! anything if it holds on the wire, not just in a mapping function. + +use std::{sync::Arc, time::Duration}; + +use { + moltis_observability::{ + BatchConfig, BatchSink, ExportProfile, ObservationKind, ObservationSink, RecorderSettings, + TokenUsage, TraceScope, TurnRecorder, + exporters::otlp::{OtlpConfig, OtlpTransport}, + sink, + }, + serde_json::Value, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }, +}; + +/// The prompt text used throughout, so leak assertions are unambiguous. +const SECRET_PROMPT: &str = "please deploy the production database migration"; +const SECRET_ANSWER: &str = "migration applied to prod-db-01"; + +fn scope() -> TraceScope { + TraceScope { + session_id: Some("agent:main:main".into()), + user_id: Some("telegram:99".into()), + tags: vec!["channel:telegram".into()], + environment: Some("production".into()), + release: Some("20260726.01".into()), + version: None, + } +} + +/// Spin up a mock collector and return it plus the captured request bodies. +async fn collector() -> MockServer { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/traces")) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + server +} + +/// Every OTLP body the mock server received, parsed. +async fn captured_bodies(server: &MockServer) -> Vec { + server + .received_requests() + .await + .unwrap_or_default() + .iter() + .filter_map(|req| serde_json::from_slice::(&req.body).ok()) + .collect() +} + +/// Install a sink pointed at `server` with `profile`, run one scripted agent +/// turn through the recorder, then flush and return the raw request bodies. +async fn run_turn_against(server: &MockServer, profile: ExportProfile) -> Vec { + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "test".into(), + endpoint: format!("{}/v1/traces", server.uri()), + headers: Default::default(), + timeout: Duration::from_secs(5), + service_name: "moltis".into(), + service_version: "20260726.01".into(), + environment: Some("production".into()), + profile, + })); + let batch_sink = Arc::new(BatchSink::spawn(transport, BatchConfig { + flush_interval: Duration::from_millis(20), + ..Default::default() + })); + sink::set_global_sink(batch_sink.clone()); + + { + let recorder = TurnRecorder::begin("agent-run", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_input(Value::String(SECRET_PROMPT.into())); + + let mut generation = recorder.step(ObservationKind::Generation, "anthropic/claude-opus-4"); + generation.set_model("claude-opus-4"); + generation.set_input(serde_json::json!([{ "role": "user", "content": SECRET_PROMPT }])); + generation.mark_first_token(); + generation.set_usage(TokenUsage::from_provider_totals(120, 40, 800, 15)); + generation.set_output(Value::String(SECRET_ANSWER.into())); + let generation_id = generation.id(); + generation.finish(); + + let mut tool = recorder.step_under(ObservationKind::Tool, "exec", generation_id); + tool.set_input(serde_json::json!({ "cmd": "psql", "api_key": "sk-live-abcdefghijkl" })); + tool.set_output(Value::String(SECRET_ANSWER.into())); + tool.finish(); + + recorder.set_output(Value::String(SECRET_ANSWER.into())); + recorder.finish(); + } + + batch_sink + .flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + sink::clear_global_sink(); + + captured_bodies(server).await +} + +/// Every span across every captured request. +fn all_spans(bodies: &[Value]) -> Vec<&Value> { + bodies + .iter() + .filter_map(|b| b["resourceSpans"].as_array()) + .flatten() + .filter_map(|rs| rs["scopeSpans"].as_array()) + .flatten() + .filter_map(|ss| ss["spans"].as_array()) + .flatten() + .collect() +} + +fn span_attr<'a>(span: &'a Value, key: &str) -> Option<&'a Value> { + span["attributes"] + .as_array()? + .iter() + .find(|kv| kv["key"] == key) + .map(|kv| &kv["value"]) +} + +#[tokio::test] +async fn langfuse_profile_delivers_the_full_conversation() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + + assert!(!bodies.is_empty(), "expected at least one export request"); + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // Langfuse is useless without the conversation, so it must be present. + assert!( + raw.contains(SECRET_PROMPT), + "prompt missing from Langfuse export" + ); + assert!( + raw.contains(SECRET_ANSWER), + "completion missing from Langfuse export" + ); + + // Langfuse receives both the in-progress span and the completion; the + // model and usage only exist on the latter. + let spans = all_spans(&bodies); + let generation = spans + .iter() + .find(|s| span_attr(s, "langfuse.observation.model.name").is_some()) + .expect("completed generation span present"); + + assert_eq!( + span_attr(generation, "langfuse.observation.model.name"), + Some(&serde_json::json!({ "stringValue": "claude-opus-4" })) + ); + assert!(span_attr(generation, "langfuse.observation.completion_start_time").is_some()); +} + +#[tokio::test] +async fn generic_otlp_profile_never_puts_conversation_on_the_wire() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::otel_generic()).await; + + assert!(!bodies.is_empty(), "expected at least one export request"); + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // The guarantee that makes it safe to point this at a shared APM. + assert!( + !raw.contains(SECRET_PROMPT), + "prompt text reached a generic OTLP backend" + ); + assert!( + !raw.contains(SECRET_ANSWER), + "completion text reached a generic OTLP backend" + ); + assert!( + !raw.contains("langfuse."), + "langfuse-specific attributes reached a generic OTLP backend" + ); + // Per-user cardinality is a billing and compliance problem in an APM. + assert!(!raw.contains("telegram:99"), "user id reached an APM"); +} + +#[tokio::test] +async fn generic_otlp_profile_still_delivers_operational_signal() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::otel_generic()).await; + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // Withholding content must not mean withholding everything: latency, + // model and token counts are the whole point of the APM export. + assert!(raw.contains("claude-opus-4"), "model missing"); + assert!( + raw.contains("gen_ai.usage.input_tokens"), + "token counts missing" + ); + assert!(raw.contains("moltis.input.bytes"), "payload sizes missing"); + assert!( + raw.contains("gen_ai.conversation.id"), + "session grouping missing" + ); +} + +#[tokio::test] +async fn tool_credentials_are_redacted_even_for_langfuse() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let raw = serde_json::to_string(&bodies).expect("serializable"); + + // The tool argument carried both a benign key and an API key. Redaction + // runs before serialization, so the credential must never reach the wire + // even on the backend that receives everything else. + assert!( + !raw.contains("sk-live-abcdefghijkl"), + "tool credential leaked into the export" + ); + assert!(raw.contains("REDACTED"), "expected a redaction marker"); + assert!(raw.contains("psql"), "benign tool argument should survive"); +} + +#[tokio::test] +async fn the_span_tree_is_correctly_nested() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let spans = all_spans(&bodies); + + let trace_ids: std::collections::HashSet<&str> = + spans.iter().filter_map(|s| s["traceId"].as_str()).collect(); + assert_eq!( + trace_ids.len(), + 1, + "one turn must produce exactly one trace" + ); + + let root = spans + .iter() + .find(|s| s.get("parentSpanId").is_none()) + .expect("a root span must exist"); + let root_id = root["spanId"].as_str().expect("root has an id"); + + // The generation hangs off the root, and the tool hangs off the + // generation. A flat tree would lose which call issued which tool. + let generation = spans + .iter() + .find(|s| s["name"] == "anthropic/claude-opus-4") + .expect("generation span present"); + assert_eq!(generation["parentSpanId"].as_str(), Some(root_id)); + + let tool = spans + .iter() + .find(|s| s["name"] == "exec") + .expect("tool span present"); + assert_eq!( + tool["parentSpanId"].as_str(), + generation["spanId"].as_str(), + "tool must nest under the generation that called it" + ); +} + +#[tokio::test] +async fn cache_tokens_survive_the_round_trip_unsummed() { + let server = collector().await; + let bodies = run_turn_against(&server, ExportProfile::langfuse()).await; + let spans = all_spans(&bodies); + + let usage = spans + .iter() + .find_map(|s| span_attr(s, "langfuse.observation.usage_details")) + .and_then(|v| v["stringValue"].as_str()) + .map(|s| serde_json::from_str::(s).expect("valid json")) + .expect("usage details present on the wire"); + + // Folding cache reads into `input` would inflate the priced fresh-input + // count by nearly an order of magnitude on a cache-heavy run. + assert_eq!(usage["input"], 120); + assert_eq!(usage["input_cache_read"], 800); + assert_eq!(usage["input_cache_write"], 15); + assert_eq!(usage["total"], 975); +} + +#[tokio::test] +async fn an_unreachable_backend_does_not_stall_the_turn() { + // Port 1 is reserved and refuses connections immediately. + let transport = Arc::new(OtlpTransport::new(OtlpConfig { + name: "dead".into(), + endpoint: "http://127.0.0.1:1/v1/traces".into(), + timeout: Duration::from_millis(50), + profile: ExportProfile::langfuse(), + ..Default::default() + })); + let batch_sink = Arc::new(BatchSink::spawn(transport, BatchConfig { + max_retries: 1, + initial_backoff: Duration::from_millis(1), + max_backoff: Duration::from_millis(2), + flush_interval: Duration::from_millis(20), + ..Default::default() + })); + sink::set_global_sink(batch_sink.clone()); + + let started = std::time::Instant::now(); + let recorder = TurnRecorder::begin("agent-run", scope(), RecorderSettings::default()) + .expect("sink installed"); + for _ in 0..200 { + recorder.step(ObservationKind::Tool, "exec").finish(); + } + recorder.finish(); + let elapsed = started.elapsed(); + + sink::clear_global_sink(); + + // Recording is a queue push; a dead backend must not be felt by the agent. + assert!( + elapsed < Duration::from_secs(1), + "recording blocked for {elapsed:?} against a dead backend" + ); +} From a5cc79f5d301741a3899a87fed171a8231d369e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 05:51:44 +0000 Subject: [PATCH 05/33] fix(observability): satisfy workspace clippy gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Takes TokenUsage::to_usage_details by value, since the workspace denies wrong_self_convention and the type is Copy. Adds the standard expect_used/unwrap_used allow to the test modules and the integration test file, matching what the rest of the workspace does — the workspace denies both in production code, and test modules opt out explicitly rather than the lint being relaxed globally. Also corrects the README operations line: it claimed OpenTelemetry tracing while the four opentelemetry workspace dependencies were unused by any crate. It now names what is actually shipped. --- README.md | 2 +- crates/observability/src/builder.rs | 1 + crates/observability/src/exporters/langfuse.rs | 1 + crates/observability/src/exporters/otlp/mapping.rs | 1 + crates/observability/src/exporters/otlp/mod.rs | 1 + crates/observability/src/exporters/otlp/wire.rs | 1 + crates/observability/src/model.rs | 3 ++- crates/observability/src/profile.rs | 1 + crates/observability/src/recorder.rs | 1 + crates/observability/src/redact.rs | 1 + crates/observability/src/runtime.rs | 1 + crates/observability/src/sink.rs | 1 + crates/observability/tests/end_to_end.rs | 1 + 13 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 605105c8a..8902b31ba 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ Verify releases with `gh attestation verify -R moltis-org/moltis` or - **Safer Agent Editing** — Automatic checkpoints before built-in skill and memory mutations, restore tooling, session branching - **Extensibility** — MCP servers (stdio + HTTP/SSE), skill system, 15 lifecycle hook events with circuit breaker, destructive command guard - **Security** — Encryption-at-rest vault (XChaCha20-Poly1305 + Argon2id), password + passkey + API key auth, sandbox isolation, SSRF/CSWSH protection -- **Operations** — Cron scheduling, OpenTelemetry tracing, Prometheus metrics, cloud deploy (Fly.io, DigitalOcean), Tailscale integration, managed SSH deploy keys, host-pinned remote targets, live tool inventory in Settings, and CLI/web remote-exec doctor flows +- **Operations** — Cron scheduling, agent instrumentation (Langfuse for LLM traces, OTLP for Grafana/Datadog), Prometheus metrics, cloud deploy (Fly.io, DigitalOcean), Tailscale integration, managed SSH deploy keys, host-pinned remote targets, live tool inventory in Settings, and CLI/web remote-exec doctor flows ## How It Works diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs index 503d35401..911992977 100644 --- a/crates/observability/src/builder.rs +++ b/crates/observability/src/builder.rs @@ -304,6 +304,7 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use secrecy::Secret; diff --git a/crates/observability/src/exporters/langfuse.rs b/crates/observability/src/exporters/langfuse.rs index 5d279bc2c..b0d2cac8d 100644 --- a/crates/observability/src/exporters/langfuse.rs +++ b/crates/observability/src/exporters/langfuse.rs @@ -304,6 +304,7 @@ impl LangfuseClient { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use wiremock::{ Mock, MockServer, ResponseTemplate, diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs index af7a175fb..56a79268c 100644 --- a/crates/observability/src/exporters/otlp/mapping.rs +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -504,6 +504,7 @@ pub fn batch_to_request(events: &[Event], ctx: &ExportContext) -> wire::ExportTr } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use serde_json::json; diff --git a/crates/observability/src/exporters/otlp/mod.rs b/crates/observability/src/exporters/otlp/mod.rs index 579d5144f..3590aa11a 100644 --- a/crates/observability/src/exporters/otlp/mod.rs +++ b/crates/observability/src/exporters/otlp/mod.rs @@ -192,6 +192,7 @@ impl Transport for OtlpTransport { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use wiremock::{ Mock, MockServer, ResponseTemplate, diff --git a/crates/observability/src/exporters/otlp/wire.rs b/crates/observability/src/exporters/otlp/wire.rs index 66862fb83..f7e9e19f4 100644 --- a/crates/observability/src/exporters/otlp/wire.rs +++ b/crates/observability/src/exporters/otlp/wire.rs @@ -234,6 +234,7 @@ pub struct ArrayValue { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; diff --git a/crates/observability/src/model.rs b/crates/observability/src/model.rs index 5add8237e..5b9e99ccf 100644 --- a/crates/observability/src/model.rs +++ b/crates/observability/src/model.rs @@ -243,7 +243,7 @@ impl TokenUsage { /// Key names match what Langfuse's ingestion recognises for cache-aware /// cost attribution. #[must_use] - pub fn to_usage_details(&self) -> BTreeMap { + pub fn to_usage_details(self) -> BTreeMap { let mut map = BTreeMap::new(); map.insert("input".to_string(), self.input); map.insert("output".to_string(), self.output); @@ -505,6 +505,7 @@ impl Event { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs index 6929ebb07..8d53899bc 100644 --- a/crates/observability/src/profile.rs +++ b/crates/observability/src/profile.rs @@ -168,6 +168,7 @@ impl Default for ExportProfile { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index 9ff70c6a1..ffced3a1f 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -348,6 +348,7 @@ impl Drop for StepGuard { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use std::sync::Mutex as StdMutex; diff --git a/crates/observability/src/redact.rs b/crates/observability/src/redact.rs index eda83ee15..e8be39b48 100644 --- a/crates/observability/src/redact.rs +++ b/crates/observability/src/redact.rs @@ -199,6 +199,7 @@ impl RedactionPolicy { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use {super::*, serde_json::json}; diff --git a/crates/observability/src/runtime.rs b/crates/observability/src/runtime.rs index a0df56146..766059be1 100644 --- a/crates/observability/src/runtime.rs +++ b/crates/observability/src/runtime.rs @@ -358,6 +358,7 @@ fn backoff_delay(config: &BatchConfig, attempt: u32) -> Duration { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use std::sync::Mutex; diff --git a/crates/observability/src/sink.rs b/crates/observability/src/sink.rs index a9a6fad6d..c80307037 100644 --- a/crates/observability/src/sink.rs +++ b/crates/observability/src/sink.rs @@ -152,6 +152,7 @@ pub fn record(event: Event) { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use std::sync::Mutex; diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs index 4dfcdbde8..072ace252 100644 --- a/crates/observability/tests/end_to_end.rs +++ b/crates/observability/tests/end_to_end.rs @@ -1,3 +1,4 @@ +#![allow(clippy::unwrap_used, clippy::expect_used)] //! End-to-end tests over the real export path. //! //! The unit tests check the mapping in isolation; these drive a recorder From 2effb2826d7d1c1700341c58479139f94f5df278 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 06:12:09 +0000 Subject: [PATCH 06/33] feat(observability): instrument the non-streaming agent loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming loop was instrumented but the non-streaming one was not, and that is the path sub-agents take. spawn_agent calls run_agent_loop_with_context_and_limits, and silent_turn calls run_agent_loop, so every sub-agent run and every silent turn produced no trace at all — dropping exactly the nested-agent case the AGENT observation type exists to represent, and leaving a parent trace that showed a tool call with no visible work underneath it. Mirrors the streaming path: a trace per turn, a GENERATION observation per completion carrying model, cache-split usage and the message array, and the same explicit close-as-failed before the retry return so a failed provider call is not rendered as a successful span by the drop guard. --- crates/agents/src/runner/non_streaming.rs | 46 ++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/crates/agents/src/runner/non_streaming.rs b/crates/agents/src/runner/non_streaming.rs index bbfc37509..56e026ad6 100644 --- a/crates/agents/src/runner/non_streaming.rs +++ b/crates/agents/src/runner/non_streaming.rs @@ -27,7 +27,7 @@ use super::{ channel_binding_from_tool_context, dispatch_after_llm_call_hook, dispatch_before_agent_start_hook, empty_tool_name_retry_prompt, explicit_shell_command_from_user_content, find_empty_tool_name_call, finish_agent_run, - has_named_tool_call, is_substantive_answer_text, log_tool_argument_diagnostic, + has_named_tool_call, instrumentation, is_substantive_answer_text, log_tool_argument_diagnostic, record_answer_text, resolve_tool_lookup, retry::{ RATE_LIMIT_MAX_RETRIES, is_context_window_error, next_retry_delay_ms, @@ -157,6 +157,20 @@ pub async fn run_agent_loop_with_context_and_limits( let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); + // Instrumentation. This path serves sub-agents (`spawn_agent`) and silent + // turns, so leaving it uninstrumented would drop exactly the nested-agent + // traces the observation taxonomy exists to represent. + let turn_recorder = Arc::new(instrumentation::begin_turn( + &session_key_for_hooks, + channel_for_hooks.as_ref(), + provider.name(), + provider.id(), + user_content, + instrumentation::recorder_settings(&config.instrumentation), + config.instrumentation.environment.clone(), + instrumentation::release(&config.instrumentation), + )); + dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -288,6 +302,19 @@ pub async fn run_agent_loop_with_context_and_limits( cb(RunnerEvent::Thinking); } + let mut generation_step = turn_recorder.as_ref().as_ref().map(|recorder| { + let mut step = recorder.step( + moltis_observability::ObservationKind::Generation, + instrumentation::generation_name(provider.name(), provider.id()), + ); + step.set_model(provider.id()); + step.set_metadata("iteration", serde_json::json!(iterations)); + step.set_input(serde_json::Value::Array( + messages.iter().map(ChatMessage::to_openai_value).collect(), + )); + step + }); + let mut response = match provider .complete_with_options(&messages, &schemas_for_api, &tool_controls) .await @@ -295,6 +322,12 @@ pub async fn run_agent_loop_with_context_and_limits( Ok(r) => r, Err(e) => { let msg = e.to_string(); + // Close as failed before any early return, so a failed call is + // not rendered as a successful span by the drop guard. + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } if is_context_window_error(&msg) { return Err(AgentRunError::ContextWindowExceeded(msg)); } @@ -329,6 +362,17 @@ pub async fn run_agent_loop_with_context_and_limits( cb(RunnerEvent::ThinkingDone); } + if let Some(mut step) = generation_step.take() { + step.set_usage(instrumentation::to_token_usage(&response.usage)); + if let Some(text) = response.text.clone() { + step.set_output(serde_json::Value::String(text)); + } + if !response.tool_calls.is_empty() { + step.set_metadata("tool_calls", serde_json::json!(response.tool_calls.len())); + } + step.finish(); + } + usage_accumulator.record_request(response.usage.clone()); info!( From bf0f261aef4bb50bd4db6109e7dfab4f6cd7b63b Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 06:16:57 +0000 Subject: [PATCH 07/33] fix(config): allow expect in the instrumentation config test module The workspace denies expect_used and unwrap_used in production code; test modules opt out explicitly, matching the rest of the workspace. The new instrumentation config tests were missing that opt-out and failed the clippy gate. --- crates/config/src/schema/instrumentation.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs index f17151027..ec585cda7 100644 --- a/crates/config/src/schema/instrumentation.rs +++ b/crates/config/src/schema/instrumentation.rs @@ -293,6 +293,7 @@ impl Default for DatadogSettings { } #[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use secrecy::ExposeSecret; From 26f34546e499c2f4975fed0397225a15f2c27373 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:03:30 +0000 Subject: [PATCH 08/33] fix(observability): give scores a real export path and de-flake the e2e suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scores were recorded and then silently discarded. The Langfuse backend carries traces over OTLP, but the OTLP mapping has no representation for a score and drops those events by design, so every `TurnRecorder::score` call ended in a void — the REST client that can post scores was built and then only used by the settings "Test connection" button. Langfuse now contributes two sinks instead of one: traces over OTLP as before, plus a `ScoreSink` onto the ingestion API. The sink filters the fanout down to score events before they reach its queue, so a busy agent's trace volume cannot evict the comparatively rare scores. A rejected batch is classified retryable, because the usual causes (the trace has not landed yet, a 5xx) are transient and user feedback is not worth dropping on the first failure. Also splits the Langfuse exporter into a module directory ahead of the prompt, dataset and media APIs landing next to it. Separately, `TurnRecorder::begin` could only read the process-wide global sink, which made the end-to-end tests flaky at about 1 run in 12: they execute in parallel in one process, so one test's `set_global_sink` would overwrite another's and its spans would land in the wrong collector, leaving the original assertion with an empty body list. Adds `begin_with_sink` so a recorder can be pointed at an explicit destination, and moves the tests onto it. 25 consecutive runs green. The same entry point is what the experiment runner will need to route dataset traces separately from live traffic. --- crates/observability/src/builder.rs | 36 +- .../observability/src/exporters/langfuse.rs | 523 ------------------ .../src/exporters/langfuse/client.rs | 189 +++++++ .../src/exporters/langfuse/config.rs | 178 ++++++ .../src/exporters/langfuse/mod.rs | 24 + .../src/exporters/langfuse/scores.rs | 386 +++++++++++++ crates/observability/src/recorder.rs | 17 +- crates/observability/tests/end_to_end.rs | 28 +- 8 files changed, 839 insertions(+), 542 deletions(-) delete mode 100644 crates/observability/src/exporters/langfuse.rs create mode 100644 crates/observability/src/exporters/langfuse/client.rs create mode 100644 crates/observability/src/exporters/langfuse/config.rs create mode 100644 crates/observability/src/exporters/langfuse/mod.rs create mode 100644 crates/observability/src/exporters/langfuse/scores.rs diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs index 911992977..4df972865 100644 --- a/crates/observability/src/builder.rs +++ b/crates/observability/src/builder.rs @@ -12,7 +12,7 @@ use { use crate::{ exporters::{ - langfuse::{LangfuseClient, LangfuseConfig}, + langfuse::{LangfuseClient, LangfuseConfig, ScoreSink}, otlp::{OtlpConfig, OtlpTransport}, }, profile::{ContentCapture, ExportProfile}, @@ -106,11 +106,16 @@ fn validate_endpoint(raw: &str) -> Result<(), String> { } /// Build the Langfuse backend. +/// +/// Returns two sinks, not one. Traces go over OTLP, but the OTLP mapping has +/// no representation for a score and drops those events, so scores need their +/// own sink onto the ingestion API — otherwise every score recorded would be +/// silently discarded on the way out. fn build_langfuse( settings: &LangfuseSettings, config: &InstrumentationConfig, release: &str, -) -> Result<(Arc, Arc), String> { +) -> Result<(Vec>, Arc), String> { if settings.public_key.trim().is_empty() { return Err("public_key is not set".to_string()); } @@ -132,9 +137,10 @@ fn build_langfuse( }; let transport = Arc::new(langfuse_config.build_transport(release.to_string())); - let sink = Arc::new(BatchSink::spawn(transport, batch_config(config))); + let traces = Arc::new(BatchSink::spawn(transport, batch_config(config))); let client = Arc::new(LangfuseClient::new(langfuse_config)); - Ok((sink, client)) + let scores = Arc::new(ScoreSink::spawn(Arc::clone(&client), batch_config(config))); + Ok((vec![traces, scores], client)) } /// Build a generic OTLP backend. @@ -225,8 +231,8 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { if config.langfuse.enabled { match build_langfuse(&config.langfuse, config, release) { - Ok((sink, client)) => { - sinks.push(sink); + Ok((langfuse_sinks, client)) => { + sinks.extend(langfuse_sinks); langfuse_client = Some(client); backends.push("langfuse".to_string()); }, @@ -415,7 +421,23 @@ mod tests { let built = build(&config, "test").built.expect("should build"); assert_eq!(built.backends, vec!["langfuse", "otlp", "datadog"]); - assert_eq!(built.sink.name(), "langfuse+otlp+datadog"); + // Langfuse contributes two sinks: traces over OTLP and scores over the + // ingestion API. It still reports as one backend to the operator. + assert_eq!(built.sink.name(), "langfuse+langfuse-scores+otlp+datadog"); + } + + #[tokio::test] + async fn langfuse_gets_a_score_sink_alongside_the_trace_sink() { + // The OTLP mapping has no representation for a score and drops those + // events, so without a second sink every score would be recorded by the + // agent and then silently discarded on the way out. + let config = InstrumentationConfig { + langfuse: valid_langfuse(), + ..enabled_config() + }; + let built = build(&config, "test").built.expect("should build"); + + assert_eq!(built.sink.name(), "langfuse+langfuse-scores"); } #[tokio::test] diff --git a/crates/observability/src/exporters/langfuse.rs b/crates/observability/src/exporters/langfuse.rs deleted file mode 100644 index b0d2cac8d..000000000 --- a/crates/observability/src/exporters/langfuse.rs +++ /dev/null @@ -1,523 +0,0 @@ -//! Langfuse backend. -//! -//! Traces reach Langfuse over OTLP — that is Langfuse's own modern ingest path, -//! the one its v3+ Python and v4+ JS SDKs use, and the only one that carries the -//! full observation taxonomy (`AGENT`, `TOOL`, `RETRIEVER`, ...). The native -//! `/api/public/ingestion` event `observation-create` is marked deprecated -//! upstream and its non-deprecated `span-create`/`generation-create` pair -//! cannot express the newer types at all. -//! -//! What OTLP *cannot* express is scores, so those go over the ingestion API. -//! This module owns both, plus the credential handling and the connection test -//! behind the settings UI's "Test connection" button. - -use std::{collections::BTreeMap, time::Duration}; - -use { - base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}, - secrecy::{ExposeSecret, SecretString}, - serde::Serialize, - tracing::debug, -}; - -use { - super::otlp::{OtlpConfig, OtlpTransport}, - crate::{ - model::{ScoreRecord, ScoreValue}, - profile::ExportProfile, - }, -}; - -/// Path Langfuse serves OTLP traces on, appended to the configured host. -pub const OTEL_TRACES_PATH: &str = "/api/public/otel/v1/traces"; -/// Batch ingestion path, used for scores. -pub const INGESTION_PATH: &str = "/api/public/ingestion"; -/// Unauthenticated liveness probe. -pub const HEALTH_PATH: &str = "/api/public/health"; - -/// Langfuse connection settings. -#[derive(Clone)] -pub struct LangfuseConfig { - /// Base host, e.g. `https://cloud.langfuse.com` or a self-hosted URL. - pub host: String, - /// Project public key (Basic auth username). - pub public_key: String, - /// Project secret key (Basic auth password). - pub secret_key: SecretString, - /// Deployment environment. - pub environment: Option, - /// Build release identifier. - pub release: Option, - /// Per-request timeout. - pub timeout: Duration, -} - -// Manual impl: the derived one would print the secret key. -impl std::fmt::Debug for LangfuseConfig { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LangfuseConfig") - .field("host", &self.host) - .field("public_key", &self.public_key) - .field("secret_key", &"[REDACTED]") - .field("environment", &self.environment) - .field("release", &self.release) - .field("timeout", &self.timeout) - .finish() - } -} - -impl LangfuseConfig { - /// Join `path` onto the configured host, tolerating a trailing slash. - #[must_use] - pub fn url(&self, path: &str) -> String { - format!("{}{path}", self.host.trim_end_matches('/')) - } - - /// `Authorization` header value for Basic auth. - /// - /// Langfuse authenticates with the public key as username and the secret - /// key as password. - #[must_use] - pub fn basic_auth_header(&self) -> String { - let raw = format!("{}:{}", self.public_key, self.secret_key.expose_secret()); - format!("Basic {}", BASE64.encode(raw)) - } - - /// Headers every authenticated Langfuse request carries. - #[must_use] - pub fn auth_headers(&self) -> BTreeMap { - let mut headers = BTreeMap::new(); - headers.insert("Authorization".to_string(), self.basic_auth_header()); - // Langfuse surfaces these in its own diagnostics. - headers.insert("X-Langfuse-Sdk-Name".to_string(), "moltis".to_string()); - headers.insert( - "X-Langfuse-Sdk-Version".to_string(), - env!("CARGO_PKG_VERSION").to_string(), - ); - headers.insert("X-Langfuse-Public-Key".to_string(), self.public_key.clone()); - headers - } - - /// Build the OTLP transport that carries traces to Langfuse. - /// - /// The profile is fixed to [`ExportProfile::langfuse`]: content capture is - /// the entire point of sending to Langfuse, and letting it be configured - /// down to `MetadataOnly` would produce traces with no conversation in - /// them, which is worse than not exporting at all. - #[must_use] - pub fn build_transport(&self, service_version: String) -> OtlpTransport { - OtlpTransport::new(OtlpConfig { - name: "langfuse".to_string(), - endpoint: self.url(OTEL_TRACES_PATH), - headers: self.auth_headers(), - timeout: self.timeout, - service_name: "moltis".to_string(), - service_version, - environment: self.environment.clone(), - profile: ExportProfile::langfuse(), - }) - } -} - -// ── Scores ────────────────────────────────────────────────────────────────── - -/// A `score-create` event in the ingestion batch envelope. -#[derive(Debug, Serialize)] -struct IngestionEvent<'a> { - id: String, - timestamp: String, - #[serde(rename = "type")] - kind: &'static str, - body: ScoreBody<'a>, -} - -/// Langfuse `ScoreBody`. -#[derive(Debug, Serialize)] -struct ScoreBody<'a> { - id: &'a str, - #[serde(rename = "traceId")] - trace_id: &'a str, - #[serde(rename = "observationId", skip_serializing_if = "Option::is_none")] - observation_id: Option<&'a str>, - name: &'a str, - value: &'a ScoreValue, - #[serde(rename = "dataType")] - data_type: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - comment: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - environment: Option<&'a str>, -} - -/// The ingestion request envelope. -#[derive(Debug, Serialize)] -struct IngestionRequest<'a> { - batch: Vec>, -} - -/// Client for the Langfuse REST surfaces that OTLP cannot express. -pub struct LangfuseClient { - config: LangfuseConfig, - http: reqwest::Client, -} - -impl LangfuseClient { - /// Build a client over the workspace HTTP client, so the configured - /// upstream proxy applies. - #[must_use] - pub fn new(config: LangfuseConfig) -> Self { - Self { - config, - http: moltis_common::http_client::build_default_http_client(), - } - } - - /// Verify that the host is reachable and the credentials are accepted. - /// - /// Backs the settings UI's "Test connection" button. The health endpoint is - /// unauthenticated, so credentials are checked with an empty ingestion - /// batch: it is the cheapest authenticated call that creates no data. - pub async fn test_connection(&self) -> anyhow::Result<()> { - let health = self - .http - .get(self.config.url(HEALTH_PATH)) - .timeout(self.config.timeout) - .send() - .await - .map_err(|e| anyhow::anyhow!("cannot reach Langfuse at {}: {e}", self.config.host))?; - - if !health.status().is_success() { - return Err(anyhow::anyhow!( - "Langfuse health check failed with HTTP {}", - health.status() - )); - } - - let auth = self - .http - .post(self.config.url(INGESTION_PATH)) - .headers(self.header_map()) - .timeout(self.config.timeout) - .json(&serde_json::json!({ "batch": [] })) - .send() - .await - .map_err(|e| anyhow::anyhow!("Langfuse credential check failed: {e}"))?; - - let status = auth.status(); - match status { - s if s.is_success() => Ok(()), - reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN => Err( - anyhow::anyhow!("Langfuse rejected the credentials (HTTP {status})"), - ), - other => Err(anyhow::anyhow!( - "unexpected response from Langfuse: HTTP {other}" - )), - } - } - - /// Submit scores. Langfuse upserts on score id. - pub async fn submit_scores(&self, scores: &[ScoreRecord]) -> anyhow::Result<()> { - if scores.is_empty() { - return Ok(()); - } - - let now = time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Rfc3339) - .unwrap_or_default(); - - let batch: Vec> = scores - .iter() - .map(|score| IngestionEvent { - id: uuid::Uuid::new_v4().to_string(), - timestamp: now.clone(), - kind: "score-create", - body: ScoreBody { - id: &score.id, - trace_id: &score.trace_id.0, - observation_id: score.observation_id.as_ref().map(|o| o.0.as_str()), - name: &score.name, - value: &score.value, - data_type: match score.value { - ScoreValue::Numeric(_) => "NUMERIC", - ScoreValue::Categorical(_) => "CATEGORICAL", - }, - comment: score.comment.as_deref(), - environment: score - .environment - .as_deref() - .or(self.config.environment.as_deref()), - }, - }) - .collect(); - - let response = self - .http - .post(self.config.url(INGESTION_PATH)) - .headers(self.header_map()) - .timeout(self.config.timeout) - .json(&IngestionRequest { batch }) - .send() - .await?; - - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "Langfuse rejected scores: HTTP {status}: {}", - body.chars().take(512).collect::() - )); - } - - // The ingestion endpoint reports per-event outcomes in a 207, so a 2xx - // alone does not mean every score landed. - let body: serde_json::Value = response.json().await.unwrap_or_default(); - if let Some(errors) = body.get("errors").and_then(|e| e.as_array()) - && !errors.is_empty() - { - return Err(anyhow::anyhow!( - "Langfuse rejected {} of {} scores: {}", - errors.len(), - scores.len(), - serde_json::to_string(errors).unwrap_or_default() - )); - } - - debug!(count = scores.len(), "submitted scores to Langfuse"); - Ok(()) - } - - /// Authenticated header map. - fn header_map(&self) -> reqwest::header::HeaderMap { - let mut map = reqwest::header::HeaderMap::new(); - for (key, value) in self.config.auth_headers() { - let Ok(name) = key.parse::() else { - continue; - }; - let Ok(mut val) = reqwest::header::HeaderValue::from_str(&value) else { - continue; - }; - val.set_sensitive(true); - map.insert(name, val); - } - map - } -} - -#[cfg(test)] -#[allow(clippy::expect_used, clippy::unwrap_used)] -mod tests { - use wiremock::{ - Mock, MockServer, ResponseTemplate, - matchers::{body_json_schema, header, method, path}, - }; - - use { - super::*, - crate::model::{ObservationId, TraceId}, - }; - - fn config(host: String) -> LangfuseConfig { - LangfuseConfig { - host, - public_key: "pk-lf-test".into(), - secret_key: SecretString::new("sk-lf-secret".to_string()), - environment: Some("production".into()), - release: Some("20260726.01".into()), - timeout: Duration::from_secs(5), - } - } - - #[test] - fn basic_auth_encodes_public_and_secret_key() { - let header = config("https://example.com".into()).basic_auth_header(); - let encoded = header.strip_prefix("Basic ").expect("basic scheme"); - let decoded = - String::from_utf8(BASE64.decode(encoded).expect("valid base64")).expect("valid utf-8"); - - assert_eq!(decoded, "pk-lf-test:sk-lf-secret"); - } - - #[test] - fn debug_never_prints_the_secret_key() { - let rendered = format!("{:?}", config("https://example.com".into())); - - assert!( - !rendered.contains("sk-lf-secret"), - "secret leaked: {rendered}" - ); - assert!(rendered.contains("[REDACTED]")); - // The public key is not a secret and is useful in diagnostics. - assert!(rendered.contains("pk-lf-test")); - } - - #[test] - fn url_join_tolerates_a_trailing_slash_on_the_host() { - let with = config("https://example.com/".into()); - let without = config("https://example.com".into()); - - assert_eq!(with.url(OTEL_TRACES_PATH), without.url(OTEL_TRACES_PATH)); - assert_eq!( - without.url(OTEL_TRACES_PATH), - "https://example.com/api/public/otel/v1/traces" - ); - } - - #[test] - fn transport_targets_the_otel_endpoint_with_the_langfuse_profile() { - let transport = config("https://cloud.langfuse.com".into()).build_transport("1.0".into()); - - assert_eq!( - transport.endpoint(), - "https://cloud.langfuse.com/api/public/otel/v1/traces" - ); - } - - #[tokio::test] - async fn test_connection_accepts_a_healthy_authenticated_host() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path(HEALTH_PATH)) - .respond_with(ResponseTemplate::new(200)) - .mount(&server) - .await; - Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .and(header( - "authorization", - "Basic cGstbGYtdGVzdDpzay1sZi1zZWNyZXQ=", - )) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [], "errors": [] - }))) - .mount(&server) - .await; - - LangfuseClient::new(config(server.uri())) - .test_connection() - .await - .expect("healthy host should pass"); - } - - #[tokio::test] - async fn test_connection_reports_bad_credentials_distinctly() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(path(HEALTH_PATH)) - .respond_with(ResponseTemplate::new(200)) - .mount(&server) - .await; - Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .respond_with(ResponseTemplate::new(401)) - .mount(&server) - .await; - - let error = LangfuseClient::new(config(server.uri())) - .test_connection() - .await - .expect_err("401 should fail"); - - // The operator needs to know it is the keys, not the host. - assert!( - error.to_string().contains("rejected the credentials"), - "unhelpful message: {error}" - ); - } - - #[tokio::test] - async fn test_connection_reports_an_unreachable_host_distinctly() { - let error = LangfuseClient::new(config("http://127.0.0.1:1".into())) - .test_connection() - .await - .expect_err("connection refused should fail"); - - assert!( - error.to_string().contains("cannot reach Langfuse"), - "unhelpful message: {error}" - ); - } - - #[tokio::test] - async fn scores_post_as_score_create_events() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .and(body_json_schema::) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [{ "id": "1", "status": 201 }], "errors": [] - }))) - .expect(1) - .mount(&server) - .await; - - let mut score = ScoreRecord::new( - TraceId("trace-1".into()), - "user-feedback", - ScoreValue::Numeric(1.0), - ); - score.observation_id = Some(ObservationId("obs-1".into())); - score.comment = Some("helpful".into()); - - LangfuseClient::new(config(server.uri())) - .submit_scores(&[score]) - .await - .expect("score submission should succeed"); - } - - #[tokio::test] - async fn partial_rejection_in_a_207_is_surfaced_as_an_error() { - let server = MockServer::start().await; - // A 2xx alone does not mean the data landed: the ingestion endpoint - // reports per-event outcomes in the body. - Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [], - "errors": [{ "id": "1", "status": 400, "message": "invalid value" }] - }))) - .mount(&server) - .await; - - let score = ScoreRecord::new( - TraceId("trace-1".into()), - "user-feedback", - ScoreValue::Numeric(1.0), - ); - let error = LangfuseClient::new(config(server.uri())) - .submit_scores(&[score]) - .await - .expect_err("partial failure should surface"); - - assert!(error.to_string().contains("rejected 1 of 1"), "{error}"); - } - - #[tokio::test] - async fn empty_score_batches_skip_the_request() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .respond_with(ResponseTemplate::new(200)) - .expect(0) - .mount(&server) - .await; - - LangfuseClient::new(config(server.uri())) - .submit_scores(&[]) - .await - .expect("empty batch is a no-op"); - } - - #[test] - fn categorical_and_numeric_scores_declare_their_data_type() { - let numeric = ScoreValue::Numeric(0.5); - let categorical = ScoreValue::Categorical("helpful".into()); - - assert_eq!( - serde_json::to_value(&numeric).expect("serializable"), - serde_json::json!(0.5) - ); - assert_eq!( - serde_json::to_value(&categorical).expect("serializable"), - serde_json::json!("helpful") - ); - } -} diff --git a/crates/observability/src/exporters/langfuse/client.rs b/crates/observability/src/exporters/langfuse/client.rs new file mode 100644 index 000000000..57749b3e6 --- /dev/null +++ b/crates/observability/src/exporters/langfuse/client.rs @@ -0,0 +1,189 @@ +//! The Langfuse REST client and the request helpers its API modules share. +//! +//! Traces reach Langfuse over OTLP; everything OTLP cannot express — scores, +//! managed prompts, datasets, media, aggregated cost — goes through here. + +use reqwest::header::HeaderMap; + +use super::config::{HEALTH_PATH, INGESTION_PATH, LangfuseConfig}; + +/// Client for the Langfuse REST surfaces that OTLP cannot express. +pub struct LangfuseClient { + config: LangfuseConfig, + http: reqwest::Client, +} + +impl LangfuseClient { + /// Build a client over the workspace HTTP client, so the configured + /// upstream proxy applies. + #[must_use] + pub fn new(config: LangfuseConfig) -> Self { + Self { + config, + http: moltis_common::http_client::build_default_http_client(), + } + } + + /// The connection settings this client was built from. + #[must_use] + pub const fn config(&self) -> &LangfuseConfig { + &self.config + } + + /// Verify that the host is reachable and the credentials are accepted. + /// + /// Backs the settings UI's "Test connection" button. The health endpoint is + /// unauthenticated, so credentials are checked with an empty ingestion + /// batch: it is the cheapest authenticated call that creates no data. + pub async fn test_connection(&self) -> anyhow::Result<()> { + let health = self + .http + .get(self.config.url(HEALTH_PATH)) + .timeout(self.config.timeout) + .send() + .await + .map_err(|e| anyhow::anyhow!("cannot reach Langfuse at {}: {e}", self.config.host))?; + + if !health.status().is_success() { + return Err(anyhow::anyhow!( + "Langfuse health check failed with HTTP {}", + health.status() + )); + } + + let auth = self + .post(INGESTION_PATH) + .json(&serde_json::json!({ "batch": [] })) + .send() + .await + .map_err(|e| anyhow::anyhow!("Langfuse credential check failed: {e}"))?; + + let status = auth.status(); + match status { + s if s.is_success() => Ok(()), + reqwest::StatusCode::UNAUTHORIZED | reqwest::StatusCode::FORBIDDEN => Err( + anyhow::anyhow!("Langfuse rejected the credentials (HTTP {status})"), + ), + other => Err(anyhow::anyhow!( + "unexpected response from Langfuse: HTTP {other}" + )), + } + } + + /// Authenticated POST against `path`. + pub(super) fn post(&self, path: &str) -> reqwest::RequestBuilder { + self.http + .post(self.config.url(path)) + .headers(self.header_map()) + .timeout(self.config.timeout) + } + + /// Authenticated header map. + /// + /// Header values are marked sensitive so they stay out of any middleware + /// that logs request headers. + fn header_map(&self) -> HeaderMap { + let mut map = HeaderMap::new(); + for (key, value) in self.config.auth_headers() { + let Ok(name) = key.parse::() else { + continue; + }; + let Ok(mut val) = reqwest::header::HeaderValue::from_str(&value) else { + continue; + }; + val.set_sensitive(true); + map.insert(name, val); + } + map + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + std::time::Duration, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }, + }; + + use {super::*, secrecy::SecretString}; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + #[tokio::test] + async fn test_connection_accepts_a_healthy_authenticated_host() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .and(header( + "authorization", + "Basic cGstbGYtdGVzdDpzay1sZi1zZWNyZXQ=", + )) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], "errors": [] + }))) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect("healthy host should pass"); + } + + #[tokio::test] + async fn test_connection_reports_bad_credentials_distinctly() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path(HEALTH_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .test_connection() + .await + .expect_err("401 should fail"); + + // The operator needs to know it is the keys, not the host. + assert!( + error.to_string().contains("rejected the credentials"), + "unhelpful message: {error}" + ); + } + + #[tokio::test] + async fn test_connection_reports_an_unreachable_host_distinctly() { + let error = LangfuseClient::new(config("http://127.0.0.1:1".into())) + .test_connection() + .await + .expect_err("connection refused should fail"); + + assert!( + error.to_string().contains("cannot reach Langfuse"), + "unhelpful message: {error}" + ); + } +} diff --git a/crates/observability/src/exporters/langfuse/config.rs b/crates/observability/src/exporters/langfuse/config.rs new file mode 100644 index 000000000..69e6c7ffa --- /dev/null +++ b/crates/observability/src/exporters/langfuse/config.rs @@ -0,0 +1,178 @@ +//! Langfuse connection settings, endpoint paths and credential handling. + +use std::{collections::BTreeMap, time::Duration}; + +use { + base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}, + secrecy::{ExposeSecret, SecretString}, +}; + +use crate::{ + exporters::otlp::{OtlpConfig, OtlpTransport}, + profile::ExportProfile, +}; + +/// Path Langfuse serves OTLP traces on, appended to the configured host. +pub const OTEL_TRACES_PATH: &str = "/api/public/otel/v1/traces"; +/// Batch ingestion path, used for scores and dataset run items. +pub const INGESTION_PATH: &str = "/api/public/ingestion"; +/// Unauthenticated liveness probe. +pub const HEALTH_PATH: &str = "/api/public/health"; +/// Managed-prompt retrieval, `{name}` appended. +pub const PROMPTS_PATH: &str = "/api/public/v2/prompts"; +/// Dataset collection. +pub const DATASETS_PATH: &str = "/api/public/v2/datasets"; +/// Dataset item collection. +pub const DATASET_ITEMS_PATH: &str = "/api/public/dataset-items"; +/// Dataset run item collection, linking a trace to a dataset item. +pub const DATASET_RUN_ITEMS_PATH: &str = "/api/public/dataset-run-items"; +/// Media upload negotiation. +pub const MEDIA_PATH: &str = "/api/public/media"; +/// Aggregated daily usage and cost. +pub const DAILY_METRICS_PATH: &str = "/api/public/metrics/daily"; + +/// Langfuse connection settings. +#[derive(Clone)] +pub struct LangfuseConfig { + /// Base host, e.g. `https://cloud.langfuse.com` or a self-hosted URL. + pub host: String, + /// Project public key (Basic auth username). + pub public_key: String, + /// Project secret key (Basic auth password). + pub secret_key: SecretString, + /// Deployment environment. + pub environment: Option, + /// Build release identifier. + pub release: Option, + /// Per-request timeout. + pub timeout: Duration, +} + +// Manual impl: the derived one would print the secret key. +impl std::fmt::Debug for LangfuseConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LangfuseConfig") + .field("host", &self.host) + .field("public_key", &self.public_key) + .field("secret_key", &"[REDACTED]") + .field("environment", &self.environment) + .field("release", &self.release) + .field("timeout", &self.timeout) + .finish() + } +} + +impl LangfuseConfig { + /// Join `path` onto the configured host, tolerating a trailing slash. + #[must_use] + pub fn url(&self, path: &str) -> String { + format!("{}{path}", self.host.trim_end_matches('/')) + } + + /// `Authorization` header value for Basic auth. + /// + /// Langfuse authenticates with the public key as username and the secret + /// key as password. + #[must_use] + pub fn basic_auth_header(&self) -> String { + let raw = format!("{}:{}", self.public_key, self.secret_key.expose_secret()); + format!("Basic {}", BASE64.encode(raw)) + } + + /// Headers every authenticated Langfuse request carries. + #[must_use] + pub fn auth_headers(&self) -> BTreeMap { + let mut headers = BTreeMap::new(); + headers.insert("Authorization".to_string(), self.basic_auth_header()); + // Langfuse surfaces these in its own diagnostics. + headers.insert("X-Langfuse-Sdk-Name".to_string(), "moltis".to_string()); + headers.insert( + "X-Langfuse-Sdk-Version".to_string(), + env!("CARGO_PKG_VERSION").to_string(), + ); + headers.insert("X-Langfuse-Public-Key".to_string(), self.public_key.clone()); + headers + } + + /// Build the OTLP transport that carries traces to Langfuse. + /// + /// The profile is fixed to [`ExportProfile::langfuse`]: content capture is + /// the entire point of sending to Langfuse, and letting it be configured + /// down to `MetadataOnly` would produce traces with no conversation in + /// them, which is worse than not exporting at all. + #[must_use] + pub fn build_transport(&self, service_version: String) -> OtlpTransport { + OtlpTransport::new(OtlpConfig { + name: "langfuse".to_string(), + endpoint: self.url(OTEL_TRACES_PATH), + headers: self.auth_headers(), + timeout: self.timeout, + service_name: "moltis".to_string(), + service_version, + environment: self.environment.clone(), + profile: ExportProfile::langfuse(), + }) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + #[test] + fn basic_auth_encodes_public_and_secret_key() { + let header = config("https://example.com".into()).basic_auth_header(); + let encoded = header.strip_prefix("Basic ").expect("basic scheme"); + let decoded = + String::from_utf8(BASE64.decode(encoded).expect("valid base64")).expect("valid utf-8"); + + assert_eq!(decoded, "pk-lf-test:sk-lf-secret"); + } + + #[test] + fn debug_never_prints_the_secret_key() { + let rendered = format!("{:?}", config("https://example.com".into())); + + assert!( + !rendered.contains("sk-lf-secret"), + "secret leaked: {rendered}" + ); + assert!(rendered.contains("[REDACTED]")); + // The public key is not a secret and is useful in diagnostics. + assert!(rendered.contains("pk-lf-test")); + } + + #[test] + fn url_join_tolerates_a_trailing_slash_on_the_host() { + let with = config("https://example.com/".into()); + let without = config("https://example.com".into()); + + assert_eq!(with.url(OTEL_TRACES_PATH), without.url(OTEL_TRACES_PATH)); + assert_eq!( + without.url(OTEL_TRACES_PATH), + "https://example.com/api/public/otel/v1/traces" + ); + } + + #[test] + fn transport_targets_the_otel_endpoint_with_the_langfuse_profile() { + let transport = config("https://cloud.langfuse.com".into()).build_transport("1.0".into()); + + assert_eq!( + transport.endpoint(), + "https://cloud.langfuse.com/api/public/otel/v1/traces" + ); + } +} diff --git a/crates/observability/src/exporters/langfuse/mod.rs b/crates/observability/src/exporters/langfuse/mod.rs new file mode 100644 index 000000000..17448b3bd --- /dev/null +++ b/crates/observability/src/exporters/langfuse/mod.rs @@ -0,0 +1,24 @@ +//! Langfuse backend. +//! +//! Traces reach Langfuse over OTLP — that is Langfuse's own modern ingest path, +//! the one its v3+ Python and v4+ JS SDKs use, and the only one that carries the +//! full observation taxonomy (`AGENT`, `TOOL`, `RETRIEVER`, ...). The native +//! `/api/public/ingestion` event `observation-create` is marked deprecated +//! upstream and its non-deprecated `span-create`/`generation-create` pair +//! cannot express the newer types at all. +//! +//! Everything OTLP *cannot* express lives in the REST modules here: scores, +//! managed prompts, datasets and experiment runs, media, and aggregated cost. + +pub mod client; +pub mod config; +pub mod scores; + +pub use { + client::LangfuseClient, + config::{ + DAILY_METRICS_PATH, DATASET_ITEMS_PATH, DATASET_RUN_ITEMS_PATH, DATASETS_PATH, HEALTH_PATH, + INGESTION_PATH, LangfuseConfig, MEDIA_PATH, OTEL_TRACES_PATH, PROMPTS_PATH, + }, + scores::{ScoreSink, ScoreTransport}, +}; diff --git a/crates/observability/src/exporters/langfuse/scores.rs b/crates/observability/src/exporters/langfuse/scores.rs new file mode 100644 index 000000000..84f763a50 --- /dev/null +++ b/crates/observability/src/exporters/langfuse/scores.rs @@ -0,0 +1,386 @@ +//! Score ingestion. +//! +//! Scores are the one part of the model OTLP cannot carry, so they take a +//! separate path: a dedicated sink that filters the fanout down to +//! [`Event::Score`] and posts batches to the ingestion API. +//! +//! Without this sink `TurnRecorder::score` would emit into a void — the OTLP +//! mapping drops score events by design, so the Langfuse trace transport alone +//! silently discards every score. + +use std::{sync::Arc, time::Duration}; + +use { + async_trait::async_trait, + serde::Serialize, + tracing::{debug, warn}, +}; + +use { + super::{client::LangfuseClient, config::INGESTION_PATH}, + crate::{ + model::{Event, ScoreRecord, ScoreValue}, + runtime::{BatchConfig, BatchSink, Transport, TransportError}, + sink::ObservationSink, + }, +}; + +/// A `score-create` event in the ingestion batch envelope. +#[derive(Debug, Serialize)] +struct IngestionEvent<'a> { + id: String, + timestamp: String, + #[serde(rename = "type")] + kind: &'static str, + body: ScoreBody<'a>, +} + +/// Langfuse `ScoreBody`. +#[derive(Debug, Serialize)] +struct ScoreBody<'a> { + id: &'a str, + #[serde(rename = "traceId")] + trace_id: &'a str, + #[serde(rename = "observationId", skip_serializing_if = "Option::is_none")] + observation_id: Option<&'a str>, + name: &'a str, + value: &'a ScoreValue, + #[serde(rename = "dataType")] + data_type: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + comment: Option<&'a str>, + #[serde(skip_serializing_if = "Option::is_none")] + environment: Option<&'a str>, +} + +/// The ingestion request envelope. +#[derive(Debug, Serialize)] +struct IngestionRequest<'a> { + batch: Vec>, +} + +impl LangfuseClient { + /// Submit scores. Langfuse upserts on score id, so re-submitting a score + /// with the same id overwrites rather than duplicating — which is what + /// makes a user changing their reaction idempotent. + pub async fn submit_scores(&self, scores: &[ScoreRecord]) -> anyhow::Result<()> { + if scores.is_empty() { + return Ok(()); + } + + let now = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_default(); + + let batch: Vec> = scores + .iter() + .map(|score| IngestionEvent { + id: uuid::Uuid::new_v4().to_string(), + timestamp: now.clone(), + kind: "score-create", + body: ScoreBody { + id: &score.id, + trace_id: &score.trace_id.0, + observation_id: score.observation_id.as_ref().map(|o| o.0.as_str()), + name: &score.name, + value: &score.value, + data_type: match score.value { + ScoreValue::Numeric(_) => "NUMERIC", + ScoreValue::Categorical(_) => "CATEGORICAL", + }, + comment: score.comment.as_deref(), + environment: score + .environment + .as_deref() + .or(self.config().environment.as_deref()), + }, + }) + .collect(); + + let response = self + .post(INGESTION_PATH) + .json(&IngestionRequest { batch }) + .send() + .await?; + + let status = response.status(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(anyhow::anyhow!( + "Langfuse rejected scores: HTTP {status}: {}", + body.chars().take(512).collect::() + )); + } + + // The ingestion endpoint reports per-event outcomes in a 207, so a 2xx + // alone does not mean every score landed. + let body: serde_json::Value = response.json().await.unwrap_or_default(); + if let Some(errors) = body.get("errors").and_then(|e| e.as_array()) + && !errors.is_empty() + { + return Err(anyhow::anyhow!( + "Langfuse rejected {} of {} scores: {}", + errors.len(), + scores.len(), + serde_json::to_string(errors).unwrap_or_default() + )); + } + + debug!(count = scores.len(), "submitted scores to Langfuse"); + Ok(()) + } +} + +/// Batch transport that posts scores to the ingestion API. +pub struct ScoreTransport { + client: Arc, +} + +impl ScoreTransport { + /// Build a transport over an existing client. + #[must_use] + pub const fn new(client: Arc) -> Self { + Self { client } + } +} + +#[async_trait] +impl Transport for ScoreTransport { + fn name(&self) -> &str { + "langfuse-scores" + } + + async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { + let scores: Vec = batch + .iter() + .filter_map(|event| match event { + Event::Score(score) => Some((**score).clone()), + _ => None, + }) + .collect(); + + if scores.is_empty() { + return Ok(()); + } + + self.client.submit_scores(&scores).await.map_err(|error| { + // A rejected score is worth retrying: the common causes are + // transient (the trace has not landed yet, a 5xx) rather than the + // payload being permanently invalid. + TransportError::Retryable(error.to_string()) + }) + } +} + +/// Sink that carries only scores. +/// +/// The fanout hands every event to every sink, so this wrapper drops +/// non-score events before they reach the queue — otherwise a busy agent +/// would fill the score queue with trace events and evict real scores. +pub struct ScoreSink { + inner: BatchSink, +} + +impl ScoreSink { + /// Spawn a score sink over `client`. + #[must_use] + pub fn spawn(client: Arc, config: BatchConfig) -> Self { + Self { + inner: BatchSink::spawn(Arc::new(ScoreTransport::new(client)), config), + } + } +} + +#[async_trait] +impl ObservationSink for ScoreSink { + fn name(&self) -> &str { + "langfuse-scores" + } + + fn record(&self, event: Event) { + if matches!(event, Event::Score(_)) { + self.inner.record(event); + } + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + if let Err(error) = self.inner.flush(timeout).await { + warn!(%error, "score sink flush failed"); + return Err(error); + } + Ok(()) + } +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use { + std::time::Duration, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }, + }; + + use { + super::*, + crate::{ + exporters::langfuse::config::LangfuseConfig, + model::{ObservationId, TraceId, TraceRecord}, + }, + secrecy::SecretString, + }; + + fn config(host: String) -> LangfuseConfig { + LangfuseConfig { + host, + public_key: "pk-lf-test".into(), + secret_key: SecretString::new("sk-lf-secret".to_string()), + environment: Some("production".into()), + release: Some("20260726.01".into()), + timeout: Duration::from_secs(5), + } + } + + fn score(name: &str) -> ScoreRecord { + ScoreRecord::new(TraceId("trace-1".into()), name, ScoreValue::Numeric(1.0)) + } + + #[tokio::test] + async fn scores_post_as_score_create_events() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [{ "id": "1", "status": 201 }], "errors": [] + }))) + .expect(1) + .mount(&server) + .await; + + let mut score = score("user-feedback"); + score.observation_id = Some(ObservationId("obs-1".into())); + score.comment = Some("helpful".into()); + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) + .await + .expect("score submission should succeed"); + } + + #[tokio::test] + async fn partial_rejection_in_a_207_is_surfaced_as_an_error() { + let server = MockServer::start().await; + // A 2xx alone does not mean the data landed: the ingestion endpoint + // reports per-event outcomes in the body. + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], + "errors": [{ "id": "1", "status": 400, "message": "invalid value" }] + }))) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .submit_scores(&[score("user-feedback")]) + .await + .expect_err("partial failure should surface"); + + assert!(error.to_string().contains("rejected 1 of 1"), "{error}"); + } + + #[tokio::test] + async fn empty_score_batches_skip_the_request() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .submit_scores(&[]) + .await + .expect("empty batch is a no-op"); + } + + #[test] + fn categorical_and_numeric_scores_declare_their_data_type() { + let numeric = ScoreValue::Numeric(0.5); + let categorical = ScoreValue::Categorical("helpful".into()); + + assert_eq!( + serde_json::to_value(&numeric).expect("serializable"), + serde_json::json!(0.5) + ); + assert_eq!( + serde_json::to_value(&categorical).expect("serializable"), + serde_json::json!("helpful") + ); + } + + #[tokio::test] + async fn the_transport_ignores_non_score_events() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(200)) + .expect(0) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Trace(Box::new(TraceRecord::new("agent-run")))]; + + transport.send(&batch).await.expect("no scores, no request"); + } + + #[tokio::test] + async fn the_sink_drops_trace_events_before_they_reach_the_queue() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ + "successes": [], "errors": [] + }))) + .expect(1) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let sink = ScoreSink::spawn(client, BatchConfig::default()); + + // A busy agent emits far more trace events than scores; if they were + // queued here they would evict the scores we actually care about. + for _ in 0..64 { + sink.record(Event::Trace(Box::new(TraceRecord::new("agent-run")))); + } + sink.record(Event::Score(Box::new(score("user-feedback")))); + + sink.flush(Duration::from_secs(5)) + .await + .expect("flush should succeed"); + } + + #[tokio::test] + async fn a_rejected_batch_is_retryable_rather_than_dropped() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(INGESTION_PATH)) + .respond_with(ResponseTemplate::new(503)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Score(Box::new(score("user-feedback")))]; + + let error = transport.send(&batch).await.expect_err("503 should fail"); + assert!( + matches!(error, TransportError::Retryable(_)), + "a transient rejection must not discard user feedback, got {error:?}" + ); + } +} diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index ffced3a1f..e1d148e86 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -83,7 +83,22 @@ impl TurnRecorder { scope: TraceScope, settings: RecorderSettings, ) -> Option { - let sink = sink::global_sink()?; + Self::begin_with_sink(sink::global_sink()?, name, scope, settings) + } + + /// Begin recording a turn against an explicit sink. + /// + /// The global sink is process-wide, which makes it unusable for anything + /// that needs its own destination: parallel tests, and the experiment + /// runner, which routes a dataset run's traces separately from live + /// traffic. Both would otherwise race with whatever was installed last. + #[must_use] + pub fn begin_with_sink( + sink: Arc, + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + ) -> Option { if !settings.sampled() { return None; } diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs index 072ace252..ee5cb8049 100644 --- a/crates/observability/tests/end_to_end.rs +++ b/crates/observability/tests/end_to_end.rs @@ -13,7 +13,6 @@ use { BatchConfig, BatchSink, ExportProfile, ObservationKind, ObservationSink, RecorderSettings, TokenUsage, TraceScope, TurnRecorder, exporters::otlp::{OtlpConfig, OtlpTransport}, - sink, }, serde_json::Value, wiremock::{ @@ -76,11 +75,18 @@ async fn run_turn_against(server: &MockServer, profile: ExportProfile) -> Vec Vec Date: Sun, 26 Jul 2026 14:08:20 +0000 Subject: [PATCH 09/33] feat(observability): classify chat reactions as user-feedback scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the channel-agnostic half of reaction feedback: a vocabulary that turns a raw reaction token into a signal, and a score builder that makes the result idempotent. Normalization matters more than it looks. The three channels send three shapes for the same gesture — Telegram and Discord send the emoji, Slack sends a shortcode, and either may carry a skin-tone modifier or a presentation selector. Without folding those away, a thumbs up with a skin tone reads as no signal at all, so a user's feedback would be silently dropped based on how their client renders an emoji. Score ids are derived as a UUIDv5 of trace, score name and user rather than random. Langfuse upserts on score id, so this makes the natural reaction patterns behave correctly: switching from a thumb up to a thumb down overwrites that user's vote instead of recording two contradictory ones, a duplicate webhook delivery is a no-op, and removing a reaction can find the score to retract. Two users on the same trace still score independently. The default vocabulary is deliberately narrow. A default that swallowed every positive-looking emoji would turn release parties and acknowledgement reactions into quality data; teams that want that can set the lists explicitly, per-side, without restating the side they did not change. Also adds `delete_score` for retraction, treating a 404 as success since an already-absent score is the state the caller asked for. --- Cargo.lock | 7 + Cargo.toml | 2 +- crates/config/src/lib.rs | 20 +- crates/config/src/schema/instrumentation.rs | 48 +++ crates/config/src/template.rs | 9 + crates/config/src/validate/schema_map.rs | 9 + .../src/exporters/langfuse/client.rs | 8 + .../src/exporters/langfuse/config.rs | 2 + .../src/exporters/langfuse/mod.rs | 2 +- .../src/exporters/langfuse/scores.rs | 77 +++- crates/observability/src/feedback.rs | 374 ++++++++++++++++++ crates/observability/src/lib.rs | 4 + 12 files changed, 549 insertions(+), 13 deletions(-) create mode 100644 crates/observability/src/feedback.rs diff --git a/Cargo.lock b/Cargo.lock index 4d4acf55e..0797106e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11746,6 +11746,12 @@ dependencies = [ "sha1 0.10.6", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -13857,6 +13863,7 @@ dependencies = [ "getrandom 0.4.2", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index de289c1fa..8689d22bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,7 +214,7 @@ sqlx = { features = ["migrate", "runtime-tokio", "sqlite"], version = "0.8" } # HTTP client reqwest = { features = ["form", "json", "multipart", "query", "socks", "stream"], version = "0.13" } # UUID -uuid = { features = ["v4"], version = "1" } +uuid = { features = ["v4", "v5"], version = "1" } # Async async-stream = "0.3" async-trait = "0.1" diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 30dc0d9ca..d3c60c29d 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -56,16 +56,16 @@ pub use { AgentIdentity, AgentMemoryWriteMode, AgentPreset, AgentRuntimeLimitSource, AgentRuntimeLimits, AgentsConfig, AuthConfig, CacheRetention, CalDavAccountConfig, CalDavConfig, ChannelToolPolicyOverride, ChannelsConfig, ChatConfig, CodeIndexTomlConfig, - CompactionConfig, CompactionMode, ContentCaptureMode, DatadogSettings, GeoLocation, - GroupToolPolicy, HeartbeatConfig, HomeAssistantAccountConfig, HomeAssistantConfig, - InstrumentationConfig, LangfuseSettings, MemoryBackend, MemoryCitationsMode, - MemoryProvider, MemoryScope, MemorySearchMergeStrategy, MemoryStyle, MessageQueueMode, - ModePreset, ModesConfig, MoltisConfig, NgrokConfig, OtlpSettings, PresetMemoryConfig, - PresetToolPolicy, PromptMemoryMode, ResolvedIdentity, SessionAccessPolicyConfig, - SessionExportMode, Timezone, ToolMode, ToolPolicyConfig, ToolRegistryMode, UserProfile, - UserProfileWriteMode, VoiceConfig, VoiceElevenLabsConfig, VoiceOpenAiConfig, - VoiceSttConfig, VoiceSttProvider, VoiceTtsConfig, VoiceTtsProvider, VoiceWhisperConfig, - VoiceWhisperLocalConfig, WireApi, parse_byte_size, + CompactionConfig, CompactionMode, ContentCaptureMode, DatadogSettings, FeedbackSettings, + GeoLocation, GroupToolPolicy, HeartbeatConfig, HomeAssistantAccountConfig, + HomeAssistantConfig, InstrumentationConfig, LangfuseSettings, MemoryBackend, + MemoryCitationsMode, MemoryProvider, MemoryScope, MemorySearchMergeStrategy, MemoryStyle, + MessageQueueMode, ModePreset, ModesConfig, MoltisConfig, NgrokConfig, OtlpSettings, + PresetMemoryConfig, PresetToolPolicy, PromptMemoryMode, ResolvedIdentity, + SessionAccessPolicyConfig, SessionExportMode, Timezone, ToolMode, ToolPolicyConfig, + ToolRegistryMode, UserProfile, UserProfileWriteMode, VoiceConfig, VoiceElevenLabsConfig, + VoiceOpenAiConfig, VoiceSttConfig, VoiceSttProvider, VoiceTtsConfig, VoiceTtsProvider, + VoiceWhisperConfig, VoiceWhisperLocalConfig, WireApi, parse_byte_size, }, validate::{Diagnostic, Severity, ValidationResult}, }; diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs index ec585cda7..67d2afc17 100644 --- a/crates/config/src/schema/instrumentation.rs +++ b/crates/config/src/schema/instrumentation.rs @@ -63,6 +63,50 @@ impl ContentCaptureMode { pub const ALL: &'static [&'static str] = &["full", "metadata_only", "none"]; } +fn default_link_retention_days() -> u32 { + 30 +} + +/// End-user feedback collected from chat reactions. +/// +/// A thumb on a reply is the cheapest quality signal available, but attributing +/// one to the turn that produced it needs the reply's trace to still be on +/// record when the reaction arrives — hence the retention window. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct FeedbackSettings { + /// Collect reaction feedback. On by default once instrumentation is + /// enabled: a scoring pipeline with no scores in it is not useful. + #[serde(default = "default_true")] + pub enabled: bool, + /// Reaction tokens counted as approval. Empty uses the built-in list. + /// + /// Accepts raw emoji or shortcodes; skin tone and presentation selectors + /// are ignored when matching. + #[serde(default)] + pub positive: Vec, + /// Reaction tokens counted as disapproval. Empty uses the built-in list. + #[serde(default)] + pub negative: Vec, + /// How long a reply stays attributable to its trace, in days. + /// + /// Reactions often arrive long after the reply. Kept bounded because the + /// link table would otherwise grow without limit. + #[serde(default = "default_link_retention_days")] + pub link_retention_days: u32, +} + +impl Default for FeedbackSettings { + fn default() -> Self { + Self { + enabled: true, + positive: Vec::new(), + negative: Vec::new(), + link_retention_days: default_link_retention_days(), + } + } +} + /// Top-level instrumentation configuration. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default)] @@ -104,6 +148,9 @@ pub struct InstrumentationConfig { /// Datadog backend, via Datadog's OTLP intake. #[serde(default)] pub datadog: DatadogSettings, + /// End-user reaction feedback. + #[serde(default)] + pub feedback: FeedbackSettings, } fn default_environment() -> String { @@ -124,6 +171,7 @@ impl Default for InstrumentationConfig { langfuse: LangfuseSettings::default(), otlp: OtlpSettings::default(), datadog: DatadogSettings::default(), + feedback: FeedbackSettings::default(), } } } diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index f3a50779c..cf7c74a62 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -612,6 +612,15 @@ port = {port} # Port number (auto-generated for this i # service = "moltis" # content = "metadata_only" +# Reaction feedback. A thumbs up/down on a reply in Telegram, Discord or Slack +# becomes a "user-feedback" score on the trace that produced it. Lists accept +# raw emoji or shortcodes; empty means the built-in vocabulary. +# [instrumentation.feedback] +# enabled = true +# positive = ["\U0001F44D", "+1", "thumbsup"] +# negative = ["\U0001F44E", "-1", "thumbsdown"] +# link_retention_days = 30 # How long a reply stays attributable + # ══════════════════════════════════════════════════════════════════════════════ # CRON # ══════════════════════════════════════════════════════════════════════════════ diff --git a/crates/config/src/validate/schema_map.rs b/crates/config/src/validate/schema_map.rs index 828c42cc3..fb94b4837 100644 --- a/crates/config/src/validate/schema_map.rs +++ b/crates/config/src/validate/schema_map.rs @@ -555,6 +555,15 @@ pub(super) fn build_schema_map() -> KnownKeys { ("timeout_secs", Leaf), ])), ), + ( + "feedback", + Struct(HashMap::from([ + ("enabled", Leaf), + ("positive", Leaf), + ("negative", Leaf), + ("link_retention_days", Leaf), + ])), + ), ])), ), ( diff --git a/crates/observability/src/exporters/langfuse/client.rs b/crates/observability/src/exporters/langfuse/client.rs index 57749b3e6..a026e478c 100644 --- a/crates/observability/src/exporters/langfuse/client.rs +++ b/crates/observability/src/exporters/langfuse/client.rs @@ -78,6 +78,14 @@ impl LangfuseClient { .timeout(self.config.timeout) } + /// Authenticated DELETE against `path`. + pub(super) fn delete(&self, path: &str) -> reqwest::RequestBuilder { + self.http + .delete(self.config.url(path)) + .headers(self.header_map()) + .timeout(self.config.timeout) + } + /// Authenticated header map. /// /// Header values are marked sensitive so they stay out of any middleware diff --git a/crates/observability/src/exporters/langfuse/config.rs b/crates/observability/src/exporters/langfuse/config.rs index 69e6c7ffa..1c606e6bc 100644 --- a/crates/observability/src/exporters/langfuse/config.rs +++ b/crates/observability/src/exporters/langfuse/config.rs @@ -18,6 +18,8 @@ pub const OTEL_TRACES_PATH: &str = "/api/public/otel/v1/traces"; pub const INGESTION_PATH: &str = "/api/public/ingestion"; /// Unauthenticated liveness probe. pub const HEALTH_PATH: &str = "/api/public/health"; +/// Individual score resource, `{id}` appended, used for retraction. +pub const SCORES_PATH: &str = "/api/public/scores"; /// Managed-prompt retrieval, `{name}` appended. pub const PROMPTS_PATH: &str = "/api/public/v2/prompts"; /// Dataset collection. diff --git a/crates/observability/src/exporters/langfuse/mod.rs b/crates/observability/src/exporters/langfuse/mod.rs index 17448b3bd..c743b959e 100644 --- a/crates/observability/src/exporters/langfuse/mod.rs +++ b/crates/observability/src/exporters/langfuse/mod.rs @@ -18,7 +18,7 @@ pub use { client::LangfuseClient, config::{ DAILY_METRICS_PATH, DATASET_ITEMS_PATH, DATASET_RUN_ITEMS_PATH, DATASETS_PATH, HEALTH_PATH, - INGESTION_PATH, LangfuseConfig, MEDIA_PATH, OTEL_TRACES_PATH, PROMPTS_PATH, + INGESTION_PATH, LangfuseConfig, MEDIA_PATH, OTEL_TRACES_PATH, PROMPTS_PATH, SCORES_PATH, }, scores::{ScoreSink, ScoreTransport}, }; diff --git a/crates/observability/src/exporters/langfuse/scores.rs b/crates/observability/src/exporters/langfuse/scores.rs index 84f763a50..f62649caa 100644 --- a/crates/observability/src/exporters/langfuse/scores.rs +++ b/crates/observability/src/exporters/langfuse/scores.rs @@ -17,7 +17,10 @@ use { }; use { - super::{client::LangfuseClient, config::INGESTION_PATH}, + super::{ + client::LangfuseClient, + config::{INGESTION_PATH, SCORES_PATH}, + }, crate::{ model::{Event, ScoreRecord, ScoreValue}, runtime::{BatchConfig, BatchSink, Transport, TransportError}, @@ -129,6 +132,30 @@ impl LangfuseClient { debug!(count = scores.len(), "submitted scores to Langfuse"); Ok(()) } + + /// Delete a score by id. + /// + /// Backs reaction removal: a user who takes their thumb back has retracted + /// the opinion, and leaving the score in place would keep counting a vote + /// they withdrew. A 404 is success — the score is already gone, which is + /// the state the caller wanted. + pub async fn delete_score(&self, score_id: &str) -> anyhow::Result<()> { + let response = self + .delete(&format!("{SCORES_PATH}/{score_id}")) + .send() + .await?; + + let status = response.status(); + if status.is_success() || status == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + + let body = response.text().await.unwrap_or_default(); + Err(anyhow::anyhow!( + "Langfuse rejected the score deletion: HTTP {status}: {}", + body.chars().take(512).collect::() + )) + } } /// Batch transport that posts scores to the ingestion API. @@ -364,6 +391,54 @@ mod tests { .expect("flush should succeed"); } + #[tokio::test] + async fn retracting_a_score_deletes_it_by_id() { + let server = MockServer::start().await; + Mock::given(method("DELETE")) + .and(path(format!("{SCORES_PATH}/score-1"))) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .delete_score("score-1") + .await + .expect("deletion should succeed"); + } + + #[tokio::test] + async fn deleting_an_already_absent_score_succeeds() { + let server = MockServer::start().await; + // Reaction-removal events can arrive twice, or after the score was + // cleaned up. Already-gone is the state the caller asked for. + Mock::given(method("DELETE")) + .respond_with(ResponseTemplate::new(404)) + .mount(&server) + .await; + + LangfuseClient::new(config(server.uri())) + .delete_score("missing") + .await + .expect("404 is success for a deletion"); + } + + #[tokio::test] + async fn a_failed_deletion_is_surfaced() { + let server = MockServer::start().await; + Mock::given(method("DELETE")) + .respond_with(ResponseTemplate::new(500)) + .mount(&server) + .await; + + let error = LangfuseClient::new(config(server.uri())) + .delete_score("score-1") + .await + .expect_err("500 should fail"); + + assert!(error.to_string().contains("500"), "{error}"); + } + #[tokio::test] async fn a_rejected_batch_is_retryable_rather_than_dropped() { let server = MockServer::start().await; diff --git a/crates/observability/src/feedback.rs b/crates/observability/src/feedback.rs new file mode 100644 index 000000000..09e6157a9 --- /dev/null +++ b/crates/observability/src/feedback.rs @@ -0,0 +1,374 @@ +//! End-user feedback and its mapping onto scores. +//! +//! A reaction on a chat message is the cheapest quality signal there is: no +//! form, no rating widget, just a thumb. This module turns one into a +//! [`ScoreRecord`] without knowing anything about the channel it came from — +//! Telegram sends a raw emoji, Slack sends a shortcode, Discord sends either, +//! and all three normalize to the same signal here. + +use std::collections::BTreeSet; + +use crate::model::{ScoreRecord, ScoreValue, TraceId}; + +/// Score name used for end-user reaction feedback. +/// +/// Fixed rather than configurable: it is the join key for every feedback +/// dashboard, and letting it drift per deployment makes those dashboards +/// silently stop matching. +pub const USER_FEEDBACK_SCORE: &str = "user-feedback"; + +/// Namespace for deriving deterministic feedback score ids. +const FEEDBACK_NAMESPACE: uuid::Uuid = uuid::uuid!("6f9f4d3a-1b2c-4e5f-8a90-0b1c2d3e4f50"); + +/// A feedback signal expressed by an end user. +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FeedbackSignal { + /// The answer was good. + Positive, + /// The answer was bad. + Negative, +} + +impl FeedbackSignal { + /// Numeric value carried to the backend. + /// + /// Langfuse treats a 0/1 numeric score as a boolean for charting, which is + /// what a thumb is. + #[must_use] + pub const fn score_value(self) -> f64 { + match self { + Self::Positive => 1.0, + Self::Negative => 0.0, + } + } +} + +/// Reaction tokens that count as feedback. +/// +/// Configurable because reaction conventions are team-specific: some treat 🎉 +/// as praise, others use it for releases only. The defaults stay deliberately +/// narrow — a broad default vocabulary turns every celebratory emoji into a +/// quality score and quietly poisons the metric. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FeedbackVocabulary { + positive: BTreeSet, + negative: BTreeSet, +} + +impl Default for FeedbackVocabulary { + fn default() -> Self { + Self::new(DEFAULT_POSITIVE, DEFAULT_NEGATIVE) + } +} + +/// Unambiguous approval, in both raw-emoji and shortcode form. +const DEFAULT_POSITIVE: &[&str] = &[ + "\u{1f44d}", // thumbs up + "+1", + "thumbsup", + "\u{2764}", // red heart + "heart", + "\u{1f4af}", // hundred points + "100", + "\u{2705}", // check mark button + "white_check_mark", +]; + +/// Unambiguous disapproval. +const DEFAULT_NEGATIVE: &[&str] = &[ + "\u{1f44e}", // thumbs down + "-1", + "thumbsdown", + "\u{274c}", // cross mark + "x", + "\u{1f621}", // enraged face + "rage", + "\u{1f4a9}", // pile of poo + "poop", +]; + +impl FeedbackVocabulary { + /// Build a vocabulary from explicit token lists. + #[must_use] + pub fn new>(positive: &[S], negative: &[S]) -> Self { + Self { + positive: positive.iter().map(|s| normalize(s.as_ref())).collect(), + negative: negative.iter().map(|s| normalize(s.as_ref())).collect(), + } + } + + /// Build from configured lists, falling back to the defaults when a list is + /// empty. + /// + /// Per-list rather than all-or-nothing so an operator can extend the + /// positive side without having to restate the negative one. + #[must_use] + pub fn from_config(positive: &[String], negative: &[String]) -> Self { + let default = Self::default(); + Self { + positive: if positive.is_empty() { + default.positive + } else { + positive.iter().map(|s| normalize(s)).collect() + }, + negative: if negative.is_empty() { + default.negative + } else { + negative.iter().map(|s| normalize(s)).collect() + }, + } + } + + /// Classify a raw reaction token, or `None` when it carries no signal. + #[must_use] + pub fn classify(&self, raw: &str) -> Option { + let token = normalize(raw); + if token.is_empty() { + return None; + } + if self.positive.contains(&token) { + return Some(FeedbackSignal::Positive); + } + if self.negative.contains(&token) { + return Some(FeedbackSignal::Negative); + } + None + } +} + +/// Reduce a reaction token to a comparable form. +/// +/// Handles the three shapes the channels actually send: a bare emoji +/// (Telegram, Discord), a shortcode (Slack), and a shortcode wrapped in colons +/// or carrying a skin-tone suffix. Skin tone and presentation selectors are +/// stripped so `👍🏾` and `👍` are the same signal — a preference about how an +/// emoji renders is not a different opinion about the answer. +fn normalize(raw: &str) -> String { + let trimmed = raw.trim().trim_matches(':'); + // Slack encodes skin tone as a `::skin-tone-4` suffix on the shortcode. + let base = trimmed.split("::").next().unwrap_or(trimmed); + + base.chars() + .filter(|c| !is_modifier(*c)) + .flat_map(char::to_lowercase) + .collect() +} + +/// Whether `c` only affects rendering rather than meaning. +const fn is_modifier(c: char) -> bool { + matches!( + c, + // Variation selectors: text vs emoji presentation. + '\u{fe0e}' | '\u{fe0f}' + // Fitzpatrick skin tone modifiers. + | '\u{1f3fb}'..='\u{1f3ff}' + ) +} + +/// Build a score for a feedback signal against a trace. +/// +/// The score id is derived from the trace, the score name and the reacting +/// user, so it is stable across submissions. Langfuse upserts on score id, +/// which makes this idempotent in exactly the way reactions need: a user who +/// switches from 👍 to 👎 overwrites their own score instead of adding a +/// second one, and a duplicate webhook delivery is a no-op rather than +/// double-counting. +#[must_use] +pub fn feedback_score( + trace_id: &TraceId, + signal: FeedbackSignal, + user_id: Option<&str>, + comment: Option, + environment: Option, +) -> ScoreRecord { + ScoreRecord { + id: feedback_score_id(trace_id, user_id), + trace_id: trace_id.clone(), + observation_id: None, + name: USER_FEEDBACK_SCORE.to_string(), + value: ScoreValue::Numeric(signal.score_value()), + comment, + environment, + } +} + +/// Deterministic score id for one user's feedback on one trace. +#[must_use] +pub fn feedback_score_id(trace_id: &TraceId, user_id: Option<&str>) -> String { + let key = format!( + "{}|{USER_FEEDBACK_SCORE}|{}", + trace_id.0, + user_id.unwrap_or("anonymous") + ); + uuid::Uuid::new_v5(&FEEDBACK_NAMESPACE, key.as_bytes()).to_string() +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + fn vocab() -> FeedbackVocabulary { + FeedbackVocabulary::default() + } + + #[test] + fn a_bare_thumb_classifies_in_both_directions() { + assert_eq!( + vocab().classify("\u{1f44d}"), + Some(FeedbackSignal::Positive) + ); + assert_eq!( + vocab().classify("\u{1f44e}"), + Some(FeedbackSignal::Negative) + ); + } + + #[test] + fn slack_shortcodes_classify_the_same_as_the_emoji() { + // Slack sends `+1`, Telegram sends the emoji; both are one thumb up. + assert_eq!(vocab().classify("+1"), Some(FeedbackSignal::Positive)); + assert_eq!(vocab().classify("thumbsup"), Some(FeedbackSignal::Positive)); + assert_eq!(vocab().classify("-1"), Some(FeedbackSignal::Negative)); + assert_eq!( + vocab().classify("thumbsdown"), + Some(FeedbackSignal::Negative) + ); + } + + #[test] + fn shortcodes_wrapped_in_colons_are_accepted() { + assert_eq!(vocab().classify(":+1:"), Some(FeedbackSignal::Positive)); + } + + #[test] + fn skin_tone_is_not_a_different_opinion() { + // A rendering preference must not read as a different signal, or as no + // signal at all. + assert_eq!( + vocab().classify("\u{1f44d}\u{1f3ff}"), + Some(FeedbackSignal::Positive) + ); + assert_eq!( + vocab().classify("+1::skin-tone-4"), + Some(FeedbackSignal::Positive) + ); + } + + #[test] + fn variation_selectors_do_not_defeat_matching() { + // Telegram sends the heart with an emoji presentation selector. + assert_eq!( + vocab().classify("\u{2764}\u{fe0f}"), + Some(FeedbackSignal::Positive) + ); + } + + #[test] + fn unrelated_reactions_carry_no_signal() { + // A default vocabulary that swallowed every emoji would turn party + // reactions into quality data. + assert_eq!(vocab().classify("\u{1f389}"), None); // tada + assert_eq!(vocab().classify("eyes"), None); + assert_eq!(vocab().classify(""), None); + assert_eq!(vocab().classify(" "), None); + } + + #[test] + fn case_is_ignored_for_shortcodes() { + assert_eq!(vocab().classify("ThumbsUp"), Some(FeedbackSignal::Positive)); + } + + #[test] + fn configured_lists_replace_only_the_side_they_set() { + let custom = FeedbackVocabulary::from_config(&["\u{1f389}".to_string()], &[]); + + assert_eq!(custom.classify("\u{1f389}"), Some(FeedbackSignal::Positive)); + // The positive override must not silently wipe the negative defaults. + assert_eq!(custom.classify("\u{1f44e}"), Some(FeedbackSignal::Negative)); + // ...but it does replace the positive defaults it overrode. + assert_eq!(custom.classify("\u{1f44d}"), None); + } + + #[test] + fn empty_config_falls_back_to_the_defaults() { + let empty = FeedbackVocabulary::from_config(&[], &[]); + assert_eq!(empty, FeedbackVocabulary::default()); + } + + #[test] + fn signals_map_to_boolean_friendly_numbers() { + assert!((FeedbackSignal::Positive.score_value() - 1.0).abs() < f64::EPSILON); + assert!(FeedbackSignal::Negative.score_value().abs() < f64::EPSILON); + } + + #[test] + fn the_score_id_is_stable_for_the_same_user_and_trace() { + let trace = TraceId("trace-1".into()); + let first = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("telegram:42"), + None, + None, + ); + let second = feedback_score( + &trace, + FeedbackSignal::Negative, + Some("telegram:42"), + None, + None, + ); + + // Langfuse upserts on id, so a user changing their mind must overwrite + // rather than add a second vote. + assert_eq!(first.id, second.id); + } + + #[test] + fn different_users_score_the_same_trace_independently() { + let trace = TraceId("trace-1".into()); + let alice = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("slack:alice"), + None, + None, + ); + let bob = feedback_score( + &trace, + FeedbackSignal::Positive, + Some("slack:bob"), + None, + None, + ); + + assert_ne!(alice.id, bob.id, "one user's vote overwrote another's"); + } + + #[test] + fn the_same_user_scores_different_traces_independently() { + let user = Some("telegram:42"); + let first = feedback_score_id(&TraceId("trace-1".into()), user); + let second = feedback_score_id(&TraceId("trace-2".into()), user); + + assert_ne!(first, second); + } + + #[test] + fn a_score_carries_the_canonical_name_and_environment() { + let score = feedback_score( + &TraceId("trace-1".into()), + FeedbackSignal::Positive, + Some("web:1"), + Some("great answer".into()), + Some("production".into()), + ); + + assert_eq!(score.name, USER_FEEDBACK_SCORE); + assert_eq!(score.value, ScoreValue::Numeric(1.0)); + assert_eq!(score.comment.as_deref(), Some("great answer")); + assert_eq!(score.environment.as_deref(), Some("production")); + } +} diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs index 950963a45..b1642007c 100644 --- a/crates/observability/src/lib.rs +++ b/crates/observability/src/lib.rs @@ -16,6 +16,7 @@ pub mod builder; pub mod exporters; +pub mod feedback; pub mod model; pub mod profile; pub mod recorder; @@ -25,6 +26,9 @@ pub mod sink; pub use { builder::{BuildOutcome, BuiltInstrumentation, SkippedBackend, build}, + feedback::{ + FeedbackSignal, FeedbackVocabulary, USER_FEEDBACK_SCORE, feedback_score, feedback_score_id, + }, model::{ Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, From 01c934e3df9672b622347cc150bb938d39d93ed5 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:18:46 +0000 Subject: [PATCH 10/33] feat(channels): correlate delivered replies with the trace that produced them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reaction names a message, not a turn. By the time someone reacts the agent run is long finished and nothing in the system remembers which trace wrote a given chat message, so attribution has to be recorded at send time. Keys on (channel_type, account_id, chat_id, message_id) rather than message id alone: channel message ids are only unique within a conversation — Telegram numbers them per chat — so a global lookup would confidently attribute a reaction in one chat to an unrelated turn in another. Stores identifiers only. The message log already owns message bodies, and a second copy here would quietly widen what the feedback feature retains. Upserts on relink so an edited or resent reply scores the newer turn, and prunes by age so the table does not grow for the life of the install. --- crates/channels/src/lib.rs | 2 + crates/channels/src/trace_link.rs | 83 +++++ .../migrations/20260726120000_trace_links.sql | 19 ++ crates/gateway/src/lib.rs | 1 + crates/gateway/src/trace_link_store.rs | 314 ++++++++++++++++++ 5 files changed, 419 insertions(+) create mode 100644 crates/channels/src/trace_link.rs create mode 100644 crates/gateway/migrations/20260726120000_trace_links.sql create mode 100644 crates/gateway/src/trace_link_store.rs diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index 46f16ba8e..af2f66c9e 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -17,6 +17,7 @@ pub mod plugin; pub mod registry; pub mod slack_api_url; pub mod store; +pub mod trace_link; pub use { channel_webhook_middleware::{ @@ -37,4 +38,5 @@ pub use { }, registry::{ChannelRegistry, RegistryOutboundRouter}, slack_api_url::{normalize_slack_api_base_url, validate_slack_api_base_url}, + trace_link::{TraceLink, TraceLinkStore, WEB_CHANNEL}, }; diff --git a/crates/channels/src/trace_link.rs b/crates/channels/src/trace_link.rs new file mode 100644 index 000000000..d15fca883 --- /dev/null +++ b/crates/channels/src/trace_link.rs @@ -0,0 +1,83 @@ +//! Correlation between a delivered reply and the trace that produced it. +//! +//! A reaction arrives naming a message, not a turn. Attributing it to the +//! agent run that wrote that message needs a record made at send time, because +//! by the time someone reacts the turn is long finished and nothing else in the +//! system remembers which trace a given chat message came from. +//! +//! Deliberately holds identifiers only — no message body. The link table exists +//! to answer "which trace produced this message id", and storing the text again +//! would duplicate content the message log already owns. + +use async_trait::async_trait; + +use crate::Result; + +/// Channel type recorded for replies delivered through the web UI. +pub const WEB_CHANNEL: &str = "web"; + +/// A delivered reply and the trace behind it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TraceLink { + /// Channel the reply went out on, e.g. `telegram`. [`WEB_CHANNEL`] for the + /// web UI. + pub channel_type: String, + /// Account that sent the reply. + pub account_id: String, + /// Conversation the reply landed in. + pub chat_id: String, + /// The reply's own id in the channel's namespace. This is what a reaction + /// event names. + pub message_id: String, + /// Trace that produced the reply. + pub trace_id: String, + /// Session the turn belonged to. + pub session_key: Option, + /// Unix seconds when the link was recorded. + pub created_at: i64, +} + +/// Store for reply/trace correlation. +#[async_trait] +pub trait TraceLinkStore: Send + Sync { + /// Record a link, replacing any existing link for the same message. + async fn link(&self, link: TraceLink) -> Result<()>; + + /// Find the trace behind a message, if it is still on record. + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> Result>; + + /// Drop links recorded before `cutoff` (unix seconds), returning how many + /// were removed. Without this the table grows for the life of the install. + async fn prune(&self, cutoff: i64) -> Result; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_link_is_identifiers_only() { + // Guards against a body field creeping in: the message log already owns + // message content, and a second copy here would widen what a feedback + // feature retains without anyone deciding to. + let link = TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "42".into(), + trace_id: "trace-1".into(), + session_key: Some("agent:main:main".into()), + created_at: 1_700_000_000, + }; + + let rendered = format!("{link:?}"); + assert!(rendered.contains("trace-1")); + assert!(!rendered.contains("body")); + } +} diff --git a/crates/gateway/migrations/20260726120000_trace_links.sql b/crates/gateway/migrations/20260726120000_trace_links.sql new file mode 100644 index 000000000..739df5322 --- /dev/null +++ b/crates/gateway/migrations/20260726120000_trace_links.sql @@ -0,0 +1,19 @@ +-- Correlates a delivered reply with the trace that produced it, so a reaction +-- arriving later can be scored against the right agent run. +-- +-- Identifiers only: message bodies stay in message_log rather than being +-- duplicated here. +CREATE TABLE IF NOT EXISTS trace_links ( + channel_type TEXT NOT NULL, + account_id TEXT NOT NULL, + chat_id TEXT NOT NULL, + message_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + session_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (channel_type, account_id, chat_id, message_id) +); + +-- Pruning scans by age. +CREATE INDEX IF NOT EXISTS idx_trace_links_created + ON trace_links (created_at); diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index dbefa210a..6ae68911d 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -58,6 +58,7 @@ pub mod state; pub mod tailscale; #[cfg(feature = "msteams")] pub mod teams_agent_tools; +pub mod trace_link_store; pub mod tts_phrases; pub mod update_check; pub mod updater; diff --git a/crates/gateway/src/trace_link_store.rs b/crates/gateway/src/trace_link_store.rs new file mode 100644 index 000000000..45bd929cf --- /dev/null +++ b/crates/gateway/src/trace_link_store.rs @@ -0,0 +1,314 @@ +//! SQLite-backed reply/trace correlation. + +use { + async_trait::async_trait, + moltis_channels::{ + Error as ChannelError, Result as ChannelResult, + trace_link::{TraceLink, TraceLinkStore}, + }, + sqlx::SqlitePool, +}; + +fn db_error(context: &'static str, source: sqlx::Error) -> ChannelError { + ChannelError::external(context, source) +} + +/// SQLite-backed [`TraceLinkStore`]. +pub struct SqliteTraceLinkStore { + pool: SqlitePool, +} + +impl SqliteTraceLinkStore { + /// Build a store over `pool`. + #[must_use] + pub const fn new(pool: SqlitePool) -> Self { + Self { pool } + } + + /// Create the schema directly. + /// + /// **Deprecated**: schema is managed by sqlx migrations. Retained for tests + /// on in-memory databases, matching the other channel stores. + #[doc(hidden)] + pub async fn init(pool: &SqlitePool) -> ChannelResult<()> { + sqlx::query( + "CREATE TABLE IF NOT EXISTS trace_links ( + channel_type TEXT NOT NULL, + account_id TEXT NOT NULL, + chat_id TEXT NOT NULL, + message_id TEXT NOT NULL, + trace_id TEXT NOT NULL, + session_key TEXT, + created_at INTEGER NOT NULL, + PRIMARY KEY (channel_type, account_id, chat_id, message_id) + )", + ) + .execute(pool) + .await + .map_err(|e| db_error("init trace_links table", e))?; + + sqlx::query( + "CREATE INDEX IF NOT EXISTS idx_trace_links_created + ON trace_links (created_at)", + ) + .execute(pool) + .await + .map_err(|e| db_error("init trace_links index", e))?; + + Ok(()) + } +} + +#[async_trait] +impl TraceLinkStore for SqliteTraceLinkStore { + async fn link(&self, link: TraceLink) -> ChannelResult<()> { + // Replace rather than ignore: an edited or resent reply keeps the same + // message id but belongs to the newer turn, and a reaction on it should + // score that turn. + sqlx::query( + "INSERT INTO trace_links + (channel_type, account_id, chat_id, message_id, trace_id, session_key, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (channel_type, account_id, chat_id, message_id) + DO UPDATE SET trace_id = excluded.trace_id, + session_key = excluded.session_key, + created_at = excluded.created_at", + ) + .bind(&link.channel_type) + .bind(&link.account_id) + .bind(&link.chat_id) + .bind(&link.message_id) + .bind(&link.trace_id) + .bind(&link.session_key) + .bind(link.created_at) + .execute(&self.pool) + .await + .map_err(|e| db_error("record trace link", e))?; + Ok(()) + } + + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> ChannelResult> { + let row = sqlx::query_as::<_, (String, Option, i64)>( + "SELECT trace_id, session_key, created_at + FROM trace_links + WHERE channel_type = ? AND account_id = ? AND chat_id = ? AND message_id = ?", + ) + .bind(channel_type) + .bind(account_id) + .bind(chat_id) + .bind(message_id) + .fetch_optional(&self.pool) + .await + .map_err(|e| db_error("look up trace link", e))?; + + Ok(row.map(|(trace_id, session_key, created_at)| TraceLink { + channel_type: channel_type.to_string(), + account_id: account_id.to_string(), + chat_id: chat_id.to_string(), + message_id: message_id.to_string(), + trace_id, + session_key, + created_at, + })) + } + + async fn prune(&self, cutoff: i64) -> ChannelResult { + let result = sqlx::query("DELETE FROM trace_links WHERE created_at < ?") + .bind(cutoff) + .execute(&self.pool) + .await + .map_err(|e| db_error("prune trace links", e))?; + Ok(result.rows_affected()) + } +} + +#[allow(clippy::unwrap_used, clippy::expect_used)] +#[cfg(test)] +mod tests { + use super::*; + + async fn store() -> SqliteTraceLinkStore { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + SqliteTraceLinkStore::init(&pool).await.unwrap(); + SqliteTraceLinkStore::new(pool) + } + + fn link(message_id: &str, trace_id: &str) -> TraceLink { + TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: message_id.into(), + trace_id: trace_id.into(), + session_key: Some("agent:main:main".into()), + created_at: 1_700_000_000, + } + } + + #[tokio::test] + async fn a_linked_reply_resolves_to_its_trace() { + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("link should resolve"); + + assert_eq!(found.trace_id, "trace-1"); + assert_eq!(found.session_key.as_deref(), Some("agent:main:main")); + } + + #[tokio::test] + async fn an_unknown_message_resolves_to_nothing() { + let store = store().await; + let found = store + .lookup("telegram", "bot-1", "chat-1", "missing") + .await + .unwrap(); + + assert!(found.is_none()); + } + + #[tokio::test] + async fn message_ids_are_scoped_per_channel_and_chat() { + // Message ids are only unique within a chat; Telegram numbers restart + // per conversation, so a global lookup would attribute a reaction in + // one chat to a completely unrelated turn in another. + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + + let other_chat = TraceLink { + chat_id: "chat-2".into(), + trace_id: "trace-2".into(), + ..link("42", "trace-2") + }; + store.link(other_chat).await.unwrap(); + + let first = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + let second = store + .lookup("telegram", "bot-1", "chat-2", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(first.trace_id, "trace-1"); + assert_eq!(second.trace_id, "trace-2"); + } + + #[tokio::test] + async fn the_same_message_id_on_another_channel_is_a_separate_link() { + let store = store().await; + store.link(link("42", "trace-1")).await.unwrap(); + store + .link(TraceLink { + channel_type: "discord".into(), + trace_id: "trace-discord".into(), + ..link("42", "trace-discord") + }) + .await + .unwrap(); + + let discord = store + .lookup("discord", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(discord.trace_id, "trace-discord"); + } + + #[tokio::test] + async fn relinking_a_message_points_at_the_newer_turn() { + let store = store().await; + store.link(link("42", "trace-old")).await.unwrap(); + store + .link(TraceLink { + created_at: 1_700_000_500, + ..link("42", "trace-new") + }) + .await + .unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(found.trace_id, "trace-new"); + assert_eq!(found.created_at, 1_700_000_500); + } + + #[tokio::test] + async fn pruning_drops_only_links_older_than_the_cutoff() { + let store = store().await; + store.link(link("old", "trace-old")).await.unwrap(); + store + .link(TraceLink { + created_at: 1_700_009_000, + ..link("new", "trace-new") + }) + .await + .unwrap(); + + let removed = store.prune(1_700_005_000).await.unwrap(); + + assert_eq!(removed, 1); + assert!( + store + .lookup("telegram", "bot-1", "chat-1", "old") + .await + .unwrap() + .is_none() + ); + assert!( + store + .lookup("telegram", "bot-1", "chat-1", "new") + .await + .unwrap() + .is_some(), + "pruning must not drop links still inside the retention window" + ); + } + + #[tokio::test] + async fn pruning_an_empty_table_is_a_no_op() { + let store = store().await; + assert_eq!(store.prune(1_700_000_000).await.unwrap(), 0); + } + + #[tokio::test] + async fn a_link_without_a_session_is_still_usable() { + // Channel replies that are not bound to an agent session still deserve + // feedback attribution. + let store = store().await; + store + .link(TraceLink { + session_key: None, + ..link("42", "trace-1") + }) + .await + .unwrap(); + + let found = store + .lookup("telegram", "bot-1", "chat-1", "42") + .await + .unwrap() + .expect("present"); + + assert_eq!(found.trace_id, "trace-1"); + assert!(found.session_key.is_none()); + } +} From 91e791ba5c8209ddd949693b4852b2d55ab5ab3c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:28:07 +0000 Subject: [PATCH 11/33] feat(feedback): turn channel reactions into user-feedback scores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the service that records which trace produced a reply and converts a later reaction on it into a score, plus the plumbing each half needs. Channel outbound gains `send_text_reporting_ids`, an additive default method rather than a change to `send_text`. Every channel API returns the delivered message id and every one of them throws it away, but changing the existing signature would touch all fifteen channels for one feature. Telegram, Discord and Slack override it; the rest fall back to reporting nothing, so a channel that cannot report ids loses feedback attribution rather than losing the message. It returns a list because all three split long replies into several messages and a reader may react to any of them — reporting only the last would leave a thumb on an earlier chunk unattributable. The trace id itself comes from a small bounded per-session registry populated at turn start. A reply is dispatched far from the agent loop that produced it, and sessions process turns serially, so "the current trace for this session" is unambiguous at the moment its reply goes out. That avoids threading a trace id through every outbound signature. Eviction is by insertion order and costs an attribution, never a message. Reactions are classified before the link lookup, so the many non-feedback reactions in a busy chat do not each cost a query. Removing a reaction retracts the score rather than recording another one. --- crates/agents/src/runner/instrumentation.rs | 4 + crates/channels/src/plugin.rs | 23 + crates/discord/src/handler/implementation.rs | 23 +- crates/discord/src/outbound.rs | 27 +- crates/gateway/src/feedback.rs | 505 +++++++++++++++++++ crates/gateway/src/lib.rs | 1 + crates/observability/src/lib.rs | 2 + crates/observability/src/recent.rs | 161 ++++++ crates/slack/src/outbound.rs | 30 ++ crates/telegram/src/outbound/send.rs | 43 ++ 10 files changed, 814 insertions(+), 5 deletions(-) create mode 100644 crates/gateway/src/feedback.rs create mode 100644 crates/observability/src/recent.rs diff --git a/crates/agents/src/runner/instrumentation.rs b/crates/agents/src/runner/instrumentation.rs index 74ece134a..2ffc1ccea 100644 --- a/crates/agents/src/runner/instrumentation.rs +++ b/crates/agents/src/runner/instrumentation.rs @@ -109,6 +109,10 @@ pub fn begin_turn( let scope = trace_scope(session_key, channel, environment, release); let recorder = TurnRecorder::begin("agent-run", scope, settings)?; + // The reply for this turn is sent by a dispatcher far from here, and needs + // the trace id to attribute a later reaction to the right run. + moltis_observability::remember_trace(session_key, recorder.trace_id()); + recorder.set_metadata("provider", serde_json::Value::String(provider.to_string())); recorder.set_metadata("model", serde_json::Value::String(model.to_string())); recorder.set_input(user_content_to_json(user_content)); diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index e7212befc..06cb80c1b 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -904,6 +904,29 @@ pub trait ChannelOutbound: Send + Sync { payload: &ReplyPayload, reply_to: Option<&str>, ) -> Result<()>; + /// Send text and report the ids of the messages it produced. + /// + /// Feedback attribution needs the id of the message the user will react + /// to, and most channel APIs return it from the send call and then throw + /// it away. Returns a list rather than one id because channels split long + /// replies into several messages, and a reader may react to any of them — + /// reporting only the last would leave a thumb on an earlier chunk + /// unattributable. + /// + /// Channels that can report ids override this; the default delegates to + /// [`Self::send_text`] and reports none, so a channel that cannot loses + /// feedback attribution rather than losing the message. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text(account_id, to, text, reply_to).await?; + Ok(Vec::new()) + } + /// Send a "typing" indicator. No-op by default. async fn send_typing(&self, _account_id: &str, _to: &str) -> Result<()> { Ok(()) diff --git a/crates/discord/src/handler/implementation.rs b/crates/discord/src/handler/implementation.rs index 08f8759f3..3298b1af1 100644 --- a/crates/discord/src/handler/implementation.rs +++ b/crates/discord/src/handler/implementation.rs @@ -1362,12 +1362,28 @@ pub async fn send_discord_message( text: &str, reference: Option, ) -> Result { + let mut sent = send_discord_message_all(http, channel_id, text, reference).await?; + // Always `Some` because `text` is non-empty and every chunk is sent. + sent.pop().ok_or_else(|| "no chunks produced".into()) +} + +/// Send `text`, returning every message the send produced. +/// +/// Long replies are split across several Discord messages and a reader may +/// react to any of them, so feedback attribution needs all the ids rather than +/// just the last. +pub async fn send_discord_message_all( + http: &serenity::http::Http, + channel_id: serenity::all::ChannelId, + text: &str, + reference: Option, +) -> Result, String> { if text.is_empty() { return Err("empty message".into()); } let chunks = chunk_message(text, 2000); - let mut last_msg = None; + let mut sent = Vec::with_capacity(chunks.len()); for (i, chunk) in chunks.iter().enumerate() { let mut create = CreateMessage::new().content(*chunk); // Only the first chunk gets the reply reference. @@ -1376,15 +1392,14 @@ pub async fn send_discord_message( { create = create.reference_message((channel_id, ref_id)); } - last_msg = Some( + sent.push( channel_id .send_message(http, create) .await .map_err(|e| format!("Discord send: {e}"))?, ); } - // `last_msg` is always `Some` because `text` is non-empty. - last_msg.ok_or_else(|| "no chunks produced".into()) + Ok(sent) } /// Split a message into chunks of at most `max_len` characters. diff --git a/crates/discord/src/outbound.rs b/crates/discord/src/outbound.rs index 6ef8ca9eb..235edfbfe 100644 --- a/crates/discord/src/outbound.rs +++ b/crates/discord/src/outbound.rs @@ -23,7 +23,7 @@ use { }; use crate::{ - handler::{send_discord_message, send_discord_text}, + handler::{send_discord_message, send_discord_message_all, send_discord_text}, state::AccountStateMap, }; @@ -419,6 +419,31 @@ impl ChannelOutbound for DiscordOutbound { Ok(()) } + /// Discord splits long replies across messages, so every id is reported. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let http = self.resolve_http(account_id)?; + let channel_id = Self::parse_channel_id(to)?; + let reference = self.resolve_reference(account_id, reply_to); + let sent = send_discord_message_all(&http, channel_id, text, reference) + .await + .map_err(|e| ChannelError::external("Discord send", std::io::Error::other(e)))?; + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + moltis_metrics::channels::MESSAGES_SENT_TOTAL, + moltis_metrics::labels::CHANNEL => "discord" + ) + .increment(1); + + Ok(sent.into_iter().map(|m| m.id.to_string()).collect()) + } + async fn send_media( &self, account_id: &str, diff --git a/crates/gateway/src/feedback.rs b/crates/gateway/src/feedback.rs new file mode 100644 index 000000000..79e456801 --- /dev/null +++ b/crates/gateway/src/feedback.rs @@ -0,0 +1,505 @@ +//! Reaction feedback: recording which trace a reply came from, and turning a +//! later reaction on that reply into a score. +//! +//! The two halves are deliberately in one place because they share the +//! correlation key. The send side writes (channel, account, chat, message) → +//! trace; the reaction side reads it back. + +use std::sync::Arc; + +use { + moltis_channels::{ + ChannelType, + trace_link::{TraceLink, TraceLinkStore}, + }, + moltis_config::FeedbackSettings, + moltis_observability::{ + FeedbackSignal, FeedbackVocabulary, TraceId, exporters::langfuse::LangfuseClient, + feedback_score, feedback_score_id, + }, + tracing::{debug, warn}, +}; + +/// What happened to an inbound reaction. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FeedbackOutcome { + /// A score was recorded for the trace. + Recorded(FeedbackSignal), + /// A previously recorded score was withdrawn. + Retracted, + /// The reaction carried no feedback signal. + NotFeedback, + /// The reply is no longer attributable to a trace. + UnknownMessage, + /// Feedback collection is switched off. + Disabled, +} + +/// Records reply/trace links and converts reactions into scores. +pub struct FeedbackService { + links: Arc, + vocabulary: FeedbackVocabulary, + enabled: bool, + environment: Option, +} + +impl FeedbackService { + /// Build a service from the instrumentation settings. + #[must_use] + pub fn new( + links: Arc, + settings: &FeedbackSettings, + environment: Option, + ) -> Self { + Self { + links, + vocabulary: FeedbackVocabulary::from_config(&settings.positive, &settings.negative), + enabled: settings.enabled, + environment, + } + } + + /// Whether feedback collection is on. + #[must_use] + pub const fn enabled(&self) -> bool { + self.enabled + } + + /// Record that `message_ids` were produced by `trace_id`. + /// + /// Failures are logged, never propagated: losing attribution for a reply + /// must not fail the reply itself, which the user has already received. + pub async fn record_reply( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_ids: &[String], + trace_id: &TraceId, + session_key: Option<&str>, + ) { + if !self.enabled || message_ids.is_empty() { + return; + } + let created_at = now_unix(); + for message_id in message_ids { + let link = TraceLink { + channel_type: channel_type.to_string(), + account_id: account_id.to_string(), + chat_id: chat_id.to_string(), + message_id: message_id.clone(), + trace_id: trace_id.0.clone(), + session_key: session_key.map(str::to_string), + created_at, + }; + if let Err(error) = self.links.link(link).await { + warn!(%error, channel_type, "failed to record trace link for reply"); + } + } + } + + /// Handle a reaction change on a channel message. + /// + /// `langfuse` is only needed to retract: adding a score goes through the + /// instrumentation sink like any other event, but deletion has no sink + /// representation and must call the API directly. + pub async fn on_reaction( + &self, + channel_type: ChannelType, + account_id: &str, + chat_id: &str, + message_id: &str, + emoji: &str, + user_id: &str, + added: bool, + langfuse: Option<&Arc>, + ) -> FeedbackOutcome { + if !self.enabled { + return FeedbackOutcome::Disabled; + } + + // Classify before the database lookup: most reactions in a busy chat + // are not feedback, and they should not cost a query each. + let Some(signal) = self.vocabulary.classify(emoji) else { + return FeedbackOutcome::NotFeedback; + }; + + let channel = channel_type.as_str(); + let link = match self + .links + .lookup(channel, account_id, chat_id, message_id) + .await + { + Ok(Some(link)) => link, + Ok(None) => { + debug!( + channel, + message_id, "reaction on a message with no trace link" + ); + return FeedbackOutcome::UnknownMessage; + }, + Err(error) => { + warn!(%error, channel, "failed to look up trace link for reaction"); + return FeedbackOutcome::UnknownMessage; + }, + }; + + let trace_id = TraceId(link.trace_id); + // Namespaced so the same numeric user id on two channels is two people. + let scoped_user = format!("{channel}:{user_id}"); + + if added { + let score = feedback_score( + &trace_id, + signal, + Some(&scoped_user), + Some(format!("{channel} reaction {emoji}")), + self.environment.clone(), + ); + moltis_observability::record(moltis_observability::Event::Score(Box::new(score))); + debug!(channel, ?signal, "recorded reaction feedback"); + return FeedbackOutcome::Recorded(signal); + } + + let score_id = feedback_score_id(&trace_id, Some(&scoped_user)); + let Some(client) = langfuse else { + // Nothing to retract against; the score sink has no delete path. + return FeedbackOutcome::Retracted; + }; + if let Err(error) = client.delete_score(&score_id).await { + warn!(%error, channel, "failed to retract reaction feedback"); + } + FeedbackOutcome::Retracted + } + + /// Drop links older than `retention_days`. + pub async fn prune(&self, retention_days: u32) -> u64 { + let cutoff = now_unix() - i64::from(retention_days) * 86_400; + match self.links.prune(cutoff).await { + Ok(removed) => removed, + Err(error) => { + warn!(%error, "failed to prune trace links"); + 0 + }, + } + } +} + +fn now_unix() -> i64 { + time::OffsetDateTime::now_utc().unix_timestamp() +} + +#[allow(clippy::unwrap_used, clippy::expect_used)] +#[cfg(test)] +mod tests { + use {moltis_channels::Result as ChannelResult, std::sync::Mutex, tokio::sync::OnceCell}; + + use super::*; + + #[derive(Default)] + struct MemoryLinks { + rows: Mutex>, + } + + #[async_trait::async_trait] + impl TraceLinkStore for MemoryLinks { + async fn link(&self, link: TraceLink) -> ChannelResult<()> { + let mut rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + rows.retain(|r| { + !(r.channel_type == link.channel_type + && r.account_id == link.account_id + && r.chat_id == link.chat_id + && r.message_id == link.message_id) + }); + rows.push(link); + Ok(()) + } + + async fn lookup( + &self, + channel_type: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + ) -> ChannelResult> { + let rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + Ok(rows + .iter() + .find(|r| { + r.channel_type == channel_type + && r.account_id == account_id + && r.chat_id == chat_id + && r.message_id == message_id + }) + .cloned()) + } + + async fn prune(&self, cutoff: i64) -> ChannelResult { + let mut rows = self.rows.lock().unwrap_or_else(|e| e.into_inner()); + let before = rows.len(); + rows.retain(|r| r.created_at >= cutoff); + Ok((before - rows.len()) as u64) + } + } + + fn service(enabled: bool) -> (FeedbackService, Arc) { + let links = Arc::new(MemoryLinks::default()); + let settings = FeedbackSettings { + enabled, + ..FeedbackSettings::default() + }; + let service = FeedbackService::new( + Arc::clone(&links) as Arc, + &settings, + Some("production".into()), + ); + (service, links) + } + + /// Collects scores emitted through the global sink. + /// + /// The sink is process-wide, so the tests that assert on emitted scores + /// share one and run behind a lock rather than racing each other. + static SINK_LOCK: OnceCell> = OnceCell::const_new(); + + async fn sink_guard() -> tokio::sync::MutexGuard<'static, ()> { + SINK_LOCK + .get_or_init(|| async { tokio::sync::Mutex::new(()) }) + .await + .lock() + .await + } + + #[derive(Default)] + struct CollectingSink { + events: Mutex>, + } + + #[async_trait::async_trait] + impl moltis_observability::ObservationSink for CollectingSink { + fn name(&self) -> &str { + "collecting" + } + + fn record(&self, event: moltis_observability::Event) { + self.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(event); + } + + async fn flush(&self, _timeout: std::time::Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + async fn link_reply(links: &MemoryLinks, message_id: &str, trace_id: &str) { + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: message_id.into(), + trace_id: trace_id.into(), + session_key: Some("agent:main:main".into()), + created_at: now_unix(), + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn every_chunk_of_a_reply_is_linked() { + // Long replies are split across messages and a reader may react to any + // of them, so all ids must resolve to the turn. + let (service, links) = service(true); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into(), "2".into(), "3".into()], + &TraceId("trace-1".into()), + Some("agent:main:main"), + ) + .await; + + for id in ["1", "2", "3"] { + let found = links + .lookup("telegram", "bot-1", "chat-1", id) + .await + .unwrap(); + assert_eq!(found.expect("linked").trace_id, "trace-1"); + } + } + + #[tokio::test] + async fn a_thumbs_up_records_a_positive_score() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); + + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Recorded(FeedbackSignal::Positive)); + let events = sink.events.lock().unwrap_or_else(|e| e.into_inner()); + let scored = events + .iter() + .filter_map(|e| match e { + moltis_observability::Event::Score(s) => Some(s), + _ => None, + }) + .count(); + assert_eq!(scored, 1); + + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn a_reaction_that_is_not_feedback_costs_no_lookup() { + let (service, _links) = service(true); + // No link recorded at all: if this returned UnknownMessage it would + // mean the vocabulary check ran after the database query. + let outcome = service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f389}", + "99", + true, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::NotFeedback); + } + + #[tokio::test] + async fn a_reaction_on_an_unlinked_message_is_ignored() { + let (service, _links) = service(true); + let outcome = service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "unknown", + "\u{1f44d}", + "99", + true, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::UnknownMessage); + } + + #[tokio::test] + async fn removing_a_reaction_retracts_rather_than_scoring_again() { + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + false, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Retracted); + } + + #[tokio::test] + async fn feedback_can_be_switched_off() { + let (service, links) = service(false); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Disabled); + } + + #[tokio::test] + async fn disabled_feedback_records_no_links() { + let (service, links) = service(false); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into()], + &TraceId("trace-1".into()), + None, + ) + .await; + + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "1") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn pruning_removes_links_past_the_retention_window() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "old".into(), + trace_id: "trace-old".into(), + session_key: None, + created_at: now_unix() - 60 * 86_400, + }) + .await + .unwrap(); + link_reply(&links, "new", "trace-new").await; + + let removed = service.prune(30).await; + + assert_eq!(removed, 1); + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "new") + .await + .unwrap() + .is_some() + ); + } +} diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 6ae68911d..1a9d1c14b 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -26,6 +26,7 @@ pub mod chat; pub mod chat_error; pub mod cron; pub mod external_agents; +pub mod feedback; #[cfg(feature = "local-llm")] pub mod local_llm_setup; pub mod logs; diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs index b1642007c..052c830c9 100644 --- a/crates/observability/src/lib.rs +++ b/crates/observability/src/lib.rs @@ -19,6 +19,7 @@ pub mod exporters; pub mod feedback; pub mod model; pub mod profile; +pub mod recent; pub mod recorder; pub mod redact; pub mod runtime; @@ -34,6 +35,7 @@ pub use { TokenUsage, TraceId, TraceRecord, TraceScope, }, profile::{ContentCapture, ExportProfile, Vocabulary}, + recent::{recent_trace, remember_trace}, recorder::{RecorderSettings, StepGuard, TurnRecorder}, redact::RedactionPolicy, runtime::{BatchConfig, BatchSink, SinkStatsSnapshot, Transport, TransportError}, diff --git a/crates/observability/src/recent.rs b/crates/observability/src/recent.rs new file mode 100644 index 000000000..82153b9c3 --- /dev/null +++ b/crates/observability/src/recent.rs @@ -0,0 +1,161 @@ +//! The most recent trace per session. +//! +//! A channel reply is delivered by code far from the agent loop that produced +//! it — the runner returns text, and a separate dispatcher sends it. Feedback +//! attribution needs the trace id at that send point, and threading it through +//! every outbound signature would touch every channel for the benefit of one +//! feature. +//! +//! Sessions process turns serially, so "the current trace for this session" is +//! unambiguous at the moment its reply goes out. That is what this records. +//! +//! Bounded, because an install with many channel conversations would otherwise +//! accumulate one entry per session forever. Eviction is by insertion order, +//! and losing an entry costs a feedback attribution, never a message. + +use std::{ + collections::HashMap, + sync::{OnceLock, RwLock}, +}; + +use crate::model::TraceId; + +/// How many sessions keep a remembered trace. +const CAPACITY: usize = 4_096; + +/// Insertion-ordered bounded map of session key to trace id. +#[derive(Default)] +struct RecentTraces { + by_session: HashMap, + order: Vec, +} + +impl RecentTraces { + fn remember(&mut self, session_key: String, trace_id: TraceId) { + if self + .by_session + .insert(session_key.clone(), trace_id) + .is_none() + { + self.order.push(session_key); + } + while self.order.len() > CAPACITY { + let oldest = self.order.remove(0); + self.by_session.remove(&oldest); + } + } + + fn get(&self, session_key: &str) -> Option { + self.by_session.get(session_key).cloned() + } +} + +fn registry() -> &'static RwLock { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| RwLock::new(RecentTraces::default())) +} + +/// Record `trace_id` as the session's current trace. +/// +/// No-op for an empty session key: those would all collapse onto one entry and +/// cross-attribute unrelated turns. +pub fn remember_trace(session_key: &str, trace_id: &TraceId) { + if session_key.is_empty() { + return; + } + let mut guard = registry() + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.remember(session_key.to_string(), trace_id.clone()); +} + +/// The session's most recent trace, if one is still remembered. +#[must_use] +pub fn recent_trace(session_key: &str) -> Option { + let guard = registry() + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.get(session_key) +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn a_remembered_trace_is_retrievable() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t1".into()))); + } + + #[test] + fn a_newer_turn_replaces_the_previous_trace_for_the_session() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + recent.remember("s1".into(), TraceId("t2".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t2".into()))); + } + + #[test] + fn sessions_do_not_share_traces() { + let mut recent = RecentTraces::default(); + recent.remember("s1".into(), TraceId("t1".into())); + recent.remember("s2".into(), TraceId("t2".into())); + + assert_eq!(recent.get("s1"), Some(TraceId("t1".into()))); + assert_eq!(recent.get("s2"), Some(TraceId("t2".into()))); + } + + #[test] + fn an_unknown_session_has_no_trace() { + assert_eq!(RecentTraces::default().get("nope"), None); + } + + #[test] + fn the_map_stays_bounded_under_many_sessions() { + // A busy install has one session per chat; without a bound this grows + // for the life of the process. + let mut recent = RecentTraces::default(); + for i in 0..(CAPACITY + 100) { + recent.remember(format!("s{i}"), TraceId(format!("t{i}"))); + } + + assert_eq!(recent.by_session.len(), CAPACITY); + assert_eq!(recent.order.len(), CAPACITY); + // The oldest were evicted, the newest retained. + assert!(recent.get("s0").is_none()); + assert!(recent.get(&format!("s{}", CAPACITY + 99)).is_some()); + } + + #[test] + fn repeated_turns_on_one_session_do_not_grow_the_map() { + let mut recent = RecentTraces::default(); + for i in 0..1_000 { + recent.remember("s1".into(), TraceId(format!("t{i}"))); + } + + assert_eq!(recent.by_session.len(), 1); + assert_eq!(recent.order.len(), 1, "insertion order list leaked entries"); + } + + #[test] + fn empty_session_keys_are_not_remembered() { + // Otherwise every unattributed turn collapses onto one entry and + // feedback lands on whichever turn happened to be last. + remember_trace("", &TraceId("t1".into())); + assert_eq!(recent_trace(""), None); + } + + #[test] + fn the_global_registry_round_trips() { + remember_trace("global-session", &TraceId("global-trace".into())); + assert_eq!( + recent_trace("global-session"), + Some(TraceId("global-trace".into())) + ); + } +} diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 8e4726fbc..ed0bfa9e5 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -486,6 +486,36 @@ impl ChannelOutbound for SlackOutbound { Ok(()) } + /// Slack identifies a message by its `ts`, which is what a + /// `reaction_added` event carries back. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let (client, token) = self.get_session(account_id)?; + let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let slack_text = markdown_to_slack(text); + + let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); + let mut ids = Vec::new(); + for chunk in chunks { + let ts = post_message(&client, &token, to, chunk, thread_ts.as_deref()).await?; + ids.push(ts.to_string()); + } + + #[cfg(feature = "metrics")] + moltis_metrics::counter!( + moltis_metrics::channels::MESSAGES_SENT_TOTAL, + moltis_metrics::labels::CHANNEL => "slack" + ) + .increment(1); + + Ok(ids) + } + async fn send_media( &self, account_id: &str, diff --git a/crates/telegram/src/outbound/send.rs b/crates/telegram/src/outbound/send.rs index 5c31687ac..72614f212 100644 --- a/crates/telegram/src/outbound/send.rs +++ b/crates/telegram/src/outbound/send.rs @@ -73,6 +73,49 @@ impl ChannelOutbound for TelegramOutbound { Ok(()) } + /// Telegram splits long replies into several messages, so every chunk id + /// is reported: a reader may react to any of them. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + let bot = self.get_bot(account_id)?; + let (chat_id, thread_id) = parse_chat_target(to)?; + let rp = self.reply_params(account_id, reply_to); + + let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; + + let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); + for chunk in chunks.iter() { + let reply_params = rp.as_ref(); + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + reply_params, + false, + ) + .await?; + ids.push(id.0.to_string()); + } + + info!( + account_id, + chat_id = to, + chunk_count = ids.len(), + "telegram outbound text sent with ids" + ); + Ok(ids) + } + async fn send_text_with_suffix( &self, account_id: &str, From b20ef3f8983efa356701e4b153ccb292317c5358 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:39:31 +0000 Subject: [PATCH 12/33] feat(gateway): wire reaction feedback end to end Completes the path from a thumb on a chat message to a score in Langfuse. The reply dispatcher lives in moltis-chat, which does not depend on the gateway, so the feedback service moved down into moltis-channels where both can reach it. `ChatRuntime` gains a defaulted `feedback()` accessor rather than a required method, so runtimes that do not collect feedback need no change. The service is constructed empty and installed during startup, mirroring how instrumentation itself is applied: reply/trace correlation needs the database pool, which does not exist at state construction. Everything is inert until installed, so a reaction arriving mid-startup is ignored rather than panicking, and re-applying settings takes effect without a restart. Reaction handling runs before the WebSocket broadcast so a slow client cannot delay scoring. Slack already emitted `ChannelEvent::ReactionChange` and nothing consumed it; that event is now the pipeline's entry point. Trace links are pruned on startup against the configured retention window. --- Cargo.lock | 4 + crates/channels/Cargo.toml | 5 + crates/{gateway => channels}/src/feedback.rs | 170 ++++++++++++++---- crates/channels/src/lib.rs | 2 + crates/chat/Cargo.toml | 1 + crates/chat/src/channels.rs | 54 +++++- crates/chat/src/runtime.rs | 8 + crates/gateway/src/channel_events/sink.rs | 39 ++++ crates/gateway/src/chat.rs | 4 + crates/gateway/src/lib.rs | 1 - .../src/server/prepare_core/post_state.rs | 25 +++ crates/gateway/src/state.rs | 4 + 12 files changed, 284 insertions(+), 33 deletions(-) rename crates/{gateway => channels}/src/feedback.rs (77%) diff --git a/Cargo.lock b/Cargo.lock index 0797106e8..366c763f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7300,16 +7300,19 @@ dependencies = [ name = "moltis-channels" version = "0.1.0" dependencies = [ + "anyhow", "async-trait", "bytes", "http 1.4.2", "moltis-common", "moltis-config", "moltis-metrics", + "moltis-observability", "rand 0.10.1", "serde", "serde_json", "thiserror 2.0.18", + "time", "tokio", "tracing", "url", @@ -7332,6 +7335,7 @@ dependencies = [ "moltis-config", "moltis-memory", "moltis-metrics", + "moltis-observability", "moltis-plugins", "moltis-projects", "moltis-providers", diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.toml index d61581fa2..ed2fe38e7 100644 --- a/crates/channels/Cargo.toml +++ b/crates/channels/Cargo.toml @@ -10,14 +10,19 @@ http = { workspace = true } moltis-common = { workspace = true } moltis-config = { workspace = true } moltis-metrics = { optional = true, workspace = true } +moltis-observability = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +time = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } url = { workspace = true } +[dev-dependencies] +anyhow = { workspace = true } + [features] default = [] metrics = ["dep:moltis-metrics"] diff --git a/crates/gateway/src/feedback.rs b/crates/channels/src/feedback.rs similarity index 77% rename from crates/gateway/src/feedback.rs rename to crates/channels/src/feedback.rs index 79e456801..07bbc83c7 100644 --- a/crates/gateway/src/feedback.rs +++ b/crates/channels/src/feedback.rs @@ -5,13 +5,9 @@ //! correlation key. The send side writes (channel, account, chat, message) → //! trace; the reaction side reads it back. -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use { - moltis_channels::{ - ChannelType, - trace_link::{TraceLink, TraceLinkStore}, - }, moltis_config::FeedbackSettings, moltis_observability::{ FeedbackSignal, FeedbackVocabulary, TraceId, exporters::langfuse::LangfuseClient, @@ -20,6 +16,11 @@ use { tracing::{debug, warn}, }; +use crate::{ + ChannelType, + trace_link::{TraceLink, TraceLinkStore}, +}; + /// What happened to an inbound reaction. #[derive(Debug, Clone, PartialEq, Eq)] pub enum FeedbackOutcome { @@ -35,34 +36,61 @@ pub enum FeedbackOutcome { Disabled, } -/// Records reply/trace links and converts reactions into scores. -pub struct FeedbackService { +/// The parts of the service that only exist once startup has a database. +struct Active { links: Arc, vocabulary: FeedbackVocabulary, enabled: bool, environment: Option, } +/// Records reply/trace links and converts reactions into scores. +/// +/// Constructed empty and filled in by [`FeedbackService::apply`] once startup +/// has a database pool and the resolved configuration, mirroring how +/// instrumentation itself is installed. Every operation is inert until then, +/// so a reaction arriving mid-startup is ignored rather than panicking. +#[derive(Default)] +pub struct FeedbackService { + active: RwLock>, +} + impl FeedbackService { - /// Build a service from the instrumentation settings. - #[must_use] - pub fn new( + /// Install the link store and settings. + /// + /// Replaces any previous setup so the settings UI can reconfigure feedback + /// without a restart. + pub fn apply( + &self, links: Arc, settings: &FeedbackSettings, environment: Option, - ) -> Self { - Self { + ) { + let next = Some(Active { links, vocabulary: FeedbackVocabulary::from_config(&settings.positive, &settings.negative), enabled: settings.enabled, environment, + }); + match self.active.write() { + Ok(mut guard) => *guard = next, + Err(poisoned) => *poisoned.into_inner() = next, } } /// Whether feedback collection is on. #[must_use] - pub const fn enabled(&self) -> bool { - self.enabled + pub fn enabled(&self) -> bool { + self.read(|active| active.enabled).unwrap_or(false) + } + + /// Run `f` against the installed configuration, if there is one. + fn read(&self, f: impl FnOnce(&Active) -> T) -> Option { + let guard = self + .active + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.as_ref().map(f) } /// Record that `message_ids` were produced by `trace_id`. @@ -78,7 +106,12 @@ impl FeedbackService { trace_id: &TraceId, session_key: Option<&str>, ) { - if !self.enabled || message_ids.is_empty() { + let Some((links, enabled)) = + self.read(|active| (Arc::clone(&active.links), active.enabled)) + else { + return; + }; + if !enabled || message_ids.is_empty() { return; } let created_at = now_unix(); @@ -92,7 +125,7 @@ impl FeedbackService { session_key: session_key.map(str::to_string), created_at, }; - if let Err(error) = self.links.link(link).await { + if let Err(error) = links.link(link).await { warn!(%error, channel_type, "failed to record trace link for reply"); } } @@ -114,22 +147,29 @@ impl FeedbackService { added: bool, langfuse: Option<&Arc>, ) -> FeedbackOutcome { - if !self.enabled { - return FeedbackOutcome::Disabled; - } - // Classify before the database lookup: most reactions in a busy chat // are not feedback, and they should not cost a query each. - let Some(signal) = self.vocabulary.classify(emoji) else { + let Some((links, signal, environment)) = self.read(|active| { + ( + Arc::clone(&active.links), + active + .enabled + .then(|| active.vocabulary.classify(emoji)) + .flatten(), + active.environment.clone(), + ) + }) else { + return FeedbackOutcome::Disabled; + }; + if !self.enabled() { + return FeedbackOutcome::Disabled; + } + let Some(signal) = signal else { return FeedbackOutcome::NotFeedback; }; let channel = channel_type.as_str(); - let link = match self - .links - .lookup(channel, account_id, chat_id, message_id) - .await - { + let link = match links.lookup(channel, account_id, chat_id, message_id).await { Ok(Some(link)) => link, Ok(None) => { debug!( @@ -154,7 +194,7 @@ impl FeedbackService { signal, Some(&scoped_user), Some(format!("{channel} reaction {emoji}")), - self.environment.clone(), + environment, ); moltis_observability::record(moltis_observability::Event::Score(Box::new(score))); debug!(channel, ?signal, "recorded reaction feedback"); @@ -175,7 +215,10 @@ impl FeedbackService { /// Drop links older than `retention_days`. pub async fn prune(&self, retention_days: u32) -> u64 { let cutoff = now_unix() - i64::from(retention_days) * 86_400; - match self.links.prune(cutoff).await { + let Some(links) = self.read(|active| Arc::clone(&active.links)) else { + return 0; + }; + match links.prune(cutoff).await { Ok(removed) => removed, Err(error) => { warn!(%error, "failed to prune trace links"); @@ -192,7 +235,7 @@ fn now_unix() -> i64 { #[allow(clippy::unwrap_used, clippy::expect_used)] #[cfg(test)] mod tests { - use {moltis_channels::Result as ChannelResult, std::sync::Mutex, tokio::sync::OnceCell}; + use {crate::Result as ChannelResult, std::sync::Mutex, tokio::sync::OnceCell}; use super::*; @@ -248,7 +291,8 @@ mod tests { enabled, ..FeedbackSettings::default() }; - let service = FeedbackService::new( + let service = FeedbackService::default(); + service.apply( Arc::clone(&links) as Arc, &settings, Some("production".into()), @@ -474,6 +518,72 @@ mod tests { ); } + #[tokio::test] + async fn an_uninitialized_service_is_inert() { + // A reaction arriving before startup finishes must be ignored rather + // than panicking or scoring against nothing. + let service = FeedbackService::default(); + + assert!(!service.enabled()); + assert_eq!( + service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + None, + ) + .await, + FeedbackOutcome::Disabled + ); + service + .record_reply( + "telegram", + "bot-1", + "chat-1", + &["1".into()], + &TraceId("trace-1".into()), + None, + ) + .await; + assert_eq!(service.prune(30).await, 0); + } + + #[tokio::test] + async fn reapplying_settings_takes_effect_without_a_restart() { + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + enabled: false, + ..FeedbackSettings::default() + }, + None, + ); + + assert_eq!( + service + .on_reaction( + ChannelType::Telegram, + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + None, + ) + .await, + FeedbackOutcome::Disabled + ); + } + #[tokio::test] async fn pruning_removes_links_past_the_retention_window() { let (service, links) = service(true); diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index af2f66c9e..62c6bf933 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -9,6 +9,7 @@ pub mod commands; pub mod config_view; pub mod contract; pub mod error; +pub mod feedback; pub mod gating; pub mod media_download; pub mod message_log; @@ -26,6 +27,7 @@ pub use { }, config_view::ChannelConfigView, error::{Error, Result}, + feedback::{FeedbackOutcome, FeedbackService}, media_download::{InboundMediaDownloader, InboundMediaSource}, plugin::{ ButtonRow, ButtonStyle, ChannelAttachment, ChannelCapabilities, ChannelDescriptor, diff --git a/crates/chat/Cargo.toml b/crates/chat/Cargo.toml index 5f85f681d..a63546770 100644 --- a/crates/chat/Cargo.toml +++ b/crates/chat/Cargo.toml @@ -15,6 +15,7 @@ moltis-agents = { workspace = true } moltis-channels = { workspace = true } moltis-common = { workspace = true } moltis-config = { workspace = true } +moltis-observability = { workspace = true } moltis-memory = { workspace = true } moltis-metrics = { optional = true, workspace = true } moltis-plugins = { workspace = true } diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 8a96719eb..90ca76ef9 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -455,6 +455,38 @@ pub(crate) async fn deliver_channel_error( } } +/// Link the messages a reply produced to the trace that generated it. +/// +/// Best-effort by design: a missing link costs feedback attribution for one +/// reply, and the user already has the message. +async fn record_reply_trace( + state: &Arc, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + message_ids: &[String], + session_key: &str, +) { + if message_ids.is_empty() { + return; + } + let Some(feedback) = state.feedback() else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + return; + }; + feedback + .record_reply( + target.channel_type.as_str(), + &target.account_id, + to, + message_ids, + &trace_id, + Some(session_key), + ) + .await; +} + async fn deliver_channel_replies_to_targets( outbound: Arc, targets: Vec, @@ -612,9 +644,27 @@ async fn deliver_channel_replies_to_targets( }, None => { let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) + // Ask for the delivered message ids so a later + // reaction can be attributed to this turn. Channels + // that cannot report them return an empty list and + // simply get no feedback attribution. + match outbound + .send_text_reporting_ids(&target.account_id, &to, &text, reply_to) .await + { + Ok(message_ids) => { + record_reply_trace( + &state, + &target, + &to, + &message_ids, + &session_key, + ) + .await; + Ok(()) + }, + Err(e) => Err(e), + } } else { outbound .send_text_with_suffix( diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index cf430dceb..ff0308d19 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -49,6 +49,14 @@ pub trait ChatRuntime: Send + Sync { /// Broadcast a WebSocket event to all connected clients. async fn broadcast(&self, topic: &str, payload: Value); + /// Reaction-feedback service, when one is configured. + /// + /// Defaulted to `None` so runtimes that do not collect feedback — tests, + /// embedders — need no change. + fn feedback(&self) -> Option> { + None + } + // ── Channel reply queue ────────────────────────────────────────────── /// Push a reply target for a session (channel message triggered a chat run). diff --git a/crates/gateway/src/channel_events/sink.rs b/crates/gateway/src/channel_events/sink.rs index 563763dcd..488646bd5 100644 --- a/crates/gateway/src/channel_events/sink.rs +++ b/crates/gateway/src/channel_events/sink.rs @@ -14,11 +14,50 @@ impl GatewayChannelEventSink { } } +/// Route a reaction change into the feedback pipeline. +/// +/// Runs before the broadcast so a slow WebSocket client cannot delay scoring. +async fn handle_reaction_feedback(state: &Arc, event: &ChannelEvent) { + let ChannelEvent::ReactionChange { + channel_type, + account_id, + chat_id, + message_id, + user_id, + emoji, + added, + } = event + else { + return; + }; + + let outcome = state + .feedback + .on_reaction( + *channel_type, + account_id, + chat_id, + message_id, + emoji, + user_id, + *added, + state.instrumentation.langfuse().as_ref(), + ) + .await; + debug!( + channel = channel_type.as_str(), + ?outcome, + "reaction feedback handled" + ); +} + pub(super) async fn emit( state: &Arc>>, event: ChannelEvent, ) { if let Some(state) = state.get() { + handle_reaction_feedback(state, &event).await; + let payload = match serde_json::to_value(&event) { Ok(v) => v, Err(e) => { diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index f27fad7cd..b030a0c02 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -39,6 +39,10 @@ impl ChatRuntime for GatewayChatRuntime { .await; } + fn feedback(&self) -> Option> { + Some(Arc::clone(&self.state.feedback)) + } + // ── Channel reply queue ───────────────────────────────────────────────── async fn push_channel_reply(&self, session_key: &str, target: ChannelReplyTarget) { diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 1a9d1c14b..6ae68911d 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -26,7 +26,6 @@ pub mod chat; pub mod chat_error; pub mod cron; pub mod external_agents; -pub mod feedback; #[cfg(feature = "local-llm")] pub mod local_llm_setup; pub mod logs; diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index 9ddb87471..fbf033429 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -425,6 +425,31 @@ pub(super) async fn complete_startup( } } + // ── Reaction feedback ──────────────────────────────────────────────── + // Needs the database pool for reply/trace correlation, so it is installed + // here rather than at state construction. + { + let links: Arc = Arc::new( + crate::trace_link_store::SqliteTraceLinkStore::new(db_pool.clone()), + ); + state.feedback.apply( + links, + &state.config.instrumentation.feedback, + Some(state.config.instrumentation.environment.clone()), + ); + + // Links accumulate one row per delivered reply; drop the ones too old + // to attribute a reaction to. + let feedback = Arc::clone(&state.feedback); + let retention_days = state.config.instrumentation.feedback.link_retention_days; + tokio::spawn(async move { + let removed = feedback.prune(retention_days).await; + if removed > 0 { + tracing::debug!(removed, "pruned expired trace links"); + } + }); + } + { let (webhook_tx, webhook_rx) = tokio::sync::mpsc::channel::(256); let webhook_store: Arc = { diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index 32ae25099..77992a0de 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -418,6 +418,8 @@ pub struct GatewayState { /// Live agent instrumentation (Langfuse / OTLP / Datadog). /// `Arc` because the RPC handlers and the shutdown path both reach it. pub instrumentation: Arc, + /// Reaction feedback: reply/trace correlation and score submission. + pub feedback: Arc, /// Whether the server is bound to a loopback address (localhost/127.0.0.1/::1). pub localhost_only: bool, /// Whether the server is known to be behind a reverse proxy. @@ -574,6 +576,8 @@ impl GatewayState { // Constructed empty; `apply` runs later from `prepare_core`, which // is inside a Tokio runtime (each backend spawns an export task). instrumentation: Arc::default(), + // Filled in by `prepare_core`, which has the database pool. + feedback: Arc::default(), localhost_only, behind_proxy, tls_active, From 22f2f5fae53f91fc48d1247832310e8847e0d086 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:45:02 +0000 Subject: [PATCH 13/33] feat(channels): emit reaction changes from Discord and Telegram MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Slack already reported reactions; Discord and Telegram did not, so two of the three channels the feedback pipeline targets were silent. Discord already requested the GUILD_MESSAGE_REACTIONS intent but implemented no handler for it. The bot's own acknowledgement reactions are filtered out — it marks incoming messages with an eyes/check/cross emoji, and treating those as opinions would score every turn the bot itself handled. Telegram needs `message_reaction` in `allowed_updates` or it never delivers reaction updates at all, and it reports them as a whole-set diff rather than an add/remove event: each update carries the reaction lists before and after. Scoring needs the difference, and in particular a swap from thumbs up to thumbs down arrives as a single update that must produce both a retraction and a new vote rather than only the latter. Anonymous channel reactions are skipped on Telegram and custom emoji on both: without a user id two people's votes collapse onto one score, and a custom emoji is an opaque id with no name a vocabulary could match. --- crates/discord/src/handler/implementation.rs | 60 +++++++ crates/telegram/src/bot.rs | 8 + crates/telegram/src/handlers.rs | 4 +- crates/telegram/src/handlers/reactions.rs | 175 +++++++++++++++++++ 4 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 crates/telegram/src/handlers/reactions.rs diff --git a/crates/discord/src/handler/implementation.rs b/crates/discord/src/handler/implementation.rs index 3298b1af1..3c59abec4 100644 --- a/crates/discord/src/handler/implementation.rs +++ b/crates/discord/src/handler/implementation.rs @@ -63,6 +63,58 @@ impl Handler { downloader: DiscordInboundMediaDownloader::new(), } } + + /// Surface a reaction change so the gateway can score it as feedback. + /// + /// The bot's own acknowledgement reactions are skipped: it marks incoming + /// messages with 👀/✅/❌, and treating those as user opinions would score + /// every turn the bot itself handled. + async fn emit_reaction_change( + &self, + ctx: &Context, + reaction: &serenity::all::Reaction, + added: bool, + ) { + let Some(user_id) = reaction.user_id else { + return; + }; + if user_id == ctx.cache.current_user().id { + return; + } + + let sink = { + let accts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); + match accts.get(&self.account_id) { + Some(state) => state.event_sink.clone(), + None => return, + } + }; + let Some(sink) = sink else { + return; + }; + + // Custom guild emoji have no unicode form; their name is what a + // feedback vocabulary can match against. + let emoji = match &reaction.emoji { + ReactionType::Unicode(raw) => raw.clone(), + ReactionType::Custom { name, .. } => name.clone().unwrap_or_default(), + _ => return, + }; + if emoji.is_empty() { + return; + } + + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Discord, + account_id: self.account_id.clone(), + chat_id: reaction.channel_id.to_string(), + message_id: reaction.message_id.to_string(), + user_id: user_id.to_string(), + emoji, + added, + }) + .await; + } } fn unix_now() -> i64 { @@ -657,6 +709,14 @@ fn set_bot_presence(ctx: &Context, account_id: &str, config: &DiscordAccountConf #[async_trait] impl EventHandler for Handler { + async fn reaction_add(&self, ctx: Context, reaction: serenity::all::Reaction) { + self.emit_reaction_change(&ctx, &reaction, true).await; + } + + async fn reaction_remove(&self, ctx: Context, reaction: serenity::all::Reaction) { + self.emit_reaction_change(&ctx, &reaction, false).await; + } + async fn message(&self, ctx: Context, msg: Message) { // Ignore messages from bots (including ourselves). if msg.author.bot { diff --git a/crates/telegram/src/bot.rs b/crates/telegram/src/bot.rs index 1a743676c..cd7f4ff82 100644 --- a/crates/telegram/src/bot.rs +++ b/crates/telegram/src/bot.rs @@ -111,6 +111,10 @@ pub async fn start_polling( AllowedUpdate::Message, AllowedUpdate::EditedMessage, AllowedUpdate::CallbackQuery, + // Telegram only delivers reaction updates when they are + // explicitly requested; without this the bot never learns + // that anyone reacted. + AllowedUpdate::MessageReaction, ]) .await; @@ -158,6 +162,10 @@ pub async fn start_polling( ); } }, + UpdateKind::MessageReaction(reaction) => { + handlers::handle_message_reaction(reaction, &aid, &poll_accounts) + .await; + }, UpdateKind::CallbackQuery(query) => { debug!( account_id = aid, diff --git a/crates/telegram/src/handlers.rs b/crates/telegram/src/handlers.rs index e96e1f05f..e63bafc1a 100644 --- a/crates/telegram/src/handlers.rs +++ b/crates/telegram/src/handlers.rs @@ -5,5 +5,7 @@ #[path = "handlers/implementation.rs"] mod implementation; +#[path = "handlers/reactions.rs"] +mod reactions; -pub use self::implementation::*; +pub use self::{implementation::*, reactions::handle_message_reaction}; diff --git a/crates/telegram/src/handlers/reactions.rs b/crates/telegram/src/handlers/reactions.rs new file mode 100644 index 000000000..3e9acf1c4 --- /dev/null +++ b/crates/telegram/src/handlers/reactions.rs @@ -0,0 +1,175 @@ +//! Inbound reaction updates. +//! +//! Telegram reports reactions as a whole-set diff rather than an add/remove +//! event: each update carries the reaction lists before and after the change. +//! The interesting part is therefore the difference between them. + +use std::sync::Arc; + +use { + moltis_channels::{ChannelEvent, ChannelType}, + teloxide::types::{MessageReactionUpdated, ReactionType}, + tracing::debug, +}; + +use crate::state::AccountStateMap; + +/// Emoji for a reaction, or `None` for kinds a vocabulary cannot match. +fn reaction_token(reaction: &ReactionType) -> Option { + match reaction { + ReactionType::Emoji { emoji } => Some(emoji.clone()), + // Custom emoji are identified by an opaque id with no readable name, + // so there is nothing a feedback vocabulary could match against. + _ => None, + } +} + +/// Reactions present in `after` but not `before`, and vice versa. +fn diff(before: &[ReactionType], after: &[ReactionType]) -> (Vec, Vec) { + let old: Vec = before.iter().filter_map(reaction_token).collect(); + let new: Vec = after.iter().filter_map(reaction_token).collect(); + + let added = new + .iter() + .filter(|e| !old.contains(e)) + .cloned() + .collect::>(); + let removed = old + .iter() + .filter(|e| !new.contains(e)) + .cloned() + .collect::>(); + (added, removed) +} + +/// Handle a `message_reaction` update. +pub async fn handle_message_reaction( + update: MessageReactionUpdated, + account_id: &str, + accounts: &AccountStateMap, +) { + // Anonymous channel reactions carry no user; without one, two different + // people's votes would collapse onto a single score id. + let Some(user) = update.actor.user() else { + return; + }; + + let event_sink = { + let accts = accounts.read().unwrap_or_else(|e| e.into_inner()); + match accts.get(account_id) { + Some(state) => state.event_sink.clone(), + None => return, + } + }; + let Some(sink) = event_sink else { + return; + }; + + let (added, removed) = diff(&update.old_reaction, &update.new_reaction); + if added.is_empty() && removed.is_empty() { + return; + } + + debug!( + account_id, + added = added.len(), + removed = removed.len(), + "telegram reaction update" + ); + + let emit = |emoji: String, was_added: bool| { + let sink = Arc::clone(&sink); + let account_id = account_id.to_string(); + let chat_id = update.chat.id.0.to_string(); + let message_id = update.message_id.0.to_string(); + let user_id = user.id.0.to_string(); + async move { + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Telegram, + account_id, + chat_id, + message_id, + user_id, + emoji, + added: was_added, + }) + .await; + } + }; + + for emoji in added { + emit(emoji, true).await; + } + for emoji in removed { + emit(emoji, false).await; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn emoji(raw: &str) -> ReactionType { + ReactionType::Emoji { + emoji: raw.to_string(), + } + } + + #[test] + fn a_new_reaction_is_reported_as_added() { + let (added, removed) = diff(&[], &[emoji("\u{1f44d}")]); + + assert_eq!(added, vec!["\u{1f44d}".to_string()]); + assert!(removed.is_empty()); + } + + #[test] + fn clearing_a_reaction_is_reported_as_removed() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[]); + + assert!(added.is_empty()); + assert_eq!(removed, vec!["\u{1f44d}".to_string()]); + } + + #[test] + fn switching_reactions_reports_both_sides() { + // Telegram sends one update for a swap; scoring it needs the retract + // and the new vote, not just whichever happened to be last. + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[emoji("\u{1f44e}")]); + + assert_eq!(added, vec!["\u{1f44e}".to_string()]); + assert_eq!(removed, vec!["\u{1f44d}".to_string()]); + } + + #[test] + fn an_unchanged_reaction_set_produces_nothing() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[emoji("\u{1f44d}")]); + + assert!(added.is_empty()); + assert!(removed.is_empty()); + } + + #[test] + fn adding_alongside_an_existing_reaction_only_reports_the_new_one() { + let (added, removed) = diff(&[emoji("\u{1f44d}")], &[ + emoji("\u{1f44d}"), + emoji("\u{2764}"), + ]); + + assert_eq!(added, vec!["\u{2764}".to_string()]); + assert!(removed.is_empty()); + } + + #[test] + fn custom_emoji_are_skipped() { + // Identified by an opaque id with no readable name, so a vocabulary + // has nothing to match. + let custom = ReactionType::CustomEmoji { + custom_emoji_id: teloxide::types::CustomEmojiId("12345".to_string()), + }; + let (added, removed) = diff(&[], &[custom]); + + assert!(added.is_empty()); + assert!(removed.is_empty()); + } +} From 0681daf45ec5bc6d65838b66a45d07d074ce1c00 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 14:59:20 +0000 Subject: [PATCH 14/33] feat(web): thumbs up/down feedback on assistant messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the score UI: a thumbs pair in the message action bar, and the `feedback.*` RPC behind it. The web is treated as one more channel rather than a special case, so a thumb in the browser resolves through the same correlation table as a Telegram reaction. Links are keyed on the run id, not the message index — an index shifts when older history is compacted away, which would silently re-attribute old feedback to the wrong turn. The RPC takes a named signal rather than a numeric score so the wire format cannot smuggle an arbitrary value into the metric, and clicking an already active thumb clears it instead of re-asserting it. The control only renders when instrumentation is active and feedback is enabled; a thumb that goes nowhere is worse than no thumb. When a submission cannot land, the reason is surfaced: a turn older than the retention window reports that it is too old to rate rather than showing a generic failure the user would retry forever. `on_reaction` now takes the channel as a string. The alternative was adding a `Web` variant to `ChannelType`, which enumerates real chat channels — the web UI is not one, and it would have leaked into every match over that enum. --- crates/channels/src/feedback.rs | 22 ++-- crates/chat/src/channels.rs | 29 +++++ crates/chat/src/service/chat_impl.rs | 1 + crates/chat/src/service/chat_impl/send.rs | 6 + crates/gateway/src/channel_events/sink.rs | 2 +- crates/gateway/src/methods/dispatch.rs | 2 + crates/gateway/src/methods/services.rs | 2 + .../gateway/src/methods/services/feedback.rs | 117 ++++++++++++++++++ .../assets/icons/masks/mask-thumbs-down.svg | 1 + .../src/assets/icons/masks/mask-thumbs-up.svg | 1 + crates/web/ui/e2e/specs/feedback.spec.js | 72 +++++++++++ crates/web/ui/input.css | 8 ++ crates/web/ui/src/message-actions.ts | 12 ++ crates/web/ui/src/message-feedback.ts | 97 +++++++++++++++ 14 files changed, 358 insertions(+), 14 deletions(-) create mode 100644 crates/gateway/src/methods/services/feedback.rs create mode 100644 crates/web/src/assets/icons/masks/mask-thumbs-down.svg create mode 100644 crates/web/src/assets/icons/masks/mask-thumbs-up.svg create mode 100644 crates/web/ui/e2e/specs/feedback.spec.js create mode 100644 crates/web/ui/src/message-feedback.ts diff --git a/crates/channels/src/feedback.rs b/crates/channels/src/feedback.rs index 07bbc83c7..b0c3880d5 100644 --- a/crates/channels/src/feedback.rs +++ b/crates/channels/src/feedback.rs @@ -16,10 +16,7 @@ use { tracing::{debug, warn}, }; -use crate::{ - ChannelType, - trace_link::{TraceLink, TraceLinkStore}, -}; +use crate::trace_link::{TraceLink, TraceLinkStore}; /// What happened to an inbound reaction. #[derive(Debug, Clone, PartialEq, Eq)] @@ -138,7 +135,7 @@ impl FeedbackService { /// representation and must call the API directly. pub async fn on_reaction( &self, - channel_type: ChannelType, + channel: &str, account_id: &str, chat_id: &str, message_id: &str, @@ -168,7 +165,6 @@ impl FeedbackService { return FeedbackOutcome::NotFeedback; }; - let channel = channel_type.as_str(); let link = match links.lookup(channel, account_id, chat_id, message_id).await { Ok(Some(link)) => link, Ok(None) => { @@ -388,7 +384,7 @@ mod tests { let outcome = service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", @@ -420,7 +416,7 @@ mod tests { // mean the vocabulary check ran after the database query. let outcome = service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", @@ -439,7 +435,7 @@ mod tests { let (service, _links) = service(true); let outcome = service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "unknown", @@ -460,7 +456,7 @@ mod tests { let outcome = service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", @@ -481,7 +477,7 @@ mod tests { let outcome = service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", @@ -528,7 +524,7 @@ mod tests { assert_eq!( service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", @@ -570,7 +566,7 @@ mod tests { assert_eq!( service .on_reaction( - ChannelType::Telegram, + "telegram", "bot-1", "chat-1", "42", diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 90ca76ef9..5b1cc0b76 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -455,6 +455,35 @@ pub(crate) async fn deliver_channel_error( } } +/// Link a web-delivered assistant message to the trace that produced it. +/// +/// The web is treated as one more channel so a thumb in the browser resolves +/// through the same correlation table as a Telegram reaction. The run id is +/// the message identity: unlike a message index it does not shift when older +/// history is compacted away. +pub(crate) async fn record_web_reply_trace( + state: &Arc, + session_key: &str, + run_id: &str, +) { + let Some(feedback) = state.feedback() else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + return; + }; + feedback + .record_reply( + moltis_channels::trace_link::WEB_CHANNEL, + moltis_channels::trace_link::WEB_CHANNEL, + session_key, + std::slice::from_ref(&run_id.to_string()), + &trace_id, + Some(session_key), + ) + .await; +} + /// Link the messages a reply produced to the trace that generated it. /// /// Best-effort by design: a missing link costs feedback attribution for one diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index 72d7c0a5a..cc1936a70 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -303,6 +303,7 @@ impl ChatService for LiveChatService { { warn!("send_sync: failed to persist assistant message: {e}"); } + crate::channels::record_web_reply_trace(&self.state, &session_key, &run_id).await; // Update metadata message count. if let Ok(count) = self.session_store.count(&session_key).await { self.session_metadata.touch(&session_key, count).await; diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index b8d33a0e3..5fee0c671 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -1286,6 +1286,12 @@ impl LiveChatService { { warn!("failed to persist assistant message: {e}"); } + crate::channels::record_web_reply_trace( + &state_for_drain, + &session_key_clone, + &run_id_clone, + ) + .await; // Update metadata counts. if let Ok(count) = session_store.count(&session_key_clone).await { session_metadata.touch(&session_key_clone, count).await; diff --git a/crates/gateway/src/channel_events/sink.rs b/crates/gateway/src/channel_events/sink.rs index 488646bd5..3330f16cc 100644 --- a/crates/gateway/src/channel_events/sink.rs +++ b/crates/gateway/src/channel_events/sink.rs @@ -34,7 +34,7 @@ async fn handle_reaction_feedback(state: &Arc, event: &ChannelEven let outcome = state .feedback .on_reaction( - *channel_type, + channel_type.as_str(), account_id, chat_id, message_id, diff --git a/crates/gateway/src/methods/dispatch.rs b/crates/gateway/src/methods/dispatch.rs index e1c57d02c..2f60b4fec 100644 --- a/crates/gateway/src/methods/dispatch.rs +++ b/crates/gateway/src/methods/dispatch.rs @@ -95,6 +95,8 @@ const READ_METHODS: &[&str] = &[ "webhooks.delivery.get", "webhooks.delivery.payload", "webhooks.delivery.actions", + "feedback.status", + "feedback.submit", "instrumentation.status", "heartbeat.status", "heartbeat.runs", diff --git a/crates/gateway/src/methods/services.rs b/crates/gateway/src/methods/services.rs index a519ebf22..ec10b8547 100644 --- a/crates/gateway/src/methods/services.rs +++ b/crates/gateway/src/methods/services.rs @@ -506,6 +506,7 @@ mod admin; mod agents; mod channels; mod core; +mod feedback; mod instrumentation; mod modes; mod sessions; @@ -521,6 +522,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { core::register(reg); system::register(reg); admin::register(reg); + feedback::register(reg); instrumentation::register(reg); voice_personas::register(reg); voicecall::register(reg); diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs new file mode 100644 index 000000000..6371369af --- /dev/null +++ b/crates/gateway/src/methods/services/feedback.rs @@ -0,0 +1,117 @@ +//! `feedback.*` RPC methods backing the thumbs up/down control in the web UI. +//! +//! The web is treated as one more channel: a reply gets a link keyed on the +//! session and the message index, and a thumb resolves that link the same way +//! a Telegram reaction does. + +use { + moltis_channels::FeedbackOutcome, + moltis_protocol::{ErrorShape, error_codes}, +}; + +use super::MethodRegistry; + +/// Register the `feedback.*` namespace. +pub(super) fn register(reg: &mut MethodRegistry) { + reg.register( + "feedback.status", + Box::new(|ctx| { + Box::pin(async move { + let config = &ctx.state.config.instrumentation; + Ok(serde_json::json!({ + // The control is only useful when something is collecting + // the result, so the UI hides it unless both are on. + "enabled": config.enabled && config.feedback.enabled, + "instrumentation_active": ctx.state.instrumentation.status().active, + "retention_days": config.feedback.link_retention_days, + })) + }) + }), + ); + + reg.register( + "feedback.submit", + Box::new(|ctx| { + Box::pin(async move { + let session_key = ctx + .params + .get("sessionKey") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + if session_key.is_empty() { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + "sessionKey is required", + )); + } + + let message_id = ctx + .params + .get("messageId") + .and_then(|v| v.as_str()) + .unwrap_or_default(); + if message_id.is_empty() { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + "messageId is required", + )); + } + + // `signal` rather than a raw score so the wire format cannot + // smuggle an arbitrary value into the metric. + let signal = ctx.params.get("signal").and_then(|v| v.as_str()); + let (emoji, added) = match signal { + Some("positive") => ("\u{1f44d}", true), + Some("negative") => ("\u{1f44e}", true), + // Clicking an active thumb clears it. + Some("clear") => ("\u{1f44d}", false), + other => { + return Err(ErrorShape::new( + error_codes::INVALID_REQUEST, + format!( + "signal must be \"positive\", \"negative\" or \"clear\", got {other:?}" + ), + )); + }, + }; + + let user_id = ctx + .params + .get("userId") + .and_then(|v| v.as_str()) + .unwrap_or("web"); + + let outcome = ctx + .state + .feedback + .on_reaction( + moltis_channels::trace_link::WEB_CHANNEL, + moltis_channels::trace_link::WEB_CHANNEL, + session_key, + message_id, + emoji, + user_id, + added, + ctx.state.instrumentation.langfuse().as_ref(), + ) + .await; + + Ok(serde_json::json!({ + "ok": matches!( + outcome, + FeedbackOutcome::Recorded(_) | FeedbackOutcome::Retracted + ), + "outcome": match outcome { + FeedbackOutcome::Recorded(_) => "recorded", + FeedbackOutcome::Retracted => "retracted", + FeedbackOutcome::NotFeedback => "not_feedback", + // The turn is older than the retention window, or + // predates instrumentation being switched on. + FeedbackOutcome::UnknownMessage => "unknown_message", + FeedbackOutcome::Disabled => "disabled", + }, + })) + }) + }), + ); +} diff --git a/crates/web/src/assets/icons/masks/mask-thumbs-down.svg b/crates/web/src/assets/icons/masks/mask-thumbs-down.svg new file mode 100644 index 000000000..9f8f7e445 --- /dev/null +++ b/crates/web/src/assets/icons/masks/mask-thumbs-down.svg @@ -0,0 +1 @@ + diff --git a/crates/web/src/assets/icons/masks/mask-thumbs-up.svg b/crates/web/src/assets/icons/masks/mask-thumbs-up.svg new file mode 100644 index 000000000..147041334 --- /dev/null +++ b/crates/web/src/assets/icons/masks/mask-thumbs-up.svg @@ -0,0 +1 @@ + diff --git a/crates/web/ui/e2e/specs/feedback.spec.js b/crates/web/ui/e2e/specs/feedback.spec.js new file mode 100644 index 000000000..04c37616b --- /dev/null +++ b/crates/web/ui/e2e/specs/feedback.spec.js @@ -0,0 +1,72 @@ +const { expect, test } = require("../base-test"); +const { navigateAndWait, watchPageErrors, expectRpcOk, sendRpcFromPage } = require("../helpers"); + +test.describe("Reaction feedback", () => { + test("status RPC responds", async ({ page }) => { + await navigateAndWait(page, "/"); + + const response = await expectRpcOk(page, "feedback.status", {}); + expect(typeof response.payload.enabled).toBe("boolean"); + expect(typeof response.payload.retention_days).toBe("number"); + }); + + test("reports feedback as unavailable while instrumentation is off", async ({ page }) => { + await navigateAndWait(page, "/"); + + // A thumb that goes nowhere is worse than no thumb, so the control is + // gated on something actually collecting the score. + const response = await expectRpcOk(page, "feedback.status", {}); + expect(response.payload.enabled).toBe(false); + }); + + test("submitting without a session key is rejected", async ({ page }) => { + await navigateAndWait(page, "/"); + + const response = await sendRpcFromPage(page, "feedback.submit", { + messageId: "run-1", + signal: "positive", + }); + + expect(response.ok).toBe(false); + }); + + test("submitting an unknown signal is rejected", async ({ page }) => { + await navigateAndWait(page, "/"); + + // The wire format must not be able to smuggle an arbitrary value into + // the score. + const response = await sendRpcFromPage(page, "feedback.submit", { + sessionKey: "agent:main:main", + messageId: "run-1", + signal: "0.7", + }); + + expect(response.ok).toBe(false); + }); + + test("a thumb on an unlinked message reports why rather than failing silently", async ({ page }) => { + await navigateAndWait(page, "/"); + + const response = await sendRpcFromPage(page, "feedback.submit", { + sessionKey: "agent:main:main", + messageId: "run-that-never-existed", + signal: "positive", + }); + + // The RPC itself succeeds; the outcome explains that the turn is not + // attributable, which is not an error the user can retry away. + expect(response.ok).toBe(true); + expect(response.payload.ok).toBe(false); + expect(["unknown_message", "disabled"]).toContain(response.payload.outcome); + }); + + test("chat page loads with no JS errors when feedback is unavailable", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/"); + + // The action bar appends thumbs asynchronously; a failed or disabled + // status check must not surface as a page error. + await expect(page.locator("#chat-input")).toBeVisible(); + expect(pageErrors).toEqual([]); + }); +}); diff --git a/crates/web/ui/input.css b/crates/web/ui/input.css index 10a59f0eb..84bafd8a8 100644 --- a/crates/web/ui/input.css +++ b/crates/web/ui/input.css @@ -2205,6 +2205,14 @@ -webkit-mask-image: url("./icons/masks/mask-git-fork.svg"); mask-image: url("./icons/masks/mask-git-fork.svg"); } + .icon-thumbs-up { + -webkit-mask-image: url("./icons/masks/mask-thumbs-up.svg"); + mask-image: url("./icons/masks/mask-thumbs-up.svg"); + } + .icon-thumbs-down { + -webkit-mask-image: url("./icons/masks/mask-thumbs-down.svg"); + mask-image: url("./icons/masks/mask-thumbs-down.svg"); + } .icon-list-plus { -webkit-mask-image: url("./icons/masks/mask-list-plus.svg"); mask-image: url("./icons/masks/mask-list-plus.svg"); diff --git a/crates/web/ui/src/message-actions.ts b/crates/web/ui/src/message-actions.ts index 1a9683e1a..c88fad04d 100644 --- a/crates/web/ui/src/message-actions.ts +++ b/crates/web/ui/src/message-actions.ts @@ -8,6 +8,7 @@ import { isChatAtBottom, scrollChatToBottom } from "./chat-ui"; import * as gon from "./gon"; import { sendRpc } from "./helpers"; +import { buildFeedbackButtons, feedbackStatus } from "./message-feedback"; import { renderPersistedAudio } from "./message-voice"; import { copyToClipboard, showToast } from "./ui"; @@ -138,6 +139,17 @@ export function appendMessageActions(ctx: MessageActionContext): void { bar.appendChild(voiceBtn); } + // ── Feedback thumbs ────────────────────────────────────── + // Appended asynchronously: availability depends on whether any backend is + // collecting scores, and the action bar must not wait on that check. + void feedbackStatus().then((status) => { + if (!status?.enabled || !status.instrumentation_active) return; + if (!bar.isConnected) return; + const buttons = buildFeedbackButtons({ sessionKey, runId: ctx.runId }, actionButton); + if (!buttons) return; + for (const button of buttons) bar.appendChild(button); + }); + // ── Fork button ────────────────────────────────────────── const forkBtn = actionButton("icon-git-fork", "Fork into new session"); forkBtn.addEventListener("click", () => { diff --git a/crates/web/ui/src/message-feedback.ts b/crates/web/ui/src/message-feedback.ts new file mode 100644 index 000000000..8c0f2d76a --- /dev/null +++ b/crates/web/ui/src/message-feedback.ts @@ -0,0 +1,97 @@ +// ── Thumbs up/down feedback on an assistant message ────────── +// +// Submits a "user-feedback" score against the trace that produced the +// message. The control only appears when instrumentation is actually +// collecting scores — a thumb that goes nowhere is worse than no thumb. + +import { sendRpc } from "./helpers"; +import { showToast } from "./ui"; + +export type FeedbackSignal = "positive" | "negative"; + +interface FeedbackStatus { + enabled: boolean; + instrumentation_active: boolean; + retention_days: number; +} + +let statusPromise: Promise | null = null; + +/** Feedback availability, fetched once per page load. */ +export function feedbackStatus(): Promise { + if (!statusPromise) { + statusPromise = sendRpc("feedback.status", {}).then((result) => { + if (!result?.ok) return null; + return (result.payload as FeedbackStatus | undefined) ?? null; + }); + } + return statusPromise; +} + +/** Reset the cached status, for tests and after a settings change. */ +export function resetFeedbackStatus(): void { + statusPromise = null; +} + +export interface FeedbackContext { + sessionKey: string; + runId?: string; +} + +/** + * Build the thumbs up/down pair, or `null` when there is nothing to attribute + * a score to. + * + * A message with no run id predates the correlation table, so a thumb on it + * could not be tied to a trace. + */ +export function buildFeedbackButtons( + ctx: FeedbackContext, + makeButton: (iconClass: string, title: string) => HTMLButtonElement, +): HTMLElement[] | null { + if (!ctx.runId) return null; + + let active: FeedbackSignal | null = null; + const up = makeButton("icon-thumbs-up", "Good response"); + const down = makeButton("icon-thumbs-down", "Bad response"); + + const submit = async (signal: FeedbackSignal, button: HTMLButtonElement): Promise => { + // Clicking the active thumb withdraws the opinion rather than + // re-asserting it. + const clearing = active === signal; + const result = await sendRpc("feedback.submit", { + sessionKey: ctx.sessionKey, + messageId: ctx.runId, + signal: clearing ? "clear" : signal, + }); + + const payload = result?.payload as { ok?: boolean; outcome?: string } | undefined; + if (!result?.ok || !payload?.ok) { + // Say why rather than failing silently: "unknown_message" means the + // turn aged out of the retention window, which is not the user's + // fault and not something a retry fixes. + const reason = + payload?.outcome === "unknown_message" + ? "This message is too old to rate" + : payload?.outcome === "disabled" + ? "Feedback collection is turned off" + : "Could not record feedback"; + showToast(reason, "error"); + return; + } + + up.classList.remove("msg-action-btn-active"); + down.classList.remove("msg-action-btn-active"); + active = clearing ? null : signal; + if (!clearing) button.classList.add("msg-action-btn-active"); + up.setAttribute("aria-pressed", String(active === "positive")); + down.setAttribute("aria-pressed", String(active === "negative")); + }; + + up.setAttribute("aria-pressed", "false"); + down.setAttribute("aria-pressed", "false"); + up.addEventListener("click", () => void submit("positive", up)); + down.addEventListener("click", () => void submit("negative", down)); + + return [up, down]; +} From 3c323148182f920235e40df736c56377ac5cb94e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 15:02:30 +0000 Subject: [PATCH 15/33] feat(observability): price token usage locally Cost was never computed anywhere in the process; `cost_details` existed on the model but was always empty, so every backend had to derive spend from a private price table of its own, and nothing could show cost without one configured. Adds a built-in USD-per-million-tokens table and prices usage at the point it is recorded. Cache buckets are priced separately rather than folded into input: a cache read can be an order of magnitude cheaper, and summing them would overstate spend on exactly the workloads caching exists to make cheaper. Lookup is longest-prefix over a normalized model id, so a dated snapshot like `claude-opus-4-5-20260101` inherits its family's price without an entry per release, `gpt-4o-mini` is not priced as `gpt-4o`, and the same model routed through OpenRouter prices identically to one called directly. An unknown model reports no cost rather than falling back to an average. A confident wrong number that looks authoritative is worse than a visible gap. `set_model` re-prices any usage already on the step, because callers are free to record usage before the model is known and the step would otherwise keep a cost of zero forever. --- crates/observability/src/lib.rs | 2 + crates/observability/src/pricing.rs | 257 +++++++++++++++++++++++++++ crates/observability/src/recorder.rs | 79 +++++++- 3 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 crates/observability/src/pricing.rs diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs index 052c830c9..91f47ab69 100644 --- a/crates/observability/src/lib.rs +++ b/crates/observability/src/lib.rs @@ -18,6 +18,7 @@ pub mod builder; pub mod exporters; pub mod feedback; pub mod model; +pub mod pricing; pub mod profile; pub mod recent; pub mod recorder; @@ -34,6 +35,7 @@ pub use { Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, }, + pricing::{ModelPrice, cost_details, price_for, total_cost}, profile::{ContentCapture, ExportProfile, Vocabulary}, recent::{recent_trace, remember_trace}, recorder::{RecorderSettings, StepGuard, TurnRecorder}, diff --git a/crates/observability/src/pricing.rs b/crates/observability/src/pricing.rs new file mode 100644 index 000000000..e75a7034d --- /dev/null +++ b/crates/observability/src/pricing.rs @@ -0,0 +1,257 @@ +//! Local model pricing. +//! +//! Cost is computed here rather than left to the backend so it exists whether +//! or not anything is configured, and so the numbers on the Metrics page do not +//! depend on a third party being reachable. +//! +//! Prices are USD per million tokens and go stale whenever a provider changes +//! them, which is the accepted cost of not needing a backend to see spend. A +//! model with no entry reports no cost rather than guessing: a wrong number +//! that looks authoritative is worse than a visible gap. + +use std::collections::BTreeMap; + +use crate::model::TokenUsage; + +/// USD per million tokens for one model. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ModelPrice { + /// Fresh input tokens. + pub input: f64, + /// Generated tokens. + pub output: f64, + /// Tokens read from the prompt cache, usually far cheaper than fresh input. + pub cache_read: f64, + /// Tokens written to the prompt cache, usually dearer than fresh input. + pub cache_write: f64, +} + +impl ModelPrice { + /// A price with no cache tiers, for providers that do not bill them + /// separately. + const fn flat(input: f64, output: f64) -> Self { + Self { + input, + output, + cache_read: input, + cache_write: input, + } + } + + /// A price with explicit cache tiers. + const fn cached(input: f64, output: f64, cache_read: f64, cache_write: f64) -> Self { + Self { + input, + output, + cache_read, + cache_write, + } + } + + /// Cost in USD for `usage`, broken down by token bucket. + /// + /// Cache buckets are priced separately rather than folded into input: + /// a cache read can be an order of magnitude cheaper, and summing them + /// would overstate spend on exactly the workloads caching is meant to help. + #[must_use] + pub fn cost_details(&self, usage: &TokenUsage) -> BTreeMap { + const PER: f64 = 1_000_000.0; + let mut details = BTreeMap::new(); + let mut put = |key: &str, tokens: u32, rate: f64| { + if tokens > 0 && rate > 0.0 { + details.insert(key.to_string(), (tokens as f64) * rate / PER); + } + }; + put("input", usage.input, self.input); + put("output", usage.output, self.output); + put("cache_read", usage.cache_read, self.cache_read); + put("cache_write", usage.cache_write, self.cache_write); + + let total: f64 = details.values().sum(); + if total > 0.0 { + details.insert("total".to_string(), total); + } + details + } +} + +/// Built-in prices, keyed by a normalized model id prefix. +/// +/// Matched by longest prefix so a dated release like +/// `claude-opus-4-5-20260101` picks up its family's price without an entry per +/// snapshot. +const PRICES: &[(&str, ModelPrice)] = &[ + // Anthropic + ("claude-opus-4", ModelPrice::cached(15.0, 75.0, 1.5, 18.75)), + ("claude-sonnet-4", ModelPrice::cached(3.0, 15.0, 0.3, 3.75)), + ("claude-haiku-4", ModelPrice::cached(1.0, 5.0, 0.1, 1.25)), + ("claude-3-5-haiku", ModelPrice::cached(0.8, 4.0, 0.08, 1.0)), + ( + "claude-3-5-sonnet", + ModelPrice::cached(3.0, 15.0, 0.3, 3.75), + ), + ("claude-3-opus", ModelPrice::cached(15.0, 75.0, 1.5, 18.75)), + // OpenAI + ("gpt-4o-mini", ModelPrice::cached(0.15, 0.6, 0.075, 0.15)), + ("gpt-4o", ModelPrice::cached(2.5, 10.0, 1.25, 2.5)), + ("gpt-4.1-mini", ModelPrice::cached(0.4, 1.6, 0.1, 0.4)), + ("gpt-4.1", ModelPrice::cached(2.0, 8.0, 0.5, 2.0)), + ("o3-mini", ModelPrice::flat(1.1, 4.4)), + ("o3", ModelPrice::flat(2.0, 8.0)), + // Google + ("gemini-2.5-pro", ModelPrice::flat(1.25, 10.0)), + ("gemini-2.5-flash", ModelPrice::flat(0.3, 2.5)), + ("gemini-2.0-flash", ModelPrice::flat(0.1, 0.4)), + // Meta / open weights, typical hosted rates + ("llama-3.3-70b", ModelPrice::flat(0.6, 0.6)), + ("llama-3.1-8b", ModelPrice::flat(0.05, 0.08)), + ("mistral-large", ModelPrice::flat(2.0, 6.0)), + ("deepseek-chat", ModelPrice::flat(0.27, 1.1)), + ("deepseek-reasoner", ModelPrice::flat(0.55, 2.19)), +]; + +/// Look up a price for `model`. +/// +/// Provider prefixes (`anthropic/claude-opus-4`) are stripped first so a model +/// routed through OpenRouter prices the same as one called directly. +#[must_use] +pub fn price_for(model: &str) -> Option { + let normalized = model + .rsplit('/') + .next() + .unwrap_or(model) + .trim() + .to_ascii_lowercase(); + if normalized.is_empty() { + return None; + } + + PRICES + .iter() + .filter(|(prefix, _)| normalized.starts_with(prefix)) + // Longest prefix wins, so `gpt-4o-mini` does not match `gpt-4o`. + .max_by_key(|(prefix, _)| prefix.len()) + .map(|(_, price)| *price) +} + +/// Cost breakdown for `usage` on `model`, empty when the model is unpriced. +#[must_use] +pub fn cost_details(model: &str, usage: &TokenUsage) -> BTreeMap { + price_for(model).map_or_else(BTreeMap::new, |price| price.cost_details(usage)) +} + +/// Total USD for `usage` on `model`, or `None` when the model is unpriced. +#[must_use] +pub fn total_cost(model: &str, usage: &TokenUsage) -> Option { + cost_details(model, usage).get("total").copied() +} + +#[cfg(test)] +#[allow(clippy::expect_used, clippy::unwrap_used)] +mod tests { + use super::*; + + fn usage(input: u32, output: u32, cache_read: u32, cache_write: u32) -> TokenUsage { + TokenUsage::from_provider_totals(input, output, cache_read, cache_write) + } + + #[test] + fn a_known_model_prices_input_and_output() { + let details = cost_details("claude-opus-4", &usage(1_000_000, 1_000_000, 0, 0)); + + assert!((details["input"] - 15.0).abs() < 1e-9); + assert!((details["output"] - 75.0).abs() < 1e-9); + assert!((details["total"] - 90.0).abs() < 1e-9); + } + + #[test] + fn cache_reads_are_not_priced_as_fresh_input() { + // Folding cache into input would overstate spend by 10x on exactly the + // workloads caching exists to make cheaper. + let details = cost_details("claude-opus-4", &usage(0, 0, 1_000_000, 0)); + + assert!((details["cache_read"] - 1.5).abs() < 1e-9); + assert!(!details.contains_key("input")); + } + + #[test] + fn cache_writes_are_priced_above_fresh_input() { + let details = cost_details("claude-opus-4", &usage(0, 0, 0, 1_000_000)); + assert!(details["cache_write"] > 15.0); + } + + #[test] + fn the_longest_matching_prefix_wins() { + // `gpt-4o-mini` must not be priced as `gpt-4o`. + let mini = price_for("gpt-4o-mini").expect("priced"); + let full = price_for("gpt-4o").expect("priced"); + + assert!((mini.input - 0.15).abs() < 1e-9); + assert!((full.input - 2.5).abs() < 1e-9); + } + + #[test] + fn dated_snapshots_inherit_the_family_price() { + let dated = price_for("claude-opus-4-5-20260101").expect("priced"); + let family = price_for("claude-opus-4").expect("priced"); + + assert_eq!(dated, family); + } + + #[test] + fn a_provider_prefix_is_stripped() { + // The same model routed through OpenRouter must price identically. + assert_eq!( + price_for("anthropic/claude-opus-4"), + price_for("claude-opus-4") + ); + assert_eq!(price_for("openai/gpt-4o"), price_for("gpt-4o")); + } + + #[test] + fn model_ids_are_matched_case_insensitively() { + assert_eq!(price_for("Claude-Opus-4"), price_for("claude-opus-4")); + } + + #[test] + fn an_unknown_model_reports_no_cost_rather_than_guessing() { + // A confident wrong number is worse than a visible gap. + assert!(price_for("some-private-finetune").is_none()); + assert!(cost_details("some-private-finetune", &usage(1000, 1000, 0, 0)).is_empty()); + assert_eq!( + total_cost("some-private-finetune", &usage(1000, 1000, 0, 0)), + None + ); + } + + #[test] + fn an_empty_model_id_is_unpriced() { + assert!(price_for("").is_none()); + assert!(price_for(" ").is_none()); + } + + #[test] + fn zero_usage_produces_no_entries() { + assert!(cost_details("claude-opus-4", &usage(0, 0, 0, 0)).is_empty()); + } + + #[test] + fn the_total_is_the_sum_of_the_buckets() { + let details = cost_details("claude-opus-4", &usage(1_000, 2_000, 3_000, 4_000)); + let summed: f64 = details + .iter() + .filter(|(k, _)| k.as_str() != "total") + .map(|(_, v)| v) + .sum(); + + assert!((details["total"] - summed).abs() < 1e-12); + } + + #[test] + fn flat_priced_models_charge_cache_reads_as_input() { + // Providers that do not bill a cache tier must not silently price + // cached tokens at zero. + let details = cost_details("gemini-2.5-pro", &usage(0, 0, 1_000_000, 0)); + assert!((details["cache_read"] - 1.25).abs() < 1e-9); + } +} diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index e1d148e86..ba32ed345 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -263,9 +263,21 @@ impl StepGuard { } /// Set the model this step used. + /// Name the model this step used. + /// + /// Re-prices any usage already recorded: callers are free to set usage + /// before the model is known, and without this the step would keep a cost + /// of zero forever. pub fn set_model(&mut self, model: impl Into) { if let Some(record) = self.record.as_mut() { - record.model = Some(model.into()); + let model = model.into(); + if let Some(usage) = record.usage { + let details = crate::pricing::cost_details(&model, &usage); + if !details.is_empty() { + record.cost_details = details; + } + } + record.model = Some(model); } } @@ -277,8 +289,19 @@ impl StepGuard { } /// Set token usage. + /// Attach token usage, pricing it locally when the model is known. + /// + /// Cost is computed here rather than left to the backend so it exists + /// without one configured, and so every backend sees the same number + /// instead of each deriving its own from a private price table. pub fn set_usage(&mut self, usage: TokenUsage) { if let Some(record) = self.record.as_mut() { + if let Some(model) = record.model.as_deref() { + let details = crate::pricing::cost_details(model, &usage); + if !details.is_empty() { + record.cost_details = details; + } + } record.usage = Some(usage); } } @@ -715,6 +738,60 @@ mod tests { }); } + #[test] + fn usage_is_priced_when_the_model_is_known() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) + .expect("recorder starts"); + let mut step = recorder.step(ObservationKind::Generation, "gen"); + step.set_model("claude-opus-4"); + step.set_usage(TokenUsage::from_provider_totals(1_000_000, 0, 0, 0)); + step.finish(); + + let observations = collected.observations(); + let record = observations.last().expect("one observation"); + assert!((record.cost_details["total"] - 15.0).abs() < 1e-9); + }); + } + + #[test] + fn usage_recorded_before_the_model_is_still_priced() { + // Callers are free to set usage first; without re-pricing on + // `set_model` the step would keep a cost of zero forever. + with_sink(|collected| { + let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) + .expect("recorder starts"); + let mut step = recorder.step(ObservationKind::Generation, "gen"); + step.set_usage(TokenUsage::from_provider_totals(1_000_000, 0, 0, 0)); + step.set_model("claude-opus-4"); + step.finish(); + + let observations = collected.observations(); + let record = observations.last().expect("one observation"); + assert!((record.cost_details["total"] - 15.0).abs() < 1e-9); + }); + } + + #[test] + fn an_unpriced_model_records_usage_without_a_cost() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) + .expect("recorder starts"); + let mut step = recorder.step(ObservationKind::Generation, "gen"); + step.set_model("some-private-finetune"); + step.set_usage(TokenUsage::from_provider_totals(1_000, 1_000, 0, 0)); + step.finish(); + + let observations = collected.observations(); + let record = observations.last().expect("one observation"); + assert!(record.usage.is_some()); + assert!( + record.cost_details.is_empty(), + "a guessed cost is worse than none" + ); + }); + } + #[test] fn scores_are_emitted_against_the_turn() { with_sink(|collected| { From 3d5f81c1188ded2e9f7631182833a355b5a31bcb Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 26 Jul 2026 15:13:52 +0000 Subject: [PATCH 16/33] refactor: split four files back under the 1500-line limit The feedback and pricing work pushed four files that were sitting just under the limit over it. Each split follows the domain boundary rather than cutting at an arbitrary line: - chat: reply/trace correlation and the compaction notice both move out of the channel delivery module into siblings. - discord: inbound reaction handling moves next to the message handler. - gateway: the feedback startup wiring moves out of post_state. - channels: ChannelType and its parsing leave the plugin trait file. No behaviour change. --- crates/channels/src/plugin.rs | 290 +---------------- crates/channels/src/plugin/channel_type.rs | 294 ++++++++++++++++++ crates/chat/src/channel_compaction.rs | 101 ++++++ crates/chat/src/channel_feedback.rs | 70 +++++ crates/chat/src/channels.rs | 156 +--------- crates/chat/src/lib.rs | 2 + crates/chat/src/run_with_tools.rs | 5 +- crates/chat/src/service/chat_impl.rs | 5 +- crates/chat/src/service/chat_impl/send.rs | 2 +- crates/discord/src/handler.rs | 2 + crates/discord/src/handler/implementation.rs | 52 ---- crates/discord/src/handler/reactions.rs | 63 ++++ crates/gateway/src/server/prepare_core.rs | 1 + .../src/server/prepare_core/feedback.rs | 36 +++ .../src/server/prepare_core/post_state.rs | 34 +- docs/src/instrumentation.md | 69 ++++ 16 files changed, 658 insertions(+), 524 deletions(-) create mode 100644 crates/channels/src/plugin/channel_type.rs create mode 100644 crates/chat/src/channel_compaction.rs create mode 100644 crates/chat/src/channel_feedback.rs create mode 100644 crates/discord/src/handler/reactions.rs create mode 100644 crates/gateway/src/server/prepare_core/feedback.rs diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index 06cb80c1b..d165cd4cf 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -1,3 +1,8 @@ +#[path = "plugin/channel_type.rs"] +mod channel_type; + +pub use self::channel_type::ChannelType; + use std::sync::Arc; use { @@ -10,291 +15,6 @@ use crate::{Error, Result, config_view::ChannelConfigView}; // ── Channel type enum ─────────────────────────────────────────────────────── -/// Supported channel types. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "lowercase")] -#[non_exhaustive] -pub enum ChannelType { - Telegram, - Whatsapp, - #[serde(rename = "msteams")] - MsTeams, - Discord, - Slack, - Matrix, - Nostr, - Signal, - Telephony, -} - -impl ChannelType { - /// Returns the channel type identifier as a string slice. - pub fn as_str(&self) -> &'static str { - match self { - Self::Telegram => "telegram", - Self::Whatsapp => "whatsapp", - Self::MsTeams => "msteams", - Self::Discord => "discord", - Self::Slack => "slack", - Self::Matrix => "matrix", - Self::Nostr => "nostr", - Self::Signal => "signal", - Self::Telephony => "telephony", - } - } - - /// Human-readable display name for UI labels. - pub fn display_name(&self) -> &'static str { - match self { - Self::Telegram => "Telegram", - Self::Whatsapp => "WhatsApp", - Self::MsTeams => "Microsoft Teams", - Self::Discord => "Discord", - Self::Slack => "Slack", - Self::Matrix => "Matrix", - Self::Nostr => "Nostr", - Self::Signal => "Signal", - Self::Telephony => "Phone Call", - } - } - - /// Best-effort chat classification for hook and prompt context. - #[must_use] - pub fn classify_chat(&self, chat_id: &str) -> Option { - match self { - Self::Telegram => { - if chat_id.starts_with("-100") { - Some("channel_or_supergroup".to_string()) - } else if chat_id.starts_with('-') { - Some("group".to_string()) - } else { - Some("private".to_string()) - } - }, - Self::Signal => { - if chat_id.starts_with("group:") { - Some("group".to_string()) - } else { - Some("direct".to_string()) - } - }, - Self::Nostr => Some("dm".to_string()), - Self::Telephony => Some("call".to_string()), - _ => None, - } - } - - /// Top-level config fields that must be treated as persisted secrets. - pub fn secret_fields(&self) -> &'static [&'static str] { - match self { - Self::Telegram => &["token"], - Self::Whatsapp => &[], - Self::MsTeams => &["app_password", "webhook_secret"], - Self::Discord => &["token"], - Self::Slack => &["bot_token", "app_token", "signing_secret"], - Self::Matrix => &["access_token", "password"], - Self::Nostr => &["secret_key"], - Self::Signal => &[], - Self::Telephony => &["auth_token"], - } - } -} - -impl std::fmt::Display for ChannelType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.as_str()) - } -} - -impl std::str::FromStr for ChannelType { - type Err = Error; - - fn from_str(s: &str) -> std::result::Result { - match s { - "telegram" => Ok(Self::Telegram), - "whatsapp" => Ok(Self::Whatsapp), - "msteams" | "microsoft_teams" | "microsoft-teams" | "teams" => Ok(Self::MsTeams), - "discord" => Ok(Self::Discord), - "slack" => Ok(Self::Slack), - "matrix" | "element" => Ok(Self::Matrix), - "nostr" => Ok(Self::Nostr), - "signal" => Ok(Self::Signal), - "telephony" | "phone" | "voice_call" | "voicecall" => Ok(Self::Telephony), - other => Err(Error::invalid_input(format!( - "unknown channel type: {other}" - ))), - } - } -} - -impl ChannelType { - /// All known channel types. - pub const ALL: &[ChannelType] = &[ - Self::Telegram, - Self::Whatsapp, - Self::MsTeams, - Self::Discord, - Self::Slack, - Self::Matrix, - Self::Nostr, - Self::Signal, - Self::Telephony, - ]; - - /// Returns the static descriptor for this channel type. - #[must_use] - pub fn descriptor(&self) -> ChannelDescriptor { - match self { - Self::Telegram => ChannelDescriptor { - channel_type: *self, - display_name: "Telegram", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Polling, - supports_outbound: true, - supports_streaming: true, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: true, - }, - }, - Self::Whatsapp => ChannelDescriptor { - channel_type: *self, - display_name: "WhatsApp", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: true, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::MsTeams => ChannelDescriptor { - channel_type: *self, - display_name: "Microsoft Teams", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Webhook, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: true, - }, - }, - Self::Discord => ChannelDescriptor { - channel_type: *self, - display_name: "Discord", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: false, - supports_reactions: false, - supports_location: true, - }, - }, - Self::Slack => ChannelDescriptor { - channel_type: *self, - display_name: "Slack", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::SocketMode, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: false, - }, - }, - Self::Matrix => ChannelDescriptor { - channel_type: *self, - display_name: "Matrix", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: true, - supports_interactive: true, - supports_threads: true, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: true, - supports_reactions: true, - supports_location: true, - }, - }, - Self::Nostr => ChannelDescriptor { - channel_type: *self, - display_name: "Nostr", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::Signal => ChannelDescriptor { - channel_type: *self, - display_name: "Signal", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::GatewayLoop, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: false, - supports_pairing: false, - supports_otp: true, - supports_reactions: false, - supports_location: false, - }, - }, - Self::Telephony => ChannelDescriptor { - channel_type: *self, - display_name: "Phone Call", - capabilities: ChannelCapabilities { - inbound_mode: InboundMode::Webhook, - supports_outbound: true, - supports_streaming: false, - supports_interactive: false, - supports_threads: false, - supports_voice_ingest: true, - supports_pairing: false, - supports_otp: false, - supports_reactions: false, - supports_location: false, - }, - }, - } - } -} - -// ── Channel capabilities ────────────────────────────────────────────────── - /// How a channel receives inbound messages. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize)] #[serde(rename_all = "snake_case")] diff --git a/crates/channels/src/plugin/channel_type.rs b/crates/channels/src/plugin/channel_type.rs new file mode 100644 index 000000000..e436d19a3 --- /dev/null +++ b/crates/channels/src/plugin/channel_type.rs @@ -0,0 +1,294 @@ +//! The channel type enum and its conversions. +//! +//! Split out of `plugin` to keep both files inside the file-size limit; +//! `ChannelType` and its parsing are a self-contained concern. + +use crate::{ + Error, + plugin::{ChannelCapabilities, ChannelDescriptor, InboundMode}, +}; + +/// Supported channel types. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +#[non_exhaustive] +pub enum ChannelType { + Telegram, + Whatsapp, + #[serde(rename = "msteams")] + MsTeams, + Discord, + Slack, + Matrix, + Nostr, + Signal, + Telephony, +} + +impl ChannelType { + /// Returns the channel type identifier as a string slice. + pub fn as_str(&self) -> &'static str { + match self { + Self::Telegram => "telegram", + Self::Whatsapp => "whatsapp", + Self::MsTeams => "msteams", + Self::Discord => "discord", + Self::Slack => "slack", + Self::Matrix => "matrix", + Self::Nostr => "nostr", + Self::Signal => "signal", + Self::Telephony => "telephony", + } + } + + /// Human-readable display name for UI labels. + pub fn display_name(&self) -> &'static str { + match self { + Self::Telegram => "Telegram", + Self::Whatsapp => "WhatsApp", + Self::MsTeams => "Microsoft Teams", + Self::Discord => "Discord", + Self::Slack => "Slack", + Self::Matrix => "Matrix", + Self::Nostr => "Nostr", + Self::Signal => "Signal", + Self::Telephony => "Phone Call", + } + } + + /// Best-effort chat classification for hook and prompt context. + #[must_use] + pub fn classify_chat(&self, chat_id: &str) -> Option { + match self { + Self::Telegram => { + if chat_id.starts_with("-100") { + Some("channel_or_supergroup".to_string()) + } else if chat_id.starts_with('-') { + Some("group".to_string()) + } else { + Some("private".to_string()) + } + }, + Self::Signal => { + if chat_id.starts_with("group:") { + Some("group".to_string()) + } else { + Some("direct".to_string()) + } + }, + Self::Nostr => Some("dm".to_string()), + Self::Telephony => Some("call".to_string()), + _ => None, + } + } + + /// Top-level config fields that must be treated as persisted secrets. + pub fn secret_fields(&self) -> &'static [&'static str] { + match self { + Self::Telegram => &["token"], + Self::Whatsapp => &[], + Self::MsTeams => &["app_password", "webhook_secret"], + Self::Discord => &["token"], + Self::Slack => &["bot_token", "app_token", "signing_secret"], + Self::Matrix => &["access_token", "password"], + Self::Nostr => &["secret_key"], + Self::Signal => &[], + Self::Telephony => &["auth_token"], + } + } +} + +impl std::fmt::Display for ChannelType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl std::str::FromStr for ChannelType { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s { + "telegram" => Ok(Self::Telegram), + "whatsapp" => Ok(Self::Whatsapp), + "msteams" | "microsoft_teams" | "microsoft-teams" | "teams" => Ok(Self::MsTeams), + "discord" => Ok(Self::Discord), + "slack" => Ok(Self::Slack), + "matrix" | "element" => Ok(Self::Matrix), + "nostr" => Ok(Self::Nostr), + "signal" => Ok(Self::Signal), + "telephony" | "phone" | "voice_call" | "voicecall" => Ok(Self::Telephony), + other => Err(Error::invalid_input(format!( + "unknown channel type: {other}" + ))), + } + } +} + +impl ChannelType { + /// All known channel types. + pub const ALL: &[ChannelType] = &[ + Self::Telegram, + Self::Whatsapp, + Self::MsTeams, + Self::Discord, + Self::Slack, + Self::Matrix, + Self::Nostr, + Self::Signal, + Self::Telephony, + ]; + + /// Returns the static descriptor for this channel type. + #[must_use] + pub fn descriptor(&self) -> ChannelDescriptor { + match self { + Self::Telegram => ChannelDescriptor { + channel_type: *self, + display_name: "Telegram", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Polling, + supports_outbound: true, + supports_streaming: true, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: true, + }, + }, + Self::Whatsapp => ChannelDescriptor { + channel_type: *self, + display_name: "WhatsApp", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: true, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::MsTeams => ChannelDescriptor { + channel_type: *self, + display_name: "Microsoft Teams", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Webhook, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: true, + }, + }, + Self::Discord => ChannelDescriptor { + channel_type: *self, + display_name: "Discord", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: false, + supports_reactions: false, + supports_location: true, + }, + }, + Self::Slack => ChannelDescriptor { + channel_type: *self, + display_name: "Slack", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::SocketMode, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: false, + }, + }, + Self::Matrix => ChannelDescriptor { + channel_type: *self, + display_name: "Matrix", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: true, + supports_interactive: true, + supports_threads: true, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: true, + supports_reactions: true, + supports_location: true, + }, + }, + Self::Nostr => ChannelDescriptor { + channel_type: *self, + display_name: "Nostr", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::Signal => ChannelDescriptor { + channel_type: *self, + display_name: "Signal", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::GatewayLoop, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: false, + supports_pairing: false, + supports_otp: true, + supports_reactions: false, + supports_location: false, + }, + }, + Self::Telephony => ChannelDescriptor { + channel_type: *self, + display_name: "Phone Call", + capabilities: ChannelCapabilities { + inbound_mode: InboundMode::Webhook, + supports_outbound: true, + supports_streaming: false, + supports_interactive: false, + supports_threads: false, + supports_voice_ingest: true, + supports_pairing: false, + supports_otp: false, + supports_reactions: false, + supports_location: false, + }, + }, + } + } +} + +// ── Channel capabilities ────────────────────────────────────────────────── diff --git a/crates/chat/src/channel_compaction.rs b/crates/chat/src/channel_compaction.rs new file mode 100644 index 000000000..2bba7a477 --- /dev/null +++ b/crates/chat/src/channel_compaction.rs @@ -0,0 +1,101 @@ +//! Channel notices for session compaction. +//! +//! Split from the main channel delivery module to keep both inside the +//! file-size limit; the compaction notice is a self-contained concern. + +use {serde_json::Value, std::sync::Arc, tracing::warn}; + +use crate::{compaction_run, runtime::ChatRuntime}; + +fn format_channel_compaction_notice( + outcome: &compaction_run::CompactionOutcome, + include_settings_hint: bool, +) -> String { + let mode_label = match outcome.effective_mode { + moltis_config::CompactionMode::Deterministic => "Deterministic", + moltis_config::CompactionMode::RecencyPreserving => "Recency preserving", + moltis_config::CompactionMode::Structured => "Structured", + moltis_config::CompactionMode::LlmReplace => "LLM replace", + }; + let total = outcome.total_tokens(); + let token_line = if total == 0 { + // Any strategy that made no LLM calls ends up here: Deterministic, + // RecencyPreserving, or a Structured run that fell back to + // recency_preserving before the LLM call landed. Report the + // actual effective mode so users don't see "deterministic + // strategy" when they picked recency_preserving. + format!( + "No LLM tokens used ({} strategy)", + mode_label.to_lowercase() + ) + } else { + format!( + "Used {total} tokens ({input} in + {output} out)", + total = total, + input = outcome.input_tokens, + output = outcome.output_tokens, + ) + }; + let body = format!( + "🧹 Conversation compacted\n\ + Mode: {mode_label}\n\ + {token_line}", + ); + if include_settings_hint { + format!("{body}\n{hint}", hint = compaction_run::SETTINGS_HINT) + } else { + body + } +} + +/// Send a silent "session compacted" notice to pending channel targets +/// without draining them. +/// +/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, +/// not drained, so the in-flight agent run can still deliver its final +/// reply to them afterward. Uses `send_text_silent` so the channel +/// integration doesn't count it toward user-visible interactive replies +/// (no TTS, no delivery receipts beyond the channel's own). +pub(crate) async fn notify_channels_of_compaction( + state: &Arc, + session_key: &str, + outcome: &compaction_run::CompactionOutcome, + include_settings_hint: bool, +) { + let targets = state.peek_channel_replies(session_key).await; + if targets.is_empty() { + return; + } + + let Some(outbound) = state.channel_outbound() else { + return; + }; + + let message = format_channel_compaction_notice(outcome, include_settings_hint); + let mut tasks = Vec::with_capacity(targets.len()); + for target in targets { + let outbound = Arc::clone(&outbound); + let message = message.clone(); + let to = target.outbound_to().into_owned(); + tasks.push(tokio::spawn(async move { + let reply_to = target.message_id.as_deref(); + if let Err(e) = outbound + .send_text_silent(&target.account_id, &to, &message, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send compaction notice to channel: {e}" + ); + } + })); + } + + for task in tasks { + if let Err(e) = task.await { + warn!(error = %e, "channel compaction notice task join failed"); + } + } +} diff --git a/crates/chat/src/channel_feedback.rs b/crates/chat/src/channel_feedback.rs new file mode 100644 index 000000000..d2c8a09fd --- /dev/null +++ b/crates/chat/src/channel_feedback.rs @@ -0,0 +1,70 @@ +//! Correlating delivered replies with the traces that produced them. +//! +//! Feedback attribution is recorded at send time because nothing else in the +//! system remembers which agent run wrote a given chat message, and by the +//! time someone reacts the turn is long finished. + +use std::sync::Arc; + +use crate::runtime::ChatRuntime; + +/// Link a web-delivered assistant message to the trace that produced it. +/// +/// The web is treated as one more channel so a thumb in the browser resolves +/// through the same correlation table as a Telegram reaction. The run id is +/// the message identity: unlike a message index it does not shift when older +/// history is compacted away. +pub(crate) async fn record_web_reply_trace( + state: &Arc, + session_key: &str, + run_id: &str, +) { + let Some(feedback) = state.feedback() else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + return; + }; + feedback + .record_reply( + moltis_channels::trace_link::WEB_CHANNEL, + moltis_channels::trace_link::WEB_CHANNEL, + session_key, + std::slice::from_ref(&run_id.to_string()), + &trace_id, + Some(session_key), + ) + .await; +} + +/// Link the messages a reply produced to the trace that generated it. +/// +/// Best-effort by design: a missing link costs feedback attribution for one +/// reply, and the user already has the message. +pub(crate) async fn record_reply_trace( + state: &Arc, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + message_ids: &[String], + session_key: &str, +) { + if message_ids.is_empty() { + return; + } + let Some(feedback) = state.feedback() else { + return; + }; + let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + return; + }; + feedback + .record_reply( + target.channel_type.as_str(), + &target.account_id, + to, + message_ids, + &trace_id, + Some(session_key), + ) + .await; +} diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 5b1cc0b76..e60801eec 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -255,99 +255,6 @@ fn format_channel_error_message(error_obj: &Value) -> String { /// `chat.compaction.show_settings_hint = false` don't see the repetitive /// hint on every compaction. Mode + token lines are always included. /// The LLM retry path never sees this text regardless. -fn format_channel_compaction_notice( - outcome: &compaction_run::CompactionOutcome, - include_settings_hint: bool, -) -> String { - let mode_label = match outcome.effective_mode { - moltis_config::CompactionMode::Deterministic => "Deterministic", - moltis_config::CompactionMode::RecencyPreserving => "Recency preserving", - moltis_config::CompactionMode::Structured => "Structured", - moltis_config::CompactionMode::LlmReplace => "LLM replace", - }; - let total = outcome.total_tokens(); - let token_line = if total == 0 { - // Any strategy that made no LLM calls ends up here: Deterministic, - // RecencyPreserving, or a Structured run that fell back to - // recency_preserving before the LLM call landed. Report the - // actual effective mode so users don't see "deterministic - // strategy" when they picked recency_preserving. - format!( - "No LLM tokens used ({} strategy)", - mode_label.to_lowercase() - ) - } else { - format!( - "Used {total} tokens ({input} in + {output} out)", - total = total, - input = outcome.input_tokens, - output = outcome.output_tokens, - ) - }; - let body = format!( - "🧹 Conversation compacted\n\ - Mode: {mode_label}\n\ - {token_line}", - ); - if include_settings_hint { - format!("{body}\n{hint}", hint = compaction_run::SETTINGS_HINT) - } else { - body - } -} - -/// Send a silent "session compacted" notice to pending channel targets -/// without draining them. -/// -/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, -/// not drained, so the in-flight agent run can still deliver its final -/// reply to them afterward. Uses `send_text_silent` so the channel -/// integration doesn't count it toward user-visible interactive replies -/// (no TTS, no delivery receipts beyond the channel's own). -pub(crate) async fn notify_channels_of_compaction( - state: &Arc, - session_key: &str, - outcome: &compaction_run::CompactionOutcome, - include_settings_hint: bool, -) { - let targets = state.peek_channel_replies(session_key).await; - if targets.is_empty() { - return; - } - - let Some(outbound) = state.channel_outbound() else { - return; - }; - - let message = format_channel_compaction_notice(outcome, include_settings_hint); - let mut tasks = Vec::with_capacity(targets.len()); - for target in targets { - let outbound = Arc::clone(&outbound); - let message = message.clone(); - let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { - let reply_to = target.message_id.as_deref(); - if let Err(e) = outbound - .send_text_silent(&target.account_id, &to, &message, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send compaction notice to channel: {e}" - ); - } - })); - } - - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel compaction notice task join failed"); - } - } -} - /// Send a short retry status update to pending channel targets without draining /// them. The final reply (or terminal error) will still use the same targets. pub(crate) async fn send_retry_status_to_channels( @@ -455,67 +362,6 @@ pub(crate) async fn deliver_channel_error( } } -/// Link a web-delivered assistant message to the trace that produced it. -/// -/// The web is treated as one more channel so a thumb in the browser resolves -/// through the same correlation table as a Telegram reaction. The run id is -/// the message identity: unlike a message index it does not shift when older -/// history is compacted away. -pub(crate) async fn record_web_reply_trace( - state: &Arc, - session_key: &str, - run_id: &str, -) { - let Some(feedback) = state.feedback() else { - return; - }; - let Some(trace_id) = moltis_observability::recent_trace(session_key) else { - return; - }; - feedback - .record_reply( - moltis_channels::trace_link::WEB_CHANNEL, - moltis_channels::trace_link::WEB_CHANNEL, - session_key, - std::slice::from_ref(&run_id.to_string()), - &trace_id, - Some(session_key), - ) - .await; -} - -/// Link the messages a reply produced to the trace that generated it. -/// -/// Best-effort by design: a missing link costs feedback attribution for one -/// reply, and the user already has the message. -async fn record_reply_trace( - state: &Arc, - target: &moltis_channels::ChannelReplyTarget, - to: &str, - message_ids: &[String], - session_key: &str, -) { - if message_ids.is_empty() { - return; - } - let Some(feedback) = state.feedback() else { - return; - }; - let Some(trace_id) = moltis_observability::recent_trace(session_key) else { - return; - }; - feedback - .record_reply( - target.channel_type.as_str(), - &target.account_id, - to, - message_ids, - &trace_id, - Some(session_key), - ) - .await; -} - async fn deliver_channel_replies_to_targets( outbound: Arc, targets: Vec, @@ -682,7 +528,7 @@ async fn deliver_channel_replies_to_targets( .await { Ok(message_ids) => { - record_reply_trace( + crate::channel_feedback::record_reply_trace( &state, &target, &to, diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index 63bcd570b..2d573bf32 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,4 +1,6 @@ mod agent_loop; +mod channel_compaction; +mod channel_feedback; mod channels; mod compaction; mod compaction_run; diff --git a/crates/chat/src/run_with_tools.rs b/crates/chat/src/run_with_tools.rs index 22be1d4b3..a668a719a 100644 --- a/crates/chat/src/run_with_tools.rs +++ b/crates/chat/src/run_with_tools.rs @@ -36,11 +36,12 @@ use crate::{ ChannelStreamDispatcher, clear_unsupported_model, compact_session, mark_unsupported_model, ordered_runner_event_callback, }, + channel_compaction::notify_channels_of_compaction, channels::{ deliver_channel_error, deliver_channel_replies, dispatch_document_to_channels, document_payload_from_data_uri, document_payload_from_ref, generate_tts_audio, - notify_channels_of_compaction, send_location_to_channels, send_retry_status_to_channels, - send_screenshot_to_channels, send_tool_result_to_channels, send_tool_status_to_channels, + send_location_to_channels, send_retry_status_to_channels, send_screenshot_to_channels, + send_tool_result_to_channels, send_tool_status_to_channels, }, chat_error::parse_chat_error, memory_tools::{effective_tool_mode, install_agent_scoped_memory_tools}, diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index cc1936a70..d24d890c6 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -32,7 +32,7 @@ use { use crate::{ agent_loop::effective_tool_mode, - channels::notify_channels_of_compaction, + channel_compaction::notify_channels_of_compaction, compaction_run, memory_tools::AgentScopedMemoryWriter, message::{ @@ -303,7 +303,8 @@ impl ChatService for LiveChatService { { warn!("send_sync: failed to persist assistant message: {e}"); } - crate::channels::record_web_reply_trace(&self.state, &session_key, &run_id).await; + crate::channel_feedback::record_web_reply_trace(&self.state, &session_key, &run_id) + .await; // Update metadata message count. if let Ok(count) = self.session_store.count(&session_key).await { self.session_metadata.touch(&session_key, count).await; diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index 5fee0c671..02975263f 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -1286,7 +1286,7 @@ impl LiveChatService { { warn!("failed to persist assistant message: {e}"); } - crate::channels::record_web_reply_trace( + crate::channel_feedback::record_web_reply_trace( &state_for_drain, &session_key_clone, &run_id_clone, diff --git a/crates/discord/src/handler.rs b/crates/discord/src/handler.rs index 25dcf5c43..94a6d7345 100644 --- a/crates/discord/src/handler.rs +++ b/crates/discord/src/handler.rs @@ -2,5 +2,7 @@ #[path = "handler/implementation.rs"] mod implementation; +#[path = "handler/reactions.rs"] +mod reactions; pub use self::implementation::*; diff --git a/crates/discord/src/handler/implementation.rs b/crates/discord/src/handler/implementation.rs index 3c59abec4..debae7d00 100644 --- a/crates/discord/src/handler/implementation.rs +++ b/crates/discord/src/handler/implementation.rs @@ -63,58 +63,6 @@ impl Handler { downloader: DiscordInboundMediaDownloader::new(), } } - - /// Surface a reaction change so the gateway can score it as feedback. - /// - /// The bot's own acknowledgement reactions are skipped: it marks incoming - /// messages with 👀/✅/❌, and treating those as user opinions would score - /// every turn the bot itself handled. - async fn emit_reaction_change( - &self, - ctx: &Context, - reaction: &serenity::all::Reaction, - added: bool, - ) { - let Some(user_id) = reaction.user_id else { - return; - }; - if user_id == ctx.cache.current_user().id { - return; - } - - let sink = { - let accts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); - match accts.get(&self.account_id) { - Some(state) => state.event_sink.clone(), - None => return, - } - }; - let Some(sink) = sink else { - return; - }; - - // Custom guild emoji have no unicode form; their name is what a - // feedback vocabulary can match against. - let emoji = match &reaction.emoji { - ReactionType::Unicode(raw) => raw.clone(), - ReactionType::Custom { name, .. } => name.clone().unwrap_or_default(), - _ => return, - }; - if emoji.is_empty() { - return; - } - - sink.emit(ChannelEvent::ReactionChange { - channel_type: ChannelType::Discord, - account_id: self.account_id.clone(), - chat_id: reaction.channel_id.to_string(), - message_id: reaction.message_id.to_string(), - user_id: user_id.to_string(), - emoji, - added, - }) - .await; - } } fn unix_now() -> i64 { diff --git a/crates/discord/src/handler/reactions.rs b/crates/discord/src/handler/reactions.rs new file mode 100644 index 000000000..5743d041f --- /dev/null +++ b/crates/discord/src/handler/reactions.rs @@ -0,0 +1,63 @@ +//! Inbound Discord reaction handling. +//! +//! Split from the main handler so both files stay inside the file-size limit. + +use serenity::all::{Context, Reaction, ReactionType}; + +use moltis_channels::{ChannelEvent, ChannelType}; + +use super::implementation::Handler; + +impl Handler { + /// Surface a reaction change so the gateway can score it as feedback. + /// + /// The bot's own acknowledgement reactions are skipped: it marks incoming + /// messages with 👀/✅/❌, and treating those as user opinions would score + /// every turn the bot itself handled. + pub(super) async fn emit_reaction_change( + &self, + ctx: &Context, + reaction: &Reaction, + added: bool, + ) { + let Some(user_id) = reaction.user_id else { + return; + }; + if user_id == ctx.cache.current_user().id { + return; + } + + let sink = { + let accts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); + match accts.get(&self.account_id) { + Some(state) => state.event_sink.clone(), + None => return, + } + }; + let Some(sink) = sink else { + return; + }; + + // Custom guild emoji have no unicode form; their name is what a + // feedback vocabulary can match against. + let emoji = match &reaction.emoji { + ReactionType::Unicode(raw) => raw.clone(), + ReactionType::Custom { name, .. } => name.clone().unwrap_or_default(), + _ => return, + }; + if emoji.is_empty() { + return; + } + + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Discord, + account_id: self.account_id.clone(), + chat_id: reaction.channel_id.to_string(), + message_id: reaction.message_id.to_string(), + user_id: user_id.to_string(), + emoji, + added, + }) + .await; + } +} diff --git a/crates/gateway/src/server/prepare_core.rs b/crates/gateway/src/server/prepare_core.rs index 69e965fb8..be5087846 100644 --- a/crates/gateway/src/server/prepare_core.rs +++ b/crates/gateway/src/server/prepare_core.rs @@ -34,6 +34,7 @@ use { std::{path::PathBuf, sync::Arc}, tracing::{debug, info, warn}, }; +mod feedback; mod log_persistence; mod post_state; mod sandbox; diff --git a/crates/gateway/src/server/prepare_core/feedback.rs b/crates/gateway/src/server/prepare_core/feedback.rs new file mode 100644 index 000000000..2c15e3790 --- /dev/null +++ b/crates/gateway/src/server/prepare_core/feedback.rs @@ -0,0 +1,36 @@ +//! Installing the reaction-feedback service during startup. +//! +//! Split out of `post_state` so that file stays inside the size limit; the +//! wiring is self-contained. + +use std::sync::Arc; + +use sqlx::SqlitePool; + +use crate::state::GatewayState; + +/// Install reply/trace correlation and start the retention prune. +/// +/// Needs the database pool, which does not exist at state construction, so it +/// runs here rather than in `GatewayState::new`. +pub(super) fn install_feedback(state: &Arc, db_pool: &SqlitePool) { + let links: Arc = Arc::new( + crate::trace_link_store::SqliteTraceLinkStore::new(db_pool.clone()), + ); + state.feedback.apply( + links, + &state.config.instrumentation.feedback, + Some(state.config.instrumentation.environment.clone()), + ); + + // Links accumulate one row per delivered reply; drop the ones too old + // to attribute a reaction to. + let feedback = Arc::clone(&state.feedback); + let retention_days = state.config.instrumentation.feedback.link_retention_days; + tokio::spawn(async move { + let removed = feedback.prune(retention_days).await; + if removed > 0 { + tracing::debug!(removed, "pruned expired trace links"); + } + }); +} diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index fbf033429..9a08028a5 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -1,6 +1,9 @@ -use std::{ - path::PathBuf, - sync::{Arc, atomic::Ordering}, +use { + super::feedback::install_feedback, + std::{ + path::PathBuf, + sync::{Arc, atomic::Ordering}, + }, }; use { @@ -425,30 +428,7 @@ pub(super) async fn complete_startup( } } - // ── Reaction feedback ──────────────────────────────────────────────── - // Needs the database pool for reply/trace correlation, so it is installed - // here rather than at state construction. - { - let links: Arc = Arc::new( - crate::trace_link_store::SqliteTraceLinkStore::new(db_pool.clone()), - ); - state.feedback.apply( - links, - &state.config.instrumentation.feedback, - Some(state.config.instrumentation.environment.clone()), - ); - - // Links accumulate one row per delivered reply; drop the ones too old - // to attribute a reaction to. - let feedback = Arc::clone(&state.feedback); - let retention_days = state.config.instrumentation.feedback.link_retention_days; - tokio::spawn(async move { - let removed = feedback.prune(retention_days).await; - if removed > 0 { - tracing::debug!(removed, "pruned expired trace links"); - } - }); - } + install_feedback(&state, &db_pool); { let (webhook_tx, webhook_rx) = tokio::sync::mpsc::channel::(256); diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md index ed8415dea..2dabb9a36 100644 --- a/docs/src/instrumentation.md +++ b/docs/src/instrumentation.md @@ -249,6 +249,75 @@ and shown in the settings UI — if you see them, raise `queue_capacity` or lowe | `content` | `"metadata_only"` | | | `timeout_secs` | `10` | | +### `[instrumentation.feedback]` + +| Key | Default | Description | +| --- | --- | --- | +| `enabled` | `true` | Collect reaction feedback. Gated by the master switch. | +| `positive` | `[]` | Reaction tokens counted as approval. Empty uses the built-in list. | +| `negative` | `[]` | Reaction tokens counted as disapproval. Empty uses the built-in list. | +| `link_retention_days` | `30` | How long a reply stays attributable to its turn. | + +## User feedback + +A thumbs up or down on a reply becomes a `user-feedback` score on the trace +that produced it — `1.0` for positive, `0.0` for negative — visible in +Langfuse alongside the conversation. + +Feedback works in three places: + +- **Telegram, Discord and Slack** — react to any message the bot sent. +- **The web UI** — thumbs appear in the action bar under each assistant + message, but only while instrumentation is active and feedback is enabled. + A control that goes nowhere is worse than no control. + +Reactions accept raw emoji or shortcodes, and skin-tone and presentation +selectors are ignored when matching, so `👍`, `👍🏾`, `:+1:` and `thumbsup` are +one signal. The default vocabulary is deliberately narrow; a default that +swallowed every positive-looking emoji would turn release parties into quality +data. Override either list to change it: + +```toml +[instrumentation.feedback] +positive = ["👍", "🎉", "ship-it"] +``` + +Setting one list leaves the other on its defaults. + +**Changing your mind works.** Score ids are derived from the trace and the +reacting user, and Langfuse upserts on that id, so switching from 👍 to 👎 +replaces your vote instead of recording both. Removing the reaction retracts +the score. Two people reacting to the same reply still count separately. + +### Why a reaction sometimes does nothing + +- **The reply is older than `link_retention_days`.** Attribution comes from a + correlation table written when the reply is sent, and it is pruned on that + schedule. The web UI says so explicitly rather than failing silently. +- **The reply predates instrumentation being switched on.** There was no trace + to attribute it to. +- **The channel cannot report the ids of messages it sends.** Telegram, Discord + and Slack can; other channels deliver the message but lose feedback + attribution. + +## Cost + +Cost is computed in-process from a built-in price table, so it is available +whether or not any backend is configured, and every backend sees the same +number instead of each deriving its own. + +Cache reads and writes are priced on their own tiers rather than folded into +input — a cache read can be an order of magnitude cheaper, and summing them +would overstate spend on exactly the workloads caching exists to help. + +Model ids are matched by longest prefix after stripping any provider prefix, +so `anthropic/claude-opus-4` and a dated snapshot like +`claude-opus-4-5-20260101` both price as their family. + +**A model with no entry reports no cost at all** rather than falling back to +an average. Prices go stale whenever a provider changes them, and a confident +wrong number that looks authoritative is worse than a visible gap. + ## How it works All three backends are fed over **OTLP/HTTP with a JSON payload**. From cd598d0263a77cfc4623a3964850f6d054c34e9f Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 00:55:23 -0400 Subject: [PATCH 17/33] fix(feedback): close the gaps that made reaction scoring unreachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four defects, all of which made the feedback path either forgeable or inert on the delivery routes people actually use. `feedback.submit` was registered in READ_METHODS while writing a score, so a client holding only `operator.read` could record or retract votes. It also took the scoring identity from the request, and since a score id is UUIDv5 over (trace, name, user), a caller-chosen `userId` could overwrite or retract someone else's vote. It is now a write method and attributes to the authenticated web operator; the UI never sent the field, so nothing on the wire changes. `RegistryOutboundRouter` never overrode `send_text_reporting_ids`, and every channel reaches the outbound trait through that router. It inherited the trait default, which reports no ids — so no channel ever produced a trace link and every reaction resolved as `UnknownMessage`. The router now delegates all three id-reporting sends. Trace links were stored under `outbound_to()`, which carries a `:thread` suffix for a Telegram forum topic, while a `message_reaction` update reports only the bare chat id. Links are now keyed on `chat_id`, which is what both sides can agree on. Two delivery paths never recorded a link at all: edit-in-place streaming (which delivers the reply itself, so the normal send path is skipped) and any reply carrying an activity logbook. Both now go through id-reporting sends — `send_stream_reporting_ids` and `send_text_with_suffix_reporting_ids`, additive with defaults so the channels that cannot report ids are untouched — and record the link. The reply fan-out moved into `channel_reply_delivery.rs`, both to stay under the file-size limit and so the "every path records a link" rule is checkable by reading one file. Also drops two imports left dead by the earlier file split, which the workspace `-D warnings` gate would have rejected. --- crates/channels/src/plugin.rs | 40 ++ crates/channels/src/registry.rs | 134 ++++++ crates/chat/src/agent_loop.rs | 124 ++++- crates/chat/src/channel_compaction.rs | 2 +- crates/chat/src/channel_feedback.rs | 110 ++++- crates/chat/src/channel_reply_delivery.rs | 443 ++++++++++++++++++ crates/chat/src/channels.rs | 276 +---------- crates/chat/src/lib.rs | 1 + crates/discord/src/outbound.rs | 81 +++- crates/gateway/src/methods/dispatch.rs | 19 +- .../gateway/src/methods/services/feedback.rs | 16 +- crates/slack/src/outbound.rs | 148 ++++-- crates/telegram/src/outbound/send.rs | 215 +++++---- crates/telegram/src/outbound/stream.rs | 81 +++- 14 files changed, 1238 insertions(+), 452 deletions(-) create mode 100644 crates/chat/src/channel_reply_delivery.rs diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index d165cd4cf..487d12851 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -665,6 +665,27 @@ pub trait ChannelOutbound: Send + Sync { let _ = suffix_html; self.send_text(account_id, to, text, reply_to).await } + + /// [`Self::send_text_with_suffix`], reporting the ids of the messages it + /// produced. + /// + /// A reply that carries an activity logbook is still a reply someone can + /// react to, so it needs the same attribution as a plain one. Kept as a + /// separate method for the same reason as + /// [`Self::send_text_reporting_ids`]: channels that cannot report ids + /// inherit the default and lose attribution, not the message. + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text_with_suffix(account_id, to, text, suffix_html, reply_to) + .await?; + Ok(Vec::new()) + } /// Send pre-formatted HTML without markdown conversion. /// /// Used for content that is already valid Telegram HTML (e.g. the activity @@ -798,6 +819,25 @@ pub trait ChannelStreamOutbound: Send + Sync { stream: StreamReceiver, ) -> Result<()>; + /// [`Self::send_stream`], reporting the ids of the messages it left behind. + /// + /// Edit-in-place streaming delivers the final reply itself, so the normal + /// send path never runs and never records a trace link. Without this the + /// message a reader actually reacts to would have no attribution at all. + /// + /// Channels that cannot report ids inherit the default and lose feedback + /// attribution for streamed replies, not the reply. + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(Vec::new()) + } + /// Whether streaming is enabled for this account. async fn is_stream_enabled(&self, _account_id: &str) -> bool { true diff --git a/crates/channels/src/registry.rs b/crates/channels/src/registry.rs index 0fc8d5a69..978038fd8 100644 --- a/crates/channels/src/registry.rs +++ b/crates/channels/src/registry.rs @@ -333,6 +333,26 @@ impl ChannelOutbound for RegistryOutboundRouter { outbound.send_text(account_id, to, text, reply_to).await } + /// Delegate rather than inherit the trait default: the default reports no + /// ids, which would silently strip attribution from every channel behind + /// this facade even though the concrete outbound can report them. + async fn send_text_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_text_reporting_ids(account_id, to, text, reply_to) + .await + } + async fn send_media( &self, account_id: &str, @@ -375,6 +395,24 @@ impl ChannelOutbound for RegistryOutboundRouter { .await } + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_text_with_suffix_reporting_ids(account_id, to, text, suffix_html, reply_to) + .await + } + async fn send_html( &self, account_id: &str, @@ -497,6 +535,23 @@ impl ChannelStreamOutbound for RegistryOutboundRouter { .await } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + let stream_out = self + .registry + .resolve_stream(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + stream_out + .send_stream_reporting_ids(account_id, to, reply_to, stream) + .await + } + async fn is_stream_enabled(&self, account_id: &str) -> bool { let Some(stream_out) = self.registry.resolve_stream(account_id).await else { return false; @@ -692,6 +747,29 @@ mod tests { ) -> Result<()> { Ok(()) } + + // Sentinel ids: the trait defaults report none, so a router that + // inherits them instead of delegating produces an empty list. + async fn send_text_reporting_ids( + &self, + _: &str, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result> { + Ok(vec!["text-1".into()]) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + _: &str, + _: &str, + _: &str, + _: &str, + _: Option<&str>, + ) -> Result> { + Ok(vec!["suffix-1".into()]) + } } struct NullStreamOutbound { @@ -716,6 +794,17 @@ mod tests { Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(vec!["stream-1".into()]) + } + async fn streams_final_replies(&self, _: &str) -> bool { self.streams_final_replies } @@ -866,6 +955,51 @@ mod tests { assert!(result.is_ok()); } + /// The id-reporting sends must be delegated, not inherited. + /// + /// Every channel reaches the outbound trait through this router, so a + /// router that fell back to the trait defaults would report no ids for any + /// channel and silently strip feedback attribution from the whole system. + #[tokio::test] + async fn outbound_router_delegates_id_reporting_sends() { + let mut registry = ChannelRegistry::new(); + registry + .register(Arc::new(RwLock::new(TestPlugin::new("telegram")))) + .await; + registry + .start_account("telegram", "bot1", serde_json::json!({})) + .await + .unwrap(); + + let router = RegistryOutboundRouter::new(Arc::new(registry)); + + assert_eq!( + router + .send_text_reporting_ids("bot1", "42", "hello", None) + .await + .unwrap(), + vec!["text-1".to_string()] + ); + assert_eq!( + router + .send_text_with_suffix_reporting_ids("bot1", "42", "hello", "log", None) + .await + .unwrap(), + vec!["suffix-1".to_string()] + ); + + let (tx, rx) = mpsc::channel(8); + tx.send(StreamEvent::Done).await.unwrap(); + drop(tx); + assert_eq!( + router + .send_stream_reporting_ids("bot1", "42", None, rx) + .await + .unwrap(), + vec!["stream-1".to_string()] + ); + } + #[tokio::test] async fn outbound_router_unknown_account_errors() { let registry = Arc::new(ChannelRegistry::new()); diff --git a/crates/chat/src/agent_loop.rs b/crates/chat/src/agent_loop.rs index 0f62e43a2..dd0a75453 100644 --- a/crates/chat/src/agent_loop.rs +++ b/crates/chat/src/agent_loop.rs @@ -163,6 +163,12 @@ pub(crate) struct ChannelStreamDispatcher { workers: Vec, tasks: Vec>, completed: Arc>>, + /// Ids of the messages each stream left behind, so a reaction on a + /// streamed reply is attributable. The normal send path never runs for + /// these targets, so this is the only place the link can be recorded. + delivered: Arc)>>>, + feedback: Option>, + session_key: String, started: bool, sent_final_delta: bool, } @@ -187,6 +193,9 @@ impl ChannelStreamDispatcher { workers: Vec::new(), tasks: Vec::new(), completed: Arc::new(Mutex::new(HashSet::new())), + delivered: Arc::new(Mutex::new(Vec::new())), + feedback: state.feedback(), + session_key: session_key.to_string(), started: false, sent_final_delta: false, }; @@ -222,10 +231,12 @@ impl ChannelStreamDispatcher { let (tx, rx) = mpsc::channel(CHANNEL_STREAM_BUFFER_SIZE); let outbound = Arc::clone(&self.outbound); let completed = Arc::clone(&self.completed); + let delivered = Arc::clone(&self.delivered); let account_id = target.account_id.clone(); let to = target.outbound_to().into_owned(); let reply_to = target.message_id.clone(); let key_for_insert = key.clone(); + let target_for_link = target.clone(); let account_for_log = account_id.clone(); let chat_for_log = target.chat_id.clone(); let thread_for_log = target.thread_id.clone(); @@ -236,13 +247,16 @@ impl ChannelStreamDispatcher { }); self.tasks.push(tokio::spawn(async move { match outbound - .send_stream(&account_id, &to, reply_to.as_deref(), rx) + .send_stream_reporting_ids(&account_id, &to, reply_to.as_deref(), rx) .await { - Ok(()) => { + Ok(message_ids) => { if streams_final_replies { completed.lock().await.insert(key_for_insert); } + if !message_ids.is_empty() { + delivered.lock().await.push((target_for_link, message_ids)); + } }, Err(e) => { warn!( @@ -294,6 +308,24 @@ impl ChannelStreamDispatcher { pub(crate) async fn finish(&mut self) { self.send_terminal(moltis_channels::StreamEvent::Done).await; self.join_workers().await; + self.record_delivered_trace_links().await; + } + + /// Attribute the streamed messages to the trace that produced them. + /// + /// Runs after the workers are joined so every id is in hand, and before + /// the caller decides which targets to skip on the normal send path. + async fn record_delivered_trace_links(&self) { + let delivered = std::mem::take(&mut *self.delivered.lock().await); + for (target, message_ids) in delivered { + crate::channel_feedback::record_reply_trace( + self.feedback.as_deref(), + &target, + &message_ids, + &self.session_key, + ) + .await; + } } async fn send_terminal(&mut self, event: moltis_channels::StreamEvent) { @@ -627,6 +659,19 @@ mod tests { Ok(()) } + /// Report an id the way a real edit-in-place channel does, so the + /// dispatcher's trace-link recording is exercised rather than skipped. + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: moltis_channels::StreamReceiver, + ) -> moltis_channels::Result> { + self.send_stream(account_id, to, reply_to, stream).await?; + Ok(vec!["streamed-1".into()]) + } + async fn is_stream_enabled(&self, _account_id: &str) -> bool { true } @@ -658,6 +703,9 @@ mod tests { workers: Vec::new(), tasks: Vec::new(), completed: Arc::new(Mutex::new(HashSet::new())), + delivered: Arc::new(Mutex::new(Vec::new())), + feedback: None, + session_key: "session".into(), started: false, sent_final_delta: false, } @@ -694,6 +742,78 @@ mod tests { assert!(dispatcher.completed_target_keys().await.is_empty()); } + /// Edit-in-place streaming delivers the reply itself, so the normal send + /// path never runs for that target. If the dispatcher did not record the + /// link, a reaction on a streamed reply would resolve as `UnknownMessage`. + #[tokio::test] + async fn streamed_replies_get_a_trace_link() { + use { + async_trait::async_trait as link_async_trait, + moltis_channels::trace_link::{TraceLink, TraceLinkStore}, + }; + + #[derive(Default)] + struct RecordingLinks { + links: std::sync::Mutex>, + } + + #[link_async_trait] + impl TraceLinkStore for RecordingLinks { + async fn link(&self, link: TraceLink) -> moltis_channels::Result<()> { + self.links + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(link); + Ok(()) + } + + async fn lookup( + &self, + _channel_type: &str, + _account_id: &str, + _chat_id: &str, + _message_id: &str, + ) -> moltis_channels::Result> { + Ok(None) + } + + async fn prune(&self, _cutoff: i64) -> moltis_channels::Result { + Ok(0) + } + } + + let session_key = "streamed-link-test-session"; + let links = Arc::new(RecordingLinks::default()); + let feedback = Arc::new(moltis_channels::FeedbackService::default()); + feedback.apply( + Arc::clone(&links) as Arc, + &moltis_config::FeedbackSettings::default(), + None, + ); + moltis_observability::remember_trace( + session_key, + &moltis_observability::TraceId("trace-streamed".into()), + ); + + let outbound = Arc::new(RecordingStreamOutbound { + streams_final_replies: true, + receives_progress_deltas: false, + events: Arc::new(Mutex::new(Vec::new())), + }); + let mut dispatcher = dispatcher_with(outbound); + dispatcher.feedback = Some(feedback); + dispatcher.session_key = session_key.to_string(); + + dispatcher.send_delta("final").await; + dispatcher.finish().await; + + let recorded = links.links.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].message_id, "streamed-1"); + assert_eq!(recorded[0].chat_id, "chat"); + assert_eq!(recorded[0].trace_id, "trace-streamed"); + } + #[tokio::test] async fn final_stream_workers_receive_progress_and_final_deltas_and_complete_targets() { let events = Arc::new(Mutex::new(Vec::new())); diff --git a/crates/chat/src/channel_compaction.rs b/crates/chat/src/channel_compaction.rs index 2bba7a477..530b1921a 100644 --- a/crates/chat/src/channel_compaction.rs +++ b/crates/chat/src/channel_compaction.rs @@ -3,7 +3,7 @@ //! Split from the main channel delivery module to keep both inside the //! file-size limit; the compaction notice is a self-contained concern. -use {serde_json::Value, std::sync::Arc, tracing::warn}; +use {std::sync::Arc, tracing::warn}; use crate::{compaction_run, runtime::ChatRuntime}; diff --git a/crates/chat/src/channel_feedback.rs b/crates/chat/src/channel_feedback.rs index d2c8a09fd..cc4617b2d 100644 --- a/crates/chat/src/channel_feedback.rs +++ b/crates/chat/src/channel_feedback.rs @@ -39,19 +39,25 @@ pub(crate) async fn record_web_reply_trace( /// Link the messages a reply produced to the trace that generated it. /// +/// Keyed on [`ChannelReplyTarget::chat_id`] rather than the outbound address: +/// for a Telegram forum topic the outbound address carries a `:thread` suffix, +/// but a `message_reaction` update reports only the bare chat id, so a link +/// stored under the suffixed form is one no reaction could ever find. +/// /// Best-effort by design: a missing link costs feedback attribution for one /// reply, and the user already has the message. +/// +/// [`ChannelReplyTarget::chat_id`]: moltis_channels::ChannelReplyTarget::chat_id pub(crate) async fn record_reply_trace( - state: &Arc, + feedback: Option<&moltis_channels::FeedbackService>, target: &moltis_channels::ChannelReplyTarget, - to: &str, message_ids: &[String], session_key: &str, ) { if message_ids.is_empty() { return; } - let Some(feedback) = state.feedback() else { + let Some(feedback) = feedback else { return; }; let Some(trace_id) = moltis_observability::recent_trace(session_key) else { @@ -61,10 +67,106 @@ pub(crate) async fn record_reply_trace( .record_reply( target.channel_type.as_str(), &target.account_id, - to, + &target.chat_id, message_ids, &trace_id, Some(session_key), ) .await; } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use std::sync::Mutex; + + use { + async_trait::async_trait, + moltis_channels::trace_link::{TraceLink, TraceLinkStore}, + moltis_config::FeedbackSettings, + moltis_observability::TraceId, + }; + + use super::*; + + #[derive(Default)] + struct RecordingLinks { + links: Mutex>, + } + + #[async_trait] + impl TraceLinkStore for RecordingLinks { + async fn link(&self, link: TraceLink) -> moltis_channels::Result<()> { + self.links.lock().unwrap().push(link); + Ok(()) + } + + async fn lookup( + &self, + _channel_type: &str, + _account_id: &str, + _chat_id: &str, + _message_id: &str, + ) -> moltis_channels::Result> { + Ok(None) + } + + async fn prune(&self, _cutoff: i64) -> moltis_channels::Result { + Ok(0) + } + } + + fn telegram_topic_target() -> moltis_channels::ChannelReplyTarget { + moltis_channels::ChannelReplyTarget { + ack_message_id: None, + channel_type: moltis_channels::ChannelType::Telegram, + account_id: "bot1".into(), + chat_id: "-1001234".into(), + message_id: Some("77".into()), + thread_id: Some("42".into()), + } + } + + /// A reply into a Telegram forum topic goes out to `chat:thread`, but the + /// `message_reaction` update that follows reports only `chat`. Storing the + /// composite address would make every topic reaction unresolvable. + #[tokio::test] + async fn topic_replies_are_keyed_on_the_bare_chat_id() { + let session_key = "topic-key-test-session"; + let links = Arc::new(RecordingLinks::default()); + let feedback = moltis_channels::FeedbackService::default(); + feedback.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings::default(), + None, + ); + moltis_observability::remember_trace(session_key, &TraceId("trace-abc".into())); + + let target = telegram_topic_target(); + assert_eq!(target.outbound_to().as_ref(), "-1001234:42"); + + record_reply_trace( + Some(&feedback), + &target, + &["901".to_string(), "902".to_string()], + session_key, + ) + .await; + + let recorded = links.links.lock().unwrap(); + assert_eq!(recorded.len(), 2); + for link in recorded.iter() { + assert_eq!(link.chat_id, "-1001234"); + assert_eq!(link.account_id, "bot1"); + assert_eq!(link.channel_type, "telegram"); + assert_eq!(link.trace_id, "trace-abc"); + } + assert_eq!(recorded[0].message_id, "901"); + assert_eq!(recorded[1].message_id, "902"); + } + + #[tokio::test] + async fn nothing_is_recorded_without_a_feedback_service() { + record_reply_trace(None, &telegram_topic_target(), &["1".to_string()], "none").await; + } +} diff --git a/crates/chat/src/channel_reply_delivery.rs b/crates/chat/src/channel_reply_delivery.rs new file mode 100644 index 000000000..00f47fafc --- /dev/null +++ b/crates/chat/src/channel_reply_delivery.rs @@ -0,0 +1,443 @@ +//! Fanning a finished reply out to the channels that asked for it. +//! +//! Split from the main channel module to stay inside the file-size limit; the +//! reply fan-out is a self-contained concern, and keeping it together is what +//! makes the "every delivery path records a trace link" rule checkable by +//! reading one file. + +use std::{collections::HashSet, sync::Arc}; + +use tracing::warn; + +use crate::{ + agent_loop::ChannelReplyTargetKey, + channels::{build_tts_payload, format_logbook_html}, + runtime::ChatRuntime, + types::ReplyMedium, +}; + +/// Send the reply text (optionally carrying an activity logbook) and link the +/// messages it produced to the trace that wrote them. +/// +/// One helper for both the Telegram and generic branches so a delivery path +/// cannot quietly lose feedback attribution by picking the non-reporting send: +/// the reporting variants are the only ones reachable from here. Channels that +/// cannot report ids return an empty list and lose attribution, not the reply. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn deliver_text_reply( + outbound: &Arc, + feedback: Option<&moltis_channels::FeedbackService>, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + text: &str, + logbook_html: &str, + reply_to: Option<&str>, + session_key: &str, +) { + let result = if logbook_html.is_empty() { + outbound + .send_text_reporting_ids(&target.account_id, to, text, reply_to) + .await + } else { + outbound + .send_text_with_suffix_reporting_ids( + &target.account_id, + to, + text, + logbook_html, + reply_to, + ) + .await + }; + match result { + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback, + target, + &message_ids, + session_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel reply: {e}" + ); + }, + } +} + +pub(crate) async fn deliver_channel_replies_to_targets( + outbound: Arc, + targets: Vec, + session_key: &str, + text: &str, + state: Arc, + desired_reply_medium: ReplyMedium, + status_log: Vec, + streamed_target_keys: &HashSet, +) { + let session_key = session_key.to_string(); + let text = text.to_string(); + let logbook_html = format_logbook_html(&status_log); + // Resolved once rather than per target: it is the same service either way. + let feedback = state.feedback(); + let mut tasks = Vec::with_capacity(targets.len()); + for target in targets { + let outbound = Arc::clone(&outbound); + let state = Arc::clone(&state); + let feedback = feedback.clone(); + let session_key = session_key.clone(); + let text = text.clone(); + let logbook_html = logbook_html.clone(); + // Text was already delivered via edit-in-place streaming — skip text + // caption/follow-up and only send the TTS voice audio. + let text_already_streamed = + streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); + let to = target.outbound_to().into_owned(); + tasks.push(tokio::spawn(async move { + let tts_payload = match desired_reply_medium { + ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, + ReplyMedium::Text => None, + }; + let reply_to = target.message_id.as_deref(); + match target.channel_type { + moltis_channels::ChannelType::Telegram => match tts_payload { + Some(mut payload) => { + let transcript = std::mem::take(&mut payload.text); + + if text_already_streamed { + // Text was already streamed — send voice audio only. + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + // Send logbook as a follow-up if present. + if !logbook_html.is_empty() + && let Err(e) = outbound + .send_html(&target.account_id, &to, &logbook_html, None) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } + } else { + // Check if transcript fits as Telegram caption (when feature enabled). + // When telegram feature is disabled, this evaluates to false and we + // send voice + follow-up text. + #[cfg(feature = "telegram")] + let fits_in_caption = transcript.len() + <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; + #[cfg(not(feature = "telegram"))] + let fits_in_caption = false; + + if fits_in_caption { + // Short transcript fits as a caption on the voice message. + payload.text = transcript; + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + // Send logbook as a follow-up if present. + if !logbook_html.is_empty() + && let Err(e) = outbound + .send_html(&target.account_id, &to, &logbook_html, None) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } + } else { + // Transcript too long for a caption — send voice + // without caption, then the full text as a follow-up. + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + let text_result = if logbook_html.is_empty() { + outbound + .send_text(&target.account_id, &to, &transcript, None) + .await + } else { + outbound + .send_text_with_suffix( + &target.account_id, + &to, + &transcript, + &logbook_html, + None, + ) + .await + }; + if let Err(e) = text_result { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send transcript follow-up: {e}" + ); + } + } + } + }, + None if text_already_streamed => { + // TTS disabled/failed but text was already streamed — + // only send logbook follow-up if present. + if !logbook_html.is_empty() + && let Err(e) = outbound + .send_html(&target.account_id, &to, &logbook_html, None) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } + }, + None => { + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + &session_key, + ) + .await; + }, + }, + _ => match tts_payload { + Some(payload) => { + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + } + }, + None if text_already_streamed => { + // TTS disabled/failed but text was already streamed — + // only send logbook follow-up if present. + if !logbook_html.is_empty() + && let Err(e) = outbound + .send_html(&target.account_id, &to, &logbook_html, None) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } + }, + None => { + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + &session_key, + ) + .await; + }, + }, + } + })); + } + + for task in tasks { + if let Err(e) = task.await { + warn!(error = %e, "channel reply task join failed"); + } + } +} + +#[cfg(test)] +mod tests { + use { + super::*, + async_trait::async_trait, + moltis_channels::{ChannelReplyTarget, ChannelType}, + moltis_common::types::ReplyPayload, + std::sync::Mutex, + }; + + /// Records which send method was used and hands back ids, so a test can + /// tell an attributable delivery from one that silently drops the ids. + #[derive(Default)] + struct ReportingOutbound { + calls: Mutex>, + } + + #[async_trait] + impl moltis_channels::ChannelOutbound for ReportingOutbound { + async fn send_text( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text"); + Ok(()) + } + + async fn send_media( + &self, + _account_id: &str, + _to: &str, + _payload: &ReplyPayload, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + Ok(()) + } + + async fn send_text_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_reporting_ids"); + Ok(vec!["plain-1".into()]) + } + + async fn send_text_with_suffix( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _suffix_html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_with_suffix"); + Ok(()) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _text: &str, + _suffix_html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_text_with_suffix_reporting_ids"); + Ok(vec!["logbook-1".into()]) + } + } + + fn reply_target() -> ChannelReplyTarget { + ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Discord, + account_id: "acct".into(), + chat_id: "chan".into(), + message_id: None, + thread_id: None, + } + } + + /// A reply carrying an activity logbook is still a reply someone can react + /// to, so it must go out through the id-reporting suffix send rather than + /// the plain one, which reports nothing and loses attribution. + #[tokio::test] + async fn a_logbook_reply_uses_the_id_reporting_suffix_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_text_reply( + &outbound, + None, + &reply_target(), + "chan", + "the answer", + "
log
", + None, + "session", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_text_with_suffix_reporting_ids"]); + } + + #[tokio::test] + async fn a_plain_reply_uses_the_id_reporting_text_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_text_reply( + &outbound, + None, + &reply_target(), + "chan", + "the answer", + "", + None, + "session", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_text_reporting_ids"]); + } +} diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index e60801eec..aa6d0dc5c 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -10,9 +10,7 @@ use { use moltis_sessions::store::SessionStore; -use crate::{ - agent_loop::ChannelReplyTargetKey, compaction_run, error, runtime::ChatRuntime, types::*, -}; +use crate::{agent_loop::ChannelReplyTargetKey, error, runtime::ChatRuntime, types::*}; /// Build the SPA URL for a push notification click-through. /// @@ -150,7 +148,7 @@ pub(crate) async fn deliver_channel_replies( } return; } - deliver_channel_replies_to_targets( + crate::channel_reply_delivery::deliver_channel_replies_to_targets( outbound, targets, session_key, @@ -165,7 +163,7 @@ pub(crate) async fn deliver_channel_replies( /// Format buffered status log entries into a Telegram expandable blockquote HTML. /// Returns an empty string if there are no entries. -fn format_logbook_html(entries: &[String]) -> String { +pub(crate) fn format_logbook_html(entries: &[String]) -> String { if entries.is_empty() { return String::new(); } @@ -362,272 +360,6 @@ pub(crate) async fn deliver_channel_error( } } -async fn deliver_channel_replies_to_targets( - outbound: Arc, - targets: Vec, - session_key: &str, - text: &str, - state: Arc, - desired_reply_medium: ReplyMedium, - status_log: Vec, - streamed_target_keys: &HashSet, -) { - let session_key = session_key.to_string(); - let text = text.to_string(); - let logbook_html = format_logbook_html(&status_log); - let mut tasks = Vec::with_capacity(targets.len()); - for target in targets { - let outbound = Arc::clone(&outbound); - let state = Arc::clone(&state); - let session_key = session_key.clone(); - let text = text.clone(); - let logbook_html = logbook_html.clone(); - // Text was already delivered via edit-in-place streaming — skip text - // caption/follow-up and only send the TTS voice audio. - let text_already_streamed = - streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); - let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { - let tts_payload = match desired_reply_medium { - ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, - ReplyMedium::Text => None, - }; - let reply_to = target.message_id.as_deref(); - match target.channel_type { - moltis_channels::ChannelType::Telegram => match tts_payload { - Some(mut payload) => { - let transcript = std::mem::take(&mut payload.text); - - if text_already_streamed { - // Text was already streamed — send voice audio only. - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - // Check if transcript fits as Telegram caption (when feature enabled). - // When telegram feature is disabled, this evaluates to false and we - // send voice + follow-up text. - #[cfg(feature = "telegram")] - let fits_in_caption = transcript.len() - <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; - #[cfg(not(feature = "telegram"))] - let fits_in_caption = false; - - if fits_in_caption { - // Short transcript fits as a caption on the voice message. - payload.text = transcript; - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - // Transcript too long for a caption — send voice - // without caption, then the full text as a follow-up. - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - let text_result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &transcript, None) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &transcript, - &logbook_html, - None, - ) - .await - }; - if let Err(e) = text_result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send transcript follow-up: {e}" - ); - } - } - } - }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - }, - None => { - let result = if logbook_html.is_empty() { - // Ask for the delivered message ids so a later - // reaction can be attributed to this turn. Channels - // that cannot report them return an empty list and - // simply get no feedback attribution. - match outbound - .send_text_reporting_ids(&target.account_id, &to, &text, reply_to) - .await - { - Ok(message_ids) => { - crate::channel_feedback::record_reply_trace( - &state, - &target, - &to, - &message_ids, - &session_key, - ) - .await; - Ok(()) - }, - Err(e) => Err(e), - } - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - } - }, - }, - _ => match tts_payload { - Some(payload) => { - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - } - }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - }, - None => { - let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - } - }, - }, - } - })); - } - - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel reply task join failed"); - } - } -} - #[derive(Debug, Deserialize)] struct TtsStatusResponse { enabled: bool, @@ -708,7 +440,7 @@ pub(crate) async fn generate_tts_audio( .map_err(|_| error::Error::message("invalid base64 audio returned by TTS provider")) } -async fn build_tts_payload( +pub(crate) async fn build_tts_payload( state: &Arc, session_key: &str, target: &moltis_channels::ChannelReplyTarget, diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index 2d573bf32..87a104cb0 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,6 +1,7 @@ mod agent_loop; mod channel_compaction; mod channel_feedback; +mod channel_reply_delivery; mod channels; mod compaction; mod compaction_run; diff --git a/crates/discord/src/outbound.rs b/crates/discord/src/outbound.rs index 235edfbfe..0f8add470 100644 --- a/crates/discord/src/outbound.rs +++ b/crates/discord/src/outbound.rs @@ -23,7 +23,7 @@ use { }; use crate::{ - handler::{send_discord_message, send_discord_message_all, send_discord_text}, + handler::{send_discord_message, send_discord_message_all}, state::AccountStateMap, }; @@ -363,6 +363,11 @@ impl DiscordOutbound { /// Inner implementation for `send_text_with_suffix` that does not handle /// ack reaction removal -- the caller is responsible for that. + /// + /// Returns the ids of the messages the reply text occupied so a later + /// reaction can be attributed to the turn that wrote them. The activity-log + /// embed is deliberately not among them: it is a separate artifact from the + /// reply, and rating it is not rating the answer. async fn send_text_with_suffix_inner( &self, account_id: &str, @@ -371,10 +376,10 @@ impl DiscordOutbound { text: &str, suffix_html: &str, reply_to: Option<&str>, - ) -> ChannelResult<()> { + ) -> ChannelResult> { // Send main response text. let reference = self.resolve_reference(account_id, reply_to); - send_discord_message(http, channel_id, text, reference) + let sent = send_discord_message_all(http, channel_id, text, reference) .await .map_err(|e| ChannelError::external("Discord send", std::io::Error::other(e)))?; @@ -382,7 +387,7 @@ impl DiscordOutbound { self.send_activity_log_embed(account_id, http, channel_id, suffix_html, None) .await?; - Ok(()) + Ok(sent.into_iter().map(|m| m.id.to_string()).collect()) } } @@ -566,6 +571,19 @@ impl ChannelOutbound for DiscordOutbound { suffix_html: &str, reply_to: Option<&str>, ) -> ChannelResult<()> { + self.send_text_with_suffix_reporting_ids(account_id, to, text, suffix_html, reply_to) + .await?; + Ok(()) + } + + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; @@ -691,15 +709,19 @@ fn truncate_at_char_boundary(s: &str, max: usize) -> &str { &s[..end] } -#[async_trait] -impl ChannelStreamOutbound for DiscordOutbound { - async fn send_stream( +impl DiscordOutbound { + /// Drive an edit-in-place stream, returning the ids of the messages the + /// final reply ended up occupying. + /// + /// Both trait entry points share this so the streaming state machine + /// exists once; [`ChannelStreamOutbound::send_stream`] discards the ids. + async fn send_stream_ids( &self, account_id: &str, to: &str, reply_to: Option<&str>, mut stream: StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; let reference = self.resolve_reference(account_id, reply_to); @@ -783,6 +805,9 @@ impl ChannelStreamOutbound for DiscordOutbound { } } + // Ids of messages this stream created besides the edited one. + let mut extra_ids: Vec = Vec::new(); + // Phase 3: final update with the complete text. if !accumulated.is_empty() { if accumulated.len() <= DISCORD_MAX_MESSAGE_LEN { @@ -798,11 +823,12 @@ impl ChannelStreamOutbound for DiscordOutbound { ); } } else { - send_discord_message(&http, channel_id, &accumulated, None) + let msg = send_discord_message(&http, channel_id, &accumulated, None) .await .map_err(|e| { ChannelError::external("Discord send", std::io::Error::other(e)) })?; + extra_ids.push(msg.id.to_string()); } } else { // Content overflows -- edit the first message with the first 2000 chars, @@ -811,13 +837,15 @@ impl ChannelStreamOutbound for DiscordOutbound { if let Some(msg_id) = sent_message_id { let edit = EditMessage::new().content(first); let _ = channel_id.edit_message(&http, msg_id, edit).await; - } else { - let _ = send_discord_text(&http, channel_id, first).await; + } else if let Ok(sent) = + send_discord_message_all(&http, channel_id, first, None).await + { + extra_ids.extend(sent.into_iter().map(|m| m.id.to_string())); } let rest = &accumulated[first.len()..]; if !rest.is_empty() { - send_discord_text(&http, channel_id, rest) + let sent = send_discord_message_all(&http, channel_id, rest, None) .await .map_err(|e| { ChannelError::external( @@ -825,6 +853,7 @@ impl ChannelStreamOutbound for DiscordOutbound { std::io::Error::other(e), ) })?; + extra_ids.extend(sent.into_iter().map(|m| m.id.to_string())); } } } @@ -840,9 +869,37 @@ impl ChannelStreamOutbound for DiscordOutbound { streamed = sent_message_id.is_some(), "discord stream completed" ); + let mut ids = Vec::with_capacity(extra_ids.len() + 1); + ids.extend(sent_message_id.map(|id| id.to_string())); + ids.extend(extra_ids); + Ok(ids) + } +} + +#[async_trait] +impl ChannelStreamOutbound for DiscordOutbound { + async fn send_stream( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> ChannelResult<()> { + self.send_stream_ids(account_id, to, reply_to, stream) + .await?; Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> ChannelResult> { + self.send_stream_ids(account_id, to, reply_to, stream).await + } + async fn is_stream_enabled(&self, _account_id: &str) -> bool { true } diff --git a/crates/gateway/src/methods/dispatch.rs b/crates/gateway/src/methods/dispatch.rs index 2f60b4fec..8347c051f 100644 --- a/crates/gateway/src/methods/dispatch.rs +++ b/crates/gateway/src/methods/dispatch.rs @@ -96,7 +96,6 @@ const READ_METHODS: &[&str] = &[ "webhooks.delivery.payload", "webhooks.delivery.actions", "feedback.status", - "feedback.submit", "instrumentation.status", "heartbeat.status", "heartbeat.runs", @@ -258,6 +257,8 @@ const WRITE_METHODS: &[&str] = &[ "webhooks.create", "webhooks.update", "webhooks.delete", + // Writes a score against a trace: a mutation, not a read. + "feedback.submit", "instrumentation.test", "heartbeat.update", "heartbeat.run", @@ -504,6 +505,22 @@ mod tests { ); } + #[test] + fn feedback_status_requires_read_but_submit_requires_write() { + // Submitting writes a score against a trace. A read-scoped client must + // not be able to record or retract one. + assert!( + authorize_method("feedback.status", "operator", &scopes(&["operator.read"])).is_none() + ); + assert_error_code( + authorize_method("feedback.submit", "operator", &scopes(&["operator.read"])), + "UNAUTHORIZED", + ); + assert!( + authorize_method("feedback.submit", "operator", &scopes(&["operator.write"])).is_none() + ); + } + #[test] fn senders_approve_requires_write() { assert!( diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs index 6371369af..edd986163 100644 --- a/crates/gateway/src/methods/services/feedback.rs +++ b/crates/gateway/src/methods/services/feedback.rs @@ -3,6 +3,10 @@ //! The web is treated as one more channel: a reply gets a link keyed on the //! session and the message index, and a thumb resolves that link the same way //! a Telegram reaction does. +//! +//! `feedback.status` is a read. `feedback.submit` writes a score and is +//! authorized as a write, with the scoring identity taken from the +//! authenticated operator rather than from the request. use { moltis_channels::FeedbackOutcome, @@ -75,11 +79,13 @@ pub(super) fn register(reg: &mut MethodRegistry) { }, }; - let user_id = ctx - .params - .get("userId") - .and_then(|v| v.as_str()) - .unwrap_or("web"); + // The score is attributed to the authenticated web operator, + // never to an identity named in the request: a caller-chosen + // `userId` would let one client overwrite or retract another's + // vote, since the score id is derived from (trace, name, user). + // A stable value is required for that upsert to keep working + // across reconnects, which rules out the connection id. + let user_id = moltis_channels::trace_link::WEB_CHANNEL; let outcome = ctx .state diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index ed0bfa9e5..3979c5362 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -177,13 +177,16 @@ impl SlackOutbound { } /// Edit-in-place streaming: post → throttled edits → final update. + /// + /// Returns the timestamps of the messages the reply occupies, which are + /// what a later reaction is keyed on. async fn send_stream_edit_in_place( &self, account_id: &str, to: &str, thread_ts: Option<&str>, stream: &mut StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let (client, token) = self.get_session(account_id)?; let throttle = self.get_edit_throttle(account_id); @@ -258,35 +261,97 @@ impl SlackOutbound { } if accumulated.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let final_text = markdown_to_slack(&accumulated); let chunks = chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); match &sent_ts { Some(ts) => { + ids.push(ts.to_string()); if let Some(first) = chunks.first() && let Err(e) = update_message(&client, &token, to, ts, first).await { warn!(account_id, to, "failed to finalize stream message: {e}"); } for chunk in chunks.iter().skip(1) { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send overflow chunk: {e}"); + match post_message(&client, &token, to, chunk, thread_ts).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send overflow chunk: {e}"), } } }, None => { for chunk in &chunks { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send stream message: {e}"); + match post_message(&client, &token, to, chunk, thread_ts).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send stream message: {e}"), } } }, } - Ok(()) + Ok(ids) + } + + /// Drive a stream in whichever mode the account is configured for, + /// returning the timestamps of the messages the reply occupies. + /// + /// Native streaming reports none: Slack's `chat.startStream` owns the + /// message and does not hand back a timestamp the reaction side could + /// match, so those replies lose attribution rather than gaining a wrong + /// one. + async fn send_stream_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: &mut StreamReceiver, + ) -> ChannelResult> { + let stream_mode = self.get_stream_mode(account_id); + let thread_ts = self.get_thread_ts(account_id, to, reply_to); + + match stream_mode { + StreamMode::Native => { + self.send_stream_native(account_id, to, thread_ts.as_deref(), stream) + .await?; + Ok(Vec::new()) + }, + StreamMode::EditInPlace => { + self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), stream) + .await + }, + StreamMode::Off => { + // Streaming disabled — accumulate and send once. + let mut accumulated = String::new(); + loop { + match stream.recv().await { + Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { + accumulated.push_str(&chunk) + }, + Some(StreamEvent::Error(e)) => { + accumulated.push_str(&format!("\n\n:warning: {e}")); + break; + }, + Some(StreamEvent::Done) | None => break, + } + } + let mut ids = Vec::new(); + if !accumulated.is_empty() { + let (client, token) = self.get_session(account_id)?; + let final_text = markdown_to_slack(&accumulated); + for chunk in chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN) { + match post_message(&client, &token, to, chunk, thread_ts.as_deref()).await { + Ok(ts) => ids.push(ts.to_string()), + Err(e) => warn!(account_id, to, "failed to send stream message: {e}"), + } + } + } + Ok(ids) + }, + } } } @@ -516,6 +581,22 @@ impl ChannelOutbound for SlackOutbound { Ok(ids) } + /// Slack has no suffix rendering, so the inherited `send_text_with_suffix` + /// drops it and posts the text alone. Route the reporting variant to + /// `send_text_reporting_ids` rather than the trait default, which would + /// throw away ids Slack can perfectly well supply. + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + _suffix_html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + self.send_text_reporting_ids(account_id, to, text, reply_to) + .await + } + async fn send_media( &self, account_id: &str, @@ -815,47 +896,20 @@ impl ChannelStreamOutbound for SlackOutbound { reply_to: Option<&str>, mut stream: StreamReceiver, ) -> ChannelResult<()> { - let stream_mode = self.get_stream_mode(account_id); - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + self.send_stream_ids(account_id, to, reply_to, &mut stream) + .await?; + Ok(()) + } - match stream_mode { - StreamMode::Native => { - self.send_stream_native(account_id, to, thread_ts.as_deref(), &mut stream) - .await - }, - StreamMode::EditInPlace => { - self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), &mut stream) - .await - }, - StreamMode::Off => { - // Streaming disabled — accumulate and send once. - let mut accumulated = String::new(); - loop { - match stream.recv().await { - Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { - accumulated.push_str(&chunk) - }, - Some(StreamEvent::Error(e)) => { - accumulated.push_str(&format!("\n\n:warning: {e}")); - break; - }, - Some(StreamEvent::Done) | None => break, - } - } - if !accumulated.is_empty() { - let (client, token) = self.get_session(account_id)?; - let final_text = markdown_to_slack(&accumulated); - for chunk in chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN) { - if let Err(e) = - post_message(&client, &token, to, chunk, thread_ts.as_deref()).await - { - warn!(account_id, to, "failed to send stream message: {e}"); - } - } - } - Ok(()) - }, - } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + mut stream: StreamReceiver, + ) -> ChannelResult> { + self.send_stream_ids(account_id, to, reply_to, &mut stream) + .await } async fn is_stream_enabled(&self, account_id: &str) -> bool { diff --git a/crates/telegram/src/outbound/send.rs b/crates/telegram/src/outbound/send.rs index 72614f212..2e8a6c2f0 100644 --- a/crates/telegram/src/outbound/send.rs +++ b/crates/telegram/src/outbound/send.rs @@ -21,6 +21,121 @@ use crate::{ use super::{TelegramOutbound, retry::RequestResultExt}; +impl TelegramOutbound { + /// Send `text` with `suffix_html` appended, returning every message id it + /// produced. + /// + /// Shared by both trait entry points so the chunking and suffix-overflow + /// rules exist once; the non-reporting one simply discards the ids. + async fn send_text_with_suffix_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + let bot = self.get_bot(account_id)?; + let (chat_id, thread_id) = parse_chat_target(to)?; + let rp = self.reply_params(account_id, reply_to); + + // Send typing indicator + let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; + + // Append the pre-formatted suffix (e.g. activity logbook) to the last chunk. + let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); + let last_idx = chunks.len().saturating_sub(1); + info!( + account_id, + chat_id = to, + reply_to = ?reply_to, + text_len = text.len(), + suffix_len = suffix_html.len(), + chunk_count = chunks.len(), + "telegram outbound text+suffix send start" + ); + + let mut ids = Vec::with_capacity(chunks.len()); + for (i, chunk) in chunks.iter().enumerate() { + let content = if i == last_idx { + // Append suffix to the last chunk. If it would exceed the limit, + // the suffix becomes a separate final message. + let combined = format!("{chunk}\n\n{suffix_html}"); + if combined.len() <= TELEGRAM_MAX_MESSAGE_LEN { + combined + } else { + // Send this chunk first, then the suffix as a separate message. + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); + // Send suffix as the final message (no reply threading). + let suffix_id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + suffix_html, + rp.as_ref(), + true, + ) + .await?; + // The logbook is part of the same reply, so a thumb on it + // still means "rate this turn". + ids.push(suffix_id.0.to_string()); + info!( + account_id, + chat_id = to, + reply_to = ?reply_to, + text_len = text.len(), + suffix_len = suffix_html.len(), + chunk_count = chunks.len(), + "telegram outbound text+suffix sent (separate suffix message)" + ); + return Ok(ids); + } + } else { + chunk.clone() + }; + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + &content, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); + } + + info!( + account_id, + chat_id = to, + reply_to = ?reply_to, + text_len = text.len(), + suffix_len = suffix_html.len(), + chunk_count = chunks.len(), + "telegram outbound text+suffix sent" + ); + Ok(ids) + } +} + #[async_trait] impl ChannelOutbound for TelegramOutbound { async fn send_text( @@ -124,97 +239,23 @@ impl ChannelOutbound for TelegramOutbound { suffix_html: &str, reply_to: Option<&str>, ) -> Result<()> { - let bot = self.get_bot(account_id)?; - let (chat_id, thread_id) = parse_chat_target(to)?; - let rp = self.reply_params(account_id, reply_to); - - // Send typing indicator - let _ = bot.send_chat_action(chat_id, ChatAction::Typing).await; - - // Append the pre-formatted suffix (e.g. activity logbook) to the last chunk. - let chunks = markdown::chunk_markdown_html(text, TELEGRAM_MAX_MESSAGE_LEN); - let last_idx = chunks.len().saturating_sub(1); - info!( - account_id, - chat_id = to, - reply_to = ?reply_to, - text_len = text.len(), - suffix_len = suffix_html.len(), - chunk_count = chunks.len(), - "telegram outbound text+suffix send start" - ); - - for (i, chunk) in chunks.iter().enumerate() { - let content = if i == last_idx { - // Append suffix to the last chunk. If it would exceed the limit, - // the suffix becomes a separate final message. - let combined = format!("{chunk}\n\n{suffix_html}"); - if combined.len() <= TELEGRAM_MAX_MESSAGE_LEN { - combined - } else { - // Send this chunk first, then the suffix as a separate message. - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - chunk, - rp.as_ref(), - false, - ) - .await?; - // Send suffix as the final message (no reply threading). - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - suffix_html, - rp.as_ref(), - true, - ) - .await?; - info!( - account_id, - chat_id = to, - reply_to = ?reply_to, - text_len = text.len(), - suffix_len = suffix_html.len(), - chunk_count = chunks.len(), - "telegram outbound text+suffix sent (separate suffix message)" - ); - return Ok(()); - } - } else { - chunk.clone() - }; - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - &content, - rp.as_ref(), - false, - ) + self.send_text_with_suffix_ids(account_id, to, text, suffix_html, reply_to) .await?; - } - - info!( - account_id, - chat_id = to, - reply_to = ?reply_to, - text_len = text.len(), - suffix_len = suffix_html.len(), - chunk_count = chunks.len(), - "telegram outbound text+suffix sent" - ); Ok(()) } + async fn send_text_with_suffix_reporting_ids( + &self, + account_id: &str, + to: &str, + text: &str, + suffix_html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_text_with_suffix_ids(account_id, to, text, suffix_html, reply_to) + .await + } + async fn send_html( &self, account_id: &str, diff --git a/crates/telegram/src/outbound/stream.rs b/crates/telegram/src/outbound/stream.rs index 632d6db54..61c626242 100644 --- a/crates/telegram/src/outbound/stream.rs +++ b/crates/telegram/src/outbound/stream.rs @@ -512,6 +512,7 @@ impl TelegramOutbound { Ok(()) } + /// Send the chunks that did not fit the edited message, reporting their ids. async fn send_remaining_final_chunks( &self, bot: &Bot, @@ -521,23 +522,26 @@ impl TelegramOutbound { thread_id: Option, reply_params: Option<&ReplyParameters>, final_state: &StreamFinalState, - ) -> Result<()> { + ) -> Result> { let chunks = markdown::chunk_markdown_html(&final_state.accumulated, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::new(); for chunk in chunks.into_iter().skip(1) { - self.send_chunk_with_fallback( - bot, - account_id, - to, - chat_id, - thread_id, - &chunk, - reply_params, - true, - ) - .await?; + let id = self + .send_chunk_with_fallback( + bot, + account_id, + to, + chat_id, + thread_id, + &chunk, + reply_params, + true, + ) + .await?; + ids.push(id.0.to_string()); } - Ok(()) + Ok(ids) } async fn finish_final_stream_message( @@ -552,9 +556,9 @@ impl TelegramOutbound { progress_message_id: &mut Option, final_state: &mut StreamFinalState, throttle: Duration, - ) -> Result<()> { + ) -> Result> { if final_state.accumulated.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let display = final_state.current_final_html(); @@ -667,15 +671,19 @@ fn trim_to_recent_chars(text: &mut String, max_chars: usize) -> bool { true } -#[async_trait] -impl ChannelStreamOutbound for TelegramOutbound { - async fn send_stream( +impl TelegramOutbound { + /// Drive an edit-in-place stream, returning the ids of the messages the + /// final reply ended up occupying. + /// + /// Both trait entry points share this so the streaming state machine + /// exists once; [`ChannelStreamOutbound::send_stream`] discards the ids. + async fn send_stream_ids( &self, account_id: &str, to: &str, reply_to: Option<&str>, mut stream: StreamReceiver, - ) -> Result<()> { + ) -> Result> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; let rp = self.reply_params(account_id, reply_to); @@ -693,12 +701,15 @@ impl ChannelStreamOutbound for TelegramOutbound { typing_interval.tick().await; let mut flush_interval = tokio::time::interval(throttle); flush_interval.tick().await; + // Chunks that spilled past the edited message; the edited message's own + // id is read off `final_state` once the loop ends. + let mut overflow_ids: Vec = Vec::new(); loop { tokio::select! { event = stream.recv() => { let Some(event) = event else { - self.finish_final_stream_message( + overflow_ids = self.finish_final_stream_message( &bot, account_id, to, @@ -836,7 +847,7 @@ impl ChannelStreamOutbound for TelegramOutbound { } }, StreamEvent::Done => { - self.finish_final_stream_message( + overflow_ids = self.finish_final_stream_message( &bot, account_id, to, @@ -915,9 +926,37 @@ impl ChannelStreamOutbound for TelegramOutbound { .await; } + let mut ids = Vec::with_capacity(overflow_ids.len() + 1); + ids.extend(final_state.message_id.map(|id| id.0.to_string())); + ids.extend(overflow_ids); + Ok(ids) + } +} + +#[async_trait] +impl ChannelStreamOutbound for TelegramOutbound { + async fn send_stream( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result<()> { + self.send_stream_ids(account_id, to, reply_to, stream) + .await?; Ok(()) } + async fn send_stream_reporting_ids( + &self, + account_id: &str, + to: &str, + reply_to: Option<&str>, + stream: StreamReceiver, + ) -> Result> { + self.send_stream_ids(account_id, to, reply_to, stream).await + } + async fn is_stream_enabled(&self, account_id: &str) -> bool { let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); accounts From d6f469c715e8c3bd47f5662bb882a7ee8595027a Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 01:08:27 -0400 Subject: [PATCH 18/33] fix(discord): link the activity-log embed to the reply's trace Telegram already linked the logbook message it emits when the suffix does not fit the last chunk, but Discord dropped the equivalent embed. A reader rating "the bot's answer" may land on either, and both map back to the same turn, so linking them recovers a score rather than losing it to an UnknownMessage lookup. --- crates/discord/src/outbound.rs | 26 +++++++++++++++----------- crates/telegram/src/outbound/send.rs | 5 +++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/discord/src/outbound.rs b/crates/discord/src/outbound.rs index 0f8add470..b057cf13c 100644 --- a/crates/discord/src/outbound.rs +++ b/crates/discord/src/outbound.rs @@ -336,9 +336,9 @@ impl DiscordOutbound { channel_id: ChannelId, suffix_html: &str, reply_to: Option<&str>, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let Some(rendered) = render_activity_log_for_discord(suffix_html) else { - return Ok(()); + return Ok(None); }; let color: u32 = if rendered.has_errors { @@ -355,19 +355,19 @@ impl DiscordOutbound { msg = msg.reference_message((channel_id, reference)); } - channel_id.send_message(http, msg).await.map_err(|e| { + let sent = channel_id.send_message(http, msg).await.map_err(|e| { ChannelError::external("Discord send embed", std::io::Error::other(e.to_string())) })?; - Ok(()) + Ok(Some(sent.id)) } /// Inner implementation for `send_text_with_suffix` that does not handle /// ack reaction removal -- the caller is responsible for that. /// - /// Returns the ids of the messages the reply text occupied so a later - /// reaction can be attributed to the turn that wrote them. The activity-log - /// embed is deliberately not among them: it is a separate artifact from the - /// reply, and rating it is not rating the answer. + /// Returns every message id the reply produced, the activity-log embed + /// included: a reader rating "the bot's answer" may land on any of them, + /// and each maps back to the same turn, so being forgiving here costs + /// nothing and recovers scores that would otherwise be dropped. async fn send_text_with_suffix_inner( &self, account_id: &str, @@ -384,10 +384,13 @@ impl DiscordOutbound { .map_err(|e| ChannelError::external("Discord send", std::io::Error::other(e)))?; // Send the activity log as a separate embed message. - self.send_activity_log_embed(account_id, http, channel_id, suffix_html, None) + let embed_id = self + .send_activity_log_embed(account_id, http, channel_id, suffix_html, None) .await?; - Ok(sent.into_iter().map(|m| m.id.to_string()).collect()) + let mut ids: Vec = sent.into_iter().map(|m| m.id.to_string()).collect(); + ids.extend(embed_id.map(|id| id.to_string())); + Ok(ids) } } @@ -616,7 +619,8 @@ impl ChannelOutbound for DiscordOutbound { let http = self.resolve_http(account_id)?; let channel_id = Self::parse_channel_id(to)?; self.send_activity_log_embed(account_id, &http, channel_id, html, reply_to) - .await + .await?; + Ok(()) } async fn send_location( diff --git a/crates/telegram/src/outbound/send.rs b/crates/telegram/src/outbound/send.rs index 2e8a6c2f0..6ae5e1478 100644 --- a/crates/telegram/src/outbound/send.rs +++ b/crates/telegram/src/outbound/send.rs @@ -91,8 +91,9 @@ impl TelegramOutbound { true, ) .await?; - // The logbook is part of the same reply, so a thumb on it - // still means "rate this turn". + // A reader rating "the bot's answer" may land on the + // logbook, and it maps back to the same turn, so linking it + // recovers a score that would otherwise be dropped. ids.push(suffix_id.0.to_string()); info!( account_id, From 9c07ec03078514382f15ff5110b1c51288ea6adc Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 01:43:48 -0400 Subject: [PATCH 19/33] test(gateway): cover feedback.submit authorization through dispatch The scope fix was only asserted at authorize_method. Dispatching the method with operator.read exercises the path a forging client would actually take, and pins the behavior against a future edit that moves the method back into READ_METHODS. --- crates/gateway/src/methods/dispatch.rs | 48 ++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/crates/gateway/src/methods/dispatch.rs b/crates/gateway/src/methods/dispatch.rs index 8347c051f..0fc2b4f64 100644 --- a/crates/gateway/src/methods/dispatch.rs +++ b/crates/gateway/src/methods/dispatch.rs @@ -776,6 +776,54 @@ mod tests { ); } + /// Guards the fix end to end rather than only at `authorize_method`: the + /// method was reachable with `operator.read` while writing a score, so a + /// read-scoped client could forge or retract feedback. + #[test] + fn feedback_submit_is_rejected_for_read_scoped_clients() { + use crate::{ + auth::{AuthMode, ResolvedAuth}, + services::GatewayServices, + state::GatewayState, + }; + + let reg = MethodRegistry::new(); + let ctx = MethodContext { + request_id: "test".into(), + method: "feedback.submit".into(), + params: serde_json::json!({ + "sessionKey": "someone-elses-session", + "messageId": "run-1", + "signal": "positive", + // Ignored by the handler, which attributes to the + // authenticated operator; present here to prove the request + // cannot choose whose vote it writes. + "userId": "victim", + }), + client_conn_id: "conn-1".into(), + client_role: "operator".into(), + client_scopes: scopes(&["operator.read"]), + state: GatewayState::new( + ResolvedAuth { + mode: AuthMode::Token, + token: None, + password: None, + }, + GatewayServices::noop(), + ), + channel: None, + }; + let rt = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime"); + let resp = rt.block_on(reg.dispatch(ctx)); + assert!(!resp.ok); + assert_eq!( + resp.error.as_ref().map(|e| e.code.as_str()), + Some("UNAUTHORIZED") + ); + } + #[test] fn chat_send_sync_is_registered_and_authorized() { let reg = MethodRegistry::new(); From adf631bb03d5ffdbdb9dca29a5a0a199ae25c4ad Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 01:49:11 -0400 Subject: [PATCH 20/33] fix(feedback): attribute voice transcript and Slack native-stream replies Two delivery paths still produced replies nobody could rate. A Telegram voice reply whose transcript is too long for a caption sends the text as a follow-up message. That branch called `send_text` directly and discarded the ids, so a reaction on the readable form of the answer found no trace link. It now goes through `deliver_text_reply` like every other text reply. Slack native streaming returned an empty id list. The timestamp was there all along: `chat.startStream` reports the `ts` of the message it creates, and the code read only `stream_id` from the response. `chat.stopStream` carries it too, used as a fallback. All three Slack stream modes now report ids, so a reaction resolves the same way however the account is configured. --- crates/chat/src/channel_reply_delivery.rs | 38 ++++++--------- crates/slack/src/outbound.rs | 59 +++++++++++++++-------- 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/crates/chat/src/channel_reply_delivery.rs b/crates/chat/src/channel_reply_delivery.rs index 00f47fafc..7928e2deb 100644 --- a/crates/chat/src/channel_reply_delivery.rs +++ b/crates/chat/src/channel_reply_delivery.rs @@ -186,29 +186,21 @@ pub(crate) async fn deliver_channel_replies_to_targets( "failed to send channel voice reply: {e}" ); } - let text_result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &transcript, None) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &transcript, - &logbook_html, - None, - ) - .await - }; - if let Err(e) = text_result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send transcript follow-up: {e}" - ); - } + // The transcript follow-up is the readable form + // of this turn's answer, so it is as much a + // reaction target as a plain text reply and + // goes out through the same attributing path. + deliver_text_reply( + &outbound, + feedback.as_deref(), + &target, + &to, + &transcript, + &logbook_html, + None, + &session_key, + ) + .await; } } }, diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 3979c5362..178e3c641 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -108,18 +108,21 @@ impl SlackOutbound { } /// Native Slack streaming using chat.startStream/appendStream/stopStream. + /// + /// Returns the timestamp of the message Slack created for the stream, so a + /// reaction on a natively streamed reply resolves like any other. async fn send_stream_native( &self, account_id: &str, to: &str, thread_ts: Option<&str>, stream: &mut StreamReceiver, - ) -> ChannelResult<()> { + ) -> ChannelResult> { let (bot_token, api_base_url, throttle) = self.get_native_stream_config(account_id)?; let http = moltis_common::http_client::build_default_http_client(); - let stream_id = - start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; + let started = start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; + let stream_id = started.stream_id; let mut pending = String::new(); let mut last_append = tokio::time::Instant::now(); @@ -168,12 +171,18 @@ impl SlackOutbound { } } - // Finalize the stream. - if let Err(e) = stop_native_stream(&http, &api_base_url, &bot_token, &stream_id).await { - warn!(account_id, to, "chat.stopStream failed: {e}"); - } + // Finalize the stream. Either call may carry the message timestamp; + // prefer the one from startStream and fall back to stopStream. + let stopped_ts = + match stop_native_stream(&http, &api_base_url, &bot_token, &stream_id).await { + Ok(ts) => ts, + Err(e) => { + warn!(account_id, to, "chat.stopStream failed: {e}"); + None + }, + }; - Ok(()) + Ok(started.ts.or(stopped_ts).into_iter().collect()) } /// Edit-in-place streaming: post → throttled edits → final update. @@ -299,10 +308,8 @@ impl SlackOutbound { /// Drive a stream in whichever mode the account is configured for, /// returning the timestamps of the messages the reply occupies. /// - /// Native streaming reports none: Slack's `chat.startStream` owns the - /// message and does not hand back a timestamp the reaction side could - /// match, so those replies lose attribution rather than gaining a wrong - /// one. + /// All three modes report ids, so a reaction resolves the same way however + /// the account happens to be configured. async fn send_stream_ids( &self, account_id: &str, @@ -316,8 +323,7 @@ impl SlackOutbound { match stream_mode { StreamMode::Native => { self.send_stream_native(account_id, to, thread_ts.as_deref(), stream) - .await?; - Ok(Vec::new()) + .await }, StreamMode::EditInPlace => { self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), stream) @@ -772,13 +778,22 @@ impl ChannelOutbound for SlackOutbound { /// Start a native Slack stream via `chat.startStream`. /// /// Returns `(stream_id, channel)` on success. +/// What `chat.startStream` hands back. +struct NativeStream { + stream_id: String, + /// Timestamp of the message Slack created to hold the stream, when the API + /// reports one. This is the id a later reaction is keyed on, so dropping it + /// would leave natively streamed replies unattributable. + ts: Option, +} + async fn start_native_stream( http: &reqwest::Client, api_base_url: &str, bot_token: &str, channel: &str, thread_ts: Option<&str>, -) -> ChannelResult { +) -> ChannelResult { let mut body = serde_json::json!({ "channel": channel }); if let Some(ts) = thread_ts { body["thread_ts"] = serde_json::json!(ts); @@ -807,10 +822,13 @@ async fn start_native_stream( ))); } - json.get("stream_id") + let stream_id = json + .get("stream_id") .and_then(|v| v.as_str()) .map(String::from) - .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing stream_id")) + .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing stream_id"))?; + let ts = json.get("ts").and_then(|v| v.as_str()).map(String::from); + Ok(NativeStream { stream_id, ts }) } /// Append text to a native Slack stream via `chat.appendStream`. @@ -853,12 +871,15 @@ async fn append_native_stream( } /// Finalize a native Slack stream via `chat.stopStream`. +/// Finalize a native stream, reporting the resulting message timestamp when +/// Slack includes one — a fallback for the `chat.startStream` response that +/// omits it. async fn stop_native_stream( http: &reqwest::Client, api_base_url: &str, bot_token: &str, stream_id: &str, -) -> ChannelResult<()> { +) -> ChannelResult> { let body = serde_json::json!({ "stream_id": stream_id }); let resp = http @@ -884,7 +905,7 @@ async fn stop_native_stream( ))); } - Ok(()) + Ok(json.get("ts").and_then(|v| v.as_str()).map(String::from)) } #[async_trait] From 0281c15f9285a3cd55daed0d7f21c741d4b0db61 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 02:00:14 -0400 Subject: [PATCH 21/33] fix(feedback): attribute the logbook that follows a streamed reply MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a reply is streamed edit-in-place, the activity logbook is sent afterwards via send_html — the one message that path creates. It went out with no trace link, so a reaction on it resolved as UnknownMessage. Adds send_html_reporting_ids alongside the other id-reporting sends, implemented for Telegram (chunk ids), Discord (the embed id) and Slack (which renders no HTML and posts the markup as text), and records the link for the streamed-target logbook follow-up. --- crates/channels/src/plugin.rs | 16 +++++++ crates/channels/src/registry.rs | 17 ++++++++ crates/chat/src/channels.rs | 39 +++++++++++++---- crates/discord/src/outbound.rs | 15 +++++++ crates/slack/src/outbound.rs | 14 +++++++ crates/telegram/src/outbound/send.rs | 63 +++++++++++++++++++--------- 6 files changed, 137 insertions(+), 27 deletions(-) diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index 487d12851..08d846d11 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -699,6 +699,22 @@ pub trait ChannelOutbound: Send + Sync { ) -> Result<()> { self.send_text(account_id, to, html, reply_to).await } + + /// [`Self::send_html`], reporting the ids of the messages it produced. + /// + /// The activity logbook that follows a streamed reply is delivered this + /// way. It belongs to the same turn, so a reaction on it should score that + /// turn rather than resolve to nothing. + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_html(account_id, to, html, reply_to).await?; + Ok(Vec::new()) + } /// Send a text message without notification (silent). Falls back to send_text by default. async fn send_text_silent( &self, diff --git a/crates/channels/src/registry.rs b/crates/channels/src/registry.rs index 978038fd8..0d208f1dd 100644 --- a/crates/channels/src/registry.rs +++ b/crates/channels/src/registry.rs @@ -428,6 +428,23 @@ impl ChannelOutbound for RegistryOutboundRouter { outbound.send_html(account_id, to, html, reply_to).await } + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + let outbound = self + .registry + .resolve_outbound(account_id) + .await + .ok_or_else(|| Error::unknown_account(account_id))?; + outbound + .send_html_reporting_ids(account_id, to, html, reply_to) + .await + } + async fn send_text_silent( &self, account_id: &str, diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index aa6d0dc5c..290302c97 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -134,6 +134,8 @@ pub(crate) async fn deliver_channel_replies( Arc::clone(&outbound), streamed_targets, &logbook_html, + state.feedback(), + session_key, ) .await; } @@ -180,32 +182,53 @@ pub(crate) fn format_logbook_html(entries: &[String]) -> String { html } +/// Send the activity logbook after a streamed reply. +/// +/// The stream delivered the answer itself, so this follow-up is the only +/// message this path creates — and a reader rating the turn may well land on +/// it. It is therefore linked to the trace like any other reply message. async fn send_channel_logbook_follow_up_to_targets( outbound: Arc, targets: Vec, logbook_html: &str, + feedback: Option>, + session_key: &str, ) { if targets.is_empty() || logbook_html.is_empty() { return; } let html = logbook_html.to_string(); + let session_key = session_key.to_string(); let mut tasks = Vec::with_capacity(targets.len()); for target in targets { let outbound = Arc::clone(&outbound); let html = html.clone(); + let feedback = feedback.clone(); + let session_key = session_key.clone(); let to = target.outbound_to().into_owned(); tasks.push(tokio::spawn(async move { - if let Err(e) = outbound - .send_html(&target.account_id, &to, &html, None) + match outbound + .send_html_reporting_ids(&target.account_id, &to, &html, None) .await { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback.as_deref(), + &target, + &message_ids, + &session_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + }, } })); } diff --git a/crates/discord/src/outbound.rs b/crates/discord/src/outbound.rs index b057cf13c..50722c888 100644 --- a/crates/discord/src/outbound.rs +++ b/crates/discord/src/outbound.rs @@ -623,6 +623,21 @@ impl ChannelOutbound for DiscordOutbound { Ok(()) } + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + let http = self.resolve_http(account_id)?; + let channel_id = Self::parse_channel_id(to)?; + let id = self + .send_activity_log_embed(account_id, &http, channel_id, html, reply_to) + .await?; + Ok(id.map(|id| id.to_string()).into_iter().collect()) + } + async fn send_location( &self, account_id: &str, diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 178e3c641..6994a7208 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -603,6 +603,20 @@ impl ChannelOutbound for SlackOutbound { .await } + /// Slack renders no HTML either, so the inherited `send_html` posts the + /// markup as text. Report its ids rather than inheriting the default that + /// drops them. + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> ChannelResult> { + self.send_text_reporting_ids(account_id, to, html, reply_to) + .await + } + async fn send_media( &self, account_id: &str, diff --git a/crates/telegram/src/outbound/send.rs b/crates/telegram/src/outbound/send.rs index 6ae5e1478..986237ceb 100644 --- a/crates/telegram/src/outbound/send.rs +++ b/crates/telegram/src/outbound/send.rs @@ -22,6 +22,39 @@ use crate::{ use super::{TelegramOutbound, retry::RequestResultExt}; impl TelegramOutbound { + /// Send raw HTML chunks, returning every message id they produced. + async fn send_html_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + let bot = self.get_bot(account_id)?; + let (chat_id, thread_id) = parse_chat_target(to)?; + let rp = self.reply_params(account_id, reply_to); + + // Send raw HTML chunks without markdown conversion. + let chunks = markdown::chunk_message(html, TELEGRAM_MAX_MESSAGE_LEN); + let mut ids = Vec::with_capacity(chunks.len()); + for chunk in &chunks { + let id = self + .send_chunk_with_fallback( + &bot, + account_id, + to, + chat_id, + thread_id, + chunk, + rp.as_ref(), + false, + ) + .await?; + ids.push(id.0.to_string()); + } + Ok(ids) + } + /// Send `text` with `suffix_html` appended, returning every message id it /// produced. /// @@ -264,28 +297,20 @@ impl ChannelOutbound for TelegramOutbound { html: &str, reply_to: Option<&str>, ) -> Result<()> { - let bot = self.get_bot(account_id)?; - let (chat_id, thread_id) = parse_chat_target(to)?; - let rp = self.reply_params(account_id, reply_to); - - // Send raw HTML chunks without markdown conversion. - let chunks = markdown::chunk_message(html, TELEGRAM_MAX_MESSAGE_LEN); - for chunk in &chunks { - self.send_chunk_with_fallback( - &bot, - account_id, - to, - chat_id, - thread_id, - chunk, - rp.as_ref(), - false, - ) - .await?; - } + self.send_html_ids(account_id, to, html, reply_to).await?; Ok(()) } + async fn send_html_reporting_ids( + &self, + account_id: &str, + to: &str, + html: &str, + reply_to: Option<&str>, + ) -> Result> { + self.send_html_ids(account_id, to, html, reply_to).await + } + async fn send_typing(&self, account_id: &str, to: &str) -> Result<()> { let bot = self.get_bot(account_id)?; let (chat_id, thread_id) = parse_chat_target(to)?; From 15f3459413a7f16dacea2baa604c2b74ad5b8a7a Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 02:05:46 -0400 Subject: [PATCH 22/33] docs(feedback): state the single-owner assumption behind the web score identity Review flagged the constant web identity as letting operators overwrite each other's scores. Moltis authenticates one account - a single password, no user table, passkeys registered as "owner" - and AuthIdentity carries only a method and scopes, so no per-person id exists to attribute to. Records that where the constant is chosen, and names the line that must change if Moltis ever grows real accounts. --- .../gateway/src/methods/services/feedback.rs | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs index edd986163..b53ad7a87 100644 --- a/crates/gateway/src/methods/services/feedback.rs +++ b/crates/gateway/src/methods/services/feedback.rs @@ -79,12 +79,23 @@ pub(super) fn register(reg: &mut MethodRegistry) { }, }; - // The score is attributed to the authenticated web operator, - // never to an identity named in the request: a caller-chosen - // `userId` would let one client overwrite or retract another's - // vote, since the score id is derived from (trace, name, user). - // A stable value is required for that upsert to keep working - // across reconnects, which rules out the connection id. + // The score is attributed to the authenticated operator, never + // to an identity named in the request: a caller-chosen `userId` + // would let one client overwrite or retract another's vote, + // since the score id is derived from (trace, name, user). A + // stable value is required for that upsert to keep working + // across reconnects, which also rules out the connection id. + // + // The web surface has exactly one identity because Moltis has + // exactly one account: a single password, no user table, and + // passkeys registered as "owner". `AuthIdentity` carries only a + // method and scopes, so there is no per-person id to derive + // here — everyone who authenticates *is* the owner, and one + // constant states that honestly rather than manufacturing a + // per-connection identity that would break the toggle. + // + // If Moltis ever grows real accounts, this is the line that has + // to change, and the score id derivation will follow. let user_id = moltis_channels::trace_link::WEB_CHANNEL; let outcome = ctx From 406ea3897d13d1a5092d6f4c356dbffc6c127393 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Mon, 27 Jul 2026 02:18:50 -0400 Subject: [PATCH 23/33] fix(feedback): attribute the logbook on every voice and streamed branch Four branches still sent the logbook follow-up through send_html, which reports no ids: the Telegram and generic voice paths, and both already-streamed paths. On each of them the answer went out as audio or was already streamed, so the logbook is the only text message of the turn and the most likely thing a reader reacts to. All four now share deliver_logbook_follow_up, which uses the id-reporting send and records the link. This is the fourth round of the same defect appearing in a different branch, so it is now pinned rather than fixed case by case: a test scans this module and fails if any delivery code calls send_text, send_text_with_suffix or send_html directly instead of the *_reporting_ids variant. Verified the guard fails on an injected violation rather than passing vacuously. --- crates/chat/src/channel_reply_delivery.rs | 226 ++++++++++++++++------ 1 file changed, 172 insertions(+), 54 deletions(-) diff --git a/crates/chat/src/channel_reply_delivery.rs b/crates/chat/src/channel_reply_delivery.rs index 7928e2deb..9aa561933 100644 --- a/crates/chat/src/channel_reply_delivery.rs +++ b/crates/chat/src/channel_reply_delivery.rs @@ -16,6 +16,51 @@ use crate::{ types::ReplyMedium, }; +/// Send the activity logbook as a follow-up message and link it to the trace. +/// +/// Reached from every branch where the answer itself went out some other way — +/// as voice audio, or already streamed edit-in-place — which leaves this as the +/// only text message of the turn and therefore a likely reaction target. Goes +/// through the id-reporting send for the same reason +/// [`deliver_text_reply`] does: the non-reporting one silently loses +/// attribution. +/// +/// A no-op when there is no logbook, so callers need no emptiness check. +async fn deliver_logbook_follow_up( + outbound: &Arc, + feedback: Option<&moltis_channels::FeedbackService>, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + logbook_html: &str, + session_key: &str, +) { + if logbook_html.is_empty() { + return; + } + match outbound + .send_html_reporting_ids(&target.account_id, to, logbook_html, None) + .await + { + Ok(message_ids) => { + crate::channel_feedback::record_reply_trace( + feedback, + target, + &message_ids, + session_key, + ) + .await; + }, + Err(e) => { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + }, + } +} + /// Send the reply text (optionally carrying an activity logbook) and link the /// messages it produced to the trace that wrote them. /// @@ -122,19 +167,17 @@ pub(crate) async fn deliver_channel_replies_to_targets( "failed to send channel voice reply: {e}" ); } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + ) + .await; } else { // Check if transcript fits as Telegram caption (when feature enabled). // When telegram feature is disabled, this evaluates to false and we @@ -159,19 +202,17 @@ pub(crate) async fn deliver_channel_replies_to_targets( "failed to send channel voice reply: {e}" ); } - // Send logbook as a follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + ) + .await; } else { // Transcript too long for a caption — send voice // without caption, then the full text as a follow-up. @@ -205,20 +246,17 @@ pub(crate) async fn deliver_channel_replies_to_targets( } }, None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + ) + .await; }, None => { deliver_text_reply( @@ -249,20 +287,17 @@ pub(crate) async fn deliver_channel_replies_to_targets( } }, None if text_already_streamed => { - // TTS disabled/failed but text was already streamed — - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } + // The answer went out some other way, so the logbook is the + // only text message of this turn. + deliver_logbook_follow_up( + &outbound, + feedback.as_deref(), + &target, + &to, + &logbook_html, + &session_key, + ) + .await; }, None => { deliver_text_reply( @@ -375,6 +410,34 @@ mod tests { .push("send_text_with_suffix_reporting_ids"); Ok(vec!["logbook-1".into()]) } + + async fn send_html( + &self, + _account_id: &str, + _to: &str, + _html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_html"); + Ok(()) + } + + async fn send_html_reporting_ids( + &self, + _account_id: &str, + _to: &str, + _html: &str, + _reply_to: Option<&str>, + ) -> moltis_channels::Result> { + self.calls + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push("send_html_reporting_ids"); + Ok(vec!["html-1".into()]) + } } fn reply_target() -> ChannelReplyTarget { @@ -391,6 +454,61 @@ mod tests { /// A reply carrying an activity logbook is still a reply someone can react /// to, so it must go out through the id-reporting suffix send rather than /// the plain one, which reports nothing and loses attribution. + /// The logbook follow-up is the only text message on the branches that + /// reach it, so it must go out through the id-reporting send. `send_html` + /// reports nothing and would leave the reaction unresolvable. + #[tokio::test] + async fn a_logbook_follow_up_uses_the_id_reporting_html_send() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_logbook_follow_up( + &outbound, + None, + &reply_target(), + "chan", + "
log
", + "session", + ) + .await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(*calls, vec!["send_html_reporting_ids"]); + } + + #[tokio::test] + async fn an_absent_logbook_sends_nothing() { + let recorder = Arc::new(ReportingOutbound::default()); + let outbound: Arc = recorder.clone(); + + deliver_logbook_follow_up(&outbound, None, &reply_target(), "chan", "", "session").await; + + let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); + assert!(calls.is_empty(), "expected no send, got {calls:?}"); + } + + /// Guards the invariant the whole module exists to hold: no delivery path + /// may reach for a send that drops message ids. A new branch that calls + /// `send_text`, `send_text_with_suffix` or `send_html` directly loses + /// feedback attribution silently, which is exactly the class of bug this + /// file kept reintroducing. + #[test] + fn no_delivery_branch_uses_a_non_reporting_send() { + let source = include_str!("channel_reply_delivery.rs"); + let body = source + .split("#[cfg(test)]") + .next() + .unwrap_or_default() + .to_string(); + for forbidden in [".send_text(", ".send_text_with_suffix(", ".send_html("] { + assert!( + !body.contains(forbidden), + "delivery code calls {forbidden} — use the *_reporting_ids variant \ + so the reply keeps its trace link" + ); + } + } + #[tokio::test] async fn a_logbook_reply_uses_the_id_reporting_suffix_send() { let recorder = Arc::new(ReportingOutbound::default()); From 01948d3a74025dd87e6d376107994443d9b5e79a Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 11:09:16 -0700 Subject: [PATCH 24/33] fix(observability): harden Langfuse v4 delivery Langfuse observations were exported with mutable or incomplete semantics, feedback attribution could reuse the wrong run, and provider usage/failover paths produced inconsistent telemetry. Shutdown and exporter failures could also silently lose buffered data. Export completed immutable v4 observations with canonical IDs, use the dedicated ordered Scores API, correlate feedback by run, normalize usage and fallback identity, strengthen redaction, and expose delivery health. Add cancellation-safe flushing, semantic validation, UI status, documentation, and coverage across streaming and non-streaming paths. --- crates/agents/src/model/mod.rs | 26 +- crates/agents/src/model/stream.rs | 72 ++ crates/agents/src/model/types.rs | 35 + crates/agents/src/provider_chain.rs | 267 ++++++- crates/agents/src/runner/instrumentation.rs | 266 ++++++- crates/agents/src/runner/non_streaming.rs | 144 ++-- crates/agents/src/runner/streaming.rs | 222 +++--- crates/agents/src/runner/tool_result.rs | 37 + crates/channels/src/feedback.rs | 69 +- crates/chat/src/agent_loop.rs | 7 + crates/chat/src/channel_feedback.rs | 15 +- crates/chat/src/channel_reply_delivery.rs | 28 +- crates/chat/src/channels.rs | 7 + crates/chat/src/run_with_tools.rs | 9 +- crates/chat/src/streaming.rs | 677 +++++++++++------- crates/config/src/schema/instrumentation.rs | 14 +- crates/config/src/template.rs | 28 +- crates/config/src/validate/semantic.rs | 105 +++ crates/config/src/validate/tests.rs | 2 + .../src/validate/tests/instrumentation.rs | 100 +++ .../gateway/src/methods/services/feedback.rs | 4 +- .../src/methods/services/instrumentation.rs | 135 +++- crates/gateway/src/server/instrumentation.rs | 252 +++++-- .../src/server/prepare_core/post_state.rs | 6 +- crates/httpd/src/server/runtime.rs | 12 +- crates/observability/src/builder.rs | 25 +- .../src/exporters/langfuse/config.rs | 49 +- .../src/exporters/langfuse/scores.rs | 361 ++++++---- .../src/exporters/otlp/mapping.rs | 294 ++++++-- .../observability/src/exporters/otlp/mod.rs | 4 + crates/observability/src/feedback.rs | 21 +- crates/observability/src/lib.rs | 9 +- crates/observability/src/model.rs | 83 ++- crates/observability/src/profile.rs | 27 - crates/observability/src/recorder.rs | 294 ++++++-- crates/observability/src/redact.rs | 22 +- crates/observability/src/runtime.rs | 133 +++- crates/observability/src/sink.rs | 20 +- crates/observability/tests/end_to_end.rs | 42 +- crates/providers/src/anthropic.rs | 22 +- crates/providers/src/async_openai_provider.rs | 24 +- crates/providers/src/genai_provider.rs | 61 +- .../src/openai/provider/completion.rs | 8 +- crates/providers/src/openai_codex.rs | 26 +- .../providers/src/openai_compat/provider.rs | 77 +- .../providers/src/openai_compat/tests/mod.rs | 45 +- crates/providers/src/registry/registration.rs | 3 + crates/swift-bridge/src/state.rs | 14 +- .../web/ui/e2e/specs/instrumentation.spec.js | 13 +- crates/web/ui/src/message-actions.ts | 2 +- crates/web/ui/src/message-feedback.ts | 2 +- .../pages/sections/InstrumentationSection.tsx | 75 +- docs/src/instrumentation.md | 105 +-- 53 files changed, 3437 insertions(+), 963 deletions(-) create mode 100644 crates/config/src/validate/tests/instrumentation.rs diff --git a/crates/agents/src/model/mod.rs b/crates/agents/src/model/mod.rs index 049c0920d..81fb528a9 100644 --- a/crates/agents/src/model/mod.rs +++ b/crates/agents/src/model/mod.rs @@ -5,8 +5,8 @@ pub use moltis_config::schema::{AgentToolControls, ReasoningEffort, ToolChoice}; mod types; pub use types::{ - CompletionResponse, MAX_CAPTURED_PROVIDER_RAW_EVENTS, ModelMetadata, TOOL_CALL_METADATA_KEYS, - ToolCall, ToolCallArgumentDiagnostic, ToolCallArgumentSource, Usage, + CompletionResponse, InputTokenAccounting, MAX_CAPTURED_PROVIDER_RAW_EVENTS, ModelMetadata, + TOOL_CALL_METADATA_KEYS, ToolCall, ToolCallArgumentDiagnostic, ToolCallArgumentSource, Usage, push_capped_provider_raw_event, }; @@ -20,7 +20,9 @@ pub use convert::{ }; mod stream; -pub use stream::{LlmProvider, StreamEvent}; +pub use stream::{ + LlmProvider, ProviderAttemptEvent, ProviderIdentity, StreamEvent, TrackedStreamEvent, +}; #[cfg(test)] fn document_absolute_path_from_media_ref(media_ref: &str) -> String { @@ -242,6 +244,24 @@ mod tests { assert_eq!(total.cache_write_tokens, 44); } + #[test] + fn usage_normalizes_inclusive_input_into_exclusive_buckets() { + let usage = Usage::from_input_tokens(InputTokenAccounting::Inclusive, 1_000, 50, 800, 20); + + assert_eq!(usage.input_tokens, 180); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); + } + + #[test] + fn usage_preserves_anthropic_exclusive_input() { + let usage = Usage::from_input_tokens(InputTokenAccounting::Exclusive, 180, 50, 800, 20); + + assert_eq!(usage.input_tokens, 180); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); + } + // ── to_openai_value ────────────────────────────────────────────── #[test] diff --git a/crates/agents/src/model/stream.rs b/crates/agents/src/model/stream.rs index 5f9846d0f..9d237cde0 100644 --- a/crates/agents/src/model/stream.rs +++ b/crates/agents/src/model/stream.rs @@ -8,6 +8,40 @@ use super::{ types::{CompletionResponse, ModelMetadata, Usage}, }; +/// Concrete provider and model selected for one completion attempt. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ProviderIdentity { + pub provider: String, + pub model: String, +} + +impl ProviderIdentity { + #[must_use] + pub fn new(provider: impl Into, model: impl Into) -> Self { + Self { + provider: provider.into(), + model: model.into(), + } + } +} + +/// Lifecycle events for attempts made by a provider wrapper. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ProviderAttemptEvent { + Started(ProviderIdentity), + Failed { + identity: ProviderIdentity, + error: String, + }, +} + +/// Event returned by the runner-facing tracked streaming API. +#[derive(Debug, Clone)] +pub enum TrackedStreamEvent { + Attempt(ProviderAttemptEvent), + Event(StreamEvent), +} + // ── Stream events ─────────────────────────────────────────────────────────── /// Events emitted during streaming LLM completion. @@ -72,6 +106,26 @@ pub trait LlmProvider: Send + Sync { self.complete(messages, tools).await } + /// Complete while reporting each concrete provider attempt. + async fn complete_with_options_tracked( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + on_attempt: &mut (dyn FnMut(ProviderAttemptEvent) + Send), + ) -> anyhow::Result { + let identity = ProviderIdentity::new(self.name(), self.id()); + on_attempt(ProviderAttemptEvent::Started(identity.clone())); + let result = self.complete_with_options(messages, tools, options).await; + if let Err(error) = &result { + on_attempt(ProviderAttemptEvent::Failed { + identity, + error: error.to_string(), + }); + } + result + } + /// Whether this provider supports tool/function calling. /// Defaults to false; providers that handle the `tools` parameter /// in `complete()` should override this to return true. @@ -134,6 +188,24 @@ pub trait LlmProvider: Send + Sync { self.stream_with_tools(messages, tools) } + /// Stream while reporting the concrete provider/model selected for the attempt. + fn stream_with_tools_and_options_tracked( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + let attempt = TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started( + ProviderIdentity::new(self.name(), self.id()), + )); + Box::pin( + tokio_stream::once(attempt).chain( + self.stream_with_tools_and_options(messages, tools, options) + .map(TrackedStreamEvent::Event), + ), + ) + } + /// Configured reasoning effort for this provider instance, if any. /// /// Providers that support extended thinking (Anthropic, OpenAI o-series) diff --git a/crates/agents/src/model/types.rs b/crates/agents/src/model/types.rs index 2b4a5eb4c..95267c397 100644 --- a/crates/agents/src/model/types.rs +++ b/crates/agents/src/model/types.rs @@ -71,13 +71,48 @@ pub const TOOL_CALL_METADATA_KEYS: &[&str] = &["thought_signature"]; #[derive(Debug, Clone, Default)] pub struct Usage { + /// Fresh input tokens, excluding cache reads and cache writes. pub input_tokens: u32, pub output_tokens: u32, pub cache_read_tokens: u32, pub cache_write_tokens: u32, } +/// Provider-reported input-token accounting semantics. +/// +/// OpenAI-style APIs report cached tokens as part of their input total, while +/// Anthropic reports fresh input, cache reads, and cache writes separately. +/// Provider adapters must identify which contract they received so `Usage` +/// always exposes mutually exclusive buckets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InputTokenAccounting { + Inclusive, + Exclusive, +} + impl Usage { + #[must_use] + pub const fn from_input_tokens( + accounting: InputTokenAccounting, + reported_input_tokens: u32, + output_tokens: u32, + cache_read_tokens: u32, + cache_write_tokens: u32, + ) -> Self { + let input_tokens = match accounting { + InputTokenAccounting::Inclusive => reported_input_tokens + .saturating_sub(cache_read_tokens) + .saturating_sub(cache_write_tokens), + InputTokenAccounting::Exclusive => reported_input_tokens, + }; + Self { + input_tokens, + output_tokens, + cache_read_tokens, + cache_write_tokens, + } + } + #[must_use] pub fn saturating_add(&self, other: &Self) -> Self { Self { diff --git a/crates/agents/src/provider_chain.rs b/crates/agents/src/provider_chain.rs index 04285a54c..f90916210 100644 --- a/crates/agents/src/provider_chain.rs +++ b/crates/agents/src/provider_chain.rs @@ -14,12 +14,19 @@ use std::{ time::{Duration, Instant}, }; -use {async_trait::async_trait, tokio_stream::Stream, tracing::warn}; +use { + async_trait::async_trait, + tokio_stream::{Stream, StreamExt}, + tracing::warn, +}; #[cfg(feature = "metrics")] use moltis_metrics::{counter, histogram, labels, llm as llm_metrics}; -use crate::model::{ChatMessage, CompletionResponse, LlmProvider, StreamEvent}; +use crate::model::{ + AgentToolControls, ChatMessage, CompletionResponse, LlmProvider, ProviderAttemptEvent, + ProviderIdentity, StreamEvent, TrackedStreamEvent, +}; /// How a provider error should be handled. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -212,6 +219,89 @@ impl ProviderChain { fn primary(&self) -> &ChainEntry { &self.chain[0] } + + fn failover_stream( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + Box::pin(async_stream::stream! { + let mut errors = Vec::new(); + + for entry in &self.chain { + if entry.state.is_tripped() { + continue; + } + + let identity = ProviderIdentity::new(entry.provider.name(), entry.provider.id()); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started( + identity.clone(), + )); + + let mut stream = entry.provider.stream_with_tools_and_options( + messages.clone(), + tools.clone(), + options.clone(), + ); + let mut emitted_output = false; + let mut retry_next = false; + + while let Some(event) = stream.next().await { + match event { + StreamEvent::Error(error) => { + let anyhow_error = anyhow::anyhow!(error.clone()); + let kind = classify_error(&anyhow_error); + entry.state.record_failure(); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { + identity: identity.clone(), + error: error.clone(), + }); + + if kind.should_failover() && !emitted_output { + warn!( + provider = entry.provider.id(), + error = %error, + kind = ?kind, + "provider stream failed, trying next in chain" + ); + errors.push(format!("{}: {error}", entry.provider.id())); + retry_next = true; + break; + } + + yield TrackedStreamEvent::Event(StreamEvent::Error(error)); + return; + }, + StreamEvent::Done(usage) => { + entry.state.record_success(); + yield TrackedStreamEvent::Event(StreamEvent::Done(usage)); + return; + }, + StreamEvent::ProviderRaw(raw) => { + yield TrackedStreamEvent::Event(StreamEvent::ProviderRaw(raw)); + }, + event => { + emitted_output = true; + yield TrackedStreamEvent::Event(event); + }, + } + } + + if retry_next { + continue; + } + + entry.state.record_success(); + return; + } + + yield TrackedStreamEvent::Event(StreamEvent::Error(format!( + "all providers in failover chain failed: {}", + errors.join("; ") + ))); + }) + } } #[async_trait] @@ -236,6 +326,28 @@ impl LlmProvider for ProviderChain { &self, messages: &[ChatMessage], tools: &[serde_json::Value], + ) -> anyhow::Result { + self.complete_with_options(messages, tools, &AgentToolControls::default()) + .await + } + + async fn complete_with_options( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + ) -> anyhow::Result { + let mut ignore_attempt = |_: ProviderAttemptEvent| {}; + self.complete_with_options_tracked(messages, tools, options, &mut ignore_attempt) + .await + } + + async fn complete_with_options_tracked( + &self, + messages: &[ChatMessage], + tools: &[serde_json::Value], + options: &AgentToolControls, + on_attempt: &mut (dyn FnMut(ProviderAttemptEvent) + Send), ) -> anyhow::Result { let mut errors = Vec::new(); #[cfg(feature = "metrics")] @@ -249,7 +361,11 @@ impl LlmProvider for ProviderChain { let provider_name = entry.provider.name().to_string(); let model_id = entry.provider.id().to_string(); - match entry.provider.complete(messages, tools).await { + match entry + .provider + .complete_with_options_tracked(messages, tools, options, on_attempt) + .await + { Ok(resp) => { entry.state.record_success(); @@ -353,16 +469,37 @@ impl LlmProvider for ProviderChain { messages: Vec, tools: Vec, ) -> Pin + Send + '_>> { - // For streaming, we try the first non-tripped provider. - // If the stream yields an Error event, we can't transparently retry mid-stream, - // so we pick the best available provider upfront. - for entry in &self.chain { - if !entry.state.is_tripped() { - return entry.provider.stream_with_tools(messages, tools); - } - } - // All tripped — try primary anyway (it may have cooled down by now). - self.primary().provider.stream_with_tools(messages, tools) + Box::pin( + self.failover_stream(messages, tools, AgentToolControls::default()) + .filter_map(|event| match event { + TrackedStreamEvent::Attempt(_) => None, + TrackedStreamEvent::Event(event) => Some(event), + }), + ) + } + + fn stream_with_tools_and_options( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + Box::pin( + self.failover_stream(messages, tools, options) + .filter_map(|event| match event { + TrackedStreamEvent::Attempt(_) => None, + TrackedStreamEvent::Event(event) => Some(event), + }), + ) + } + + fn stream_with_tools_and_options_tracked( + &self, + messages: Vec, + tools: Vec, + options: AgentToolControls, + ) -> Pin + Send + '_>> { + self.failover_stream(messages, tools, options) } } @@ -479,6 +616,38 @@ mod tests { assert_eq!(resp.text.as_deref(), Some("ok")); } + #[tokio::test] + async fn non_streaming_attempts_report_fallback_identity() { + let chain = ProviderChain::new(vec![ + Arc::new(FailingProvider { + id: "primary", + error_msg: "429 rate limit exceeded", + }), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + let mut attempts = Vec::new(); + + let response = chain + .complete_with_options_tracked( + &[], + &[], + &AgentToolControls::default(), + &mut |attempt| attempts.push(attempt), + ) + .await + .unwrap(); + + assert_eq!(response.text.as_deref(), Some("ok")); + assert_eq!(attempts, vec![ + ProviderAttemptEvent::Started(ProviderIdentity::new("failing", "primary")), + ProviderAttemptEvent::Failed { + identity: ProviderIdentity::new("failing", "primary"), + error: "429 rate limit exceeded".into(), + }, + ProviderAttemptEvent::Started(ProviderIdentity::new("success", "fallback")), + ]); + } + #[tokio::test] async fn failover_on_server_error() { let chain = ProviderChain::new(vec![ @@ -584,15 +753,79 @@ mod tests { Arc::new(SuccessProvider { id: "backup" }), ]); - // Trip the first provider. + // Trip the first provider through stream failures. for _ in 0..3 { - let _ = chain.complete(&[], &[]).await; + chain.stream(vec![]).collect::>().await; } + assert!(chain.chain[0].state.is_tripped()); // Stream should use backup. - let mut stream = chain.stream(vec![]); + let mut stream = chain.stream_with_tools_and_options_tracked( + vec![], + vec![], + AgentToolControls::default(), + ); let event = stream.next().await.unwrap(); - assert!(matches!(event, StreamEvent::Done(_))); + assert!(matches!( + event, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == ProviderIdentity::new("success", "backup") + )); + assert!(matches!( + stream.next().await, + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn stream_error_fails_over_and_records_attempts() { + let chain = ProviderChain::new(vec![ + Arc::new(FailingProvider { + id: "primary", + error_msg: "503 service unavailable", + }), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(matches!( + &events[0], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == &ProviderIdentity::new("failing", "primary") + )); + assert!(matches!( + &events[1], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { identity, error }) + if identity == &ProviderIdentity::new("failing", "primary") + && error == "503 service unavailable" + )); + assert!(matches!( + &events[2], + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) + if identity == &ProviderIdentity::new("success", "fallback") + )); + assert!(matches!( + events[3], + TrackedStreamEvent::Event(StreamEvent::Done(_)) + )); + assert_eq!( + chain.chain[0] + .state + .consecutive_failures + .load(Ordering::SeqCst), + 1 + ); + assert_eq!( + chain.chain[1] + .state + .consecutive_failures + .load(Ordering::SeqCst), + 0 + ); } #[test] diff --git a/crates/agents/src/runner/instrumentation.rs b/crates/agents/src/runner/instrumentation.rs index 2ffc1ccea..2eb6d130d 100644 --- a/crates/agents/src/runner/instrumentation.rs +++ b/crates/agents/src/runner/instrumentation.rs @@ -11,7 +11,10 @@ use { }, }; -use crate::model::{Usage, UserContent}; +use crate::{ + model::{AgentToolControls, ChatMessage, LlmProvider, ProviderIdentity, Usage, UserContent}, + runner::{AgentRunError, AgentRunResult, tool_result::tool_result_failure}, +}; /// Derive the trace scope from the session and channel context. /// @@ -98,6 +101,7 @@ pub fn release(config: &moltis_config::InstrumentationConfig) -> String { #[must_use] pub fn begin_turn( session_key: &str, + trace_correlation_key: Option<&str>, channel: Option<&ChannelBinding>, provider: &str, model: &str, @@ -109,9 +113,12 @@ pub fn begin_turn( let scope = trace_scope(session_key, channel, environment, release); let recorder = TurnRecorder::begin("agent-run", scope, settings)?; - // The reply for this turn is sent by a dispatcher far from here, and needs - // the trace id to attribute a later reaction to the right run. - moltis_observability::remember_trace(session_key, recorder.trace_id()); + // Only top-level chat runs provide a correlation key. Sub-agents deliberately + // omit it, so they cannot overwrite the trace used to attribute the parent + // run's delivered reply. + if let Some(key) = trace_correlation_key { + moltis_observability::remember_trace(key, recorder.trace_id()); + } recorder.set_metadata("provider", serde_json::Value::String(provider.to_string())); recorder.set_metadata("model", serde_json::Value::String(model.to_string())); @@ -157,6 +164,111 @@ pub fn generation_name(provider: &str, model: &str) -> String { format!("{provider}/{model}") } +/// Attach request parameters already exposed by the provider and agent model. +pub fn set_generation_parameters( + step: &mut moltis_observability::StepGuard, + provider: &dyn LlmProvider, + controls: &AgentToolControls, +) { + if let Some(effort) = provider.reasoning_effort() { + step.set_model_parameter("reasoning_effort", serde_json::json!(effort.as_str())); + } + if let Some(tool_choice) = &controls.tool_choice { + step.set_model_parameter("tool_choice", serde_json::json!(tool_choice)); + } +} + +/// Open a generation observation for the concrete provider selected for an attempt. +pub fn begin_generation_step( + recorder: Option<&TurnRecorder>, + identity: &ProviderIdentity, + provider: &dyn LlmProvider, + controls: &AgentToolControls, + iteration: usize, + tool_count: usize, + messages: &[ChatMessage], +) -> Option { + recorder.map(|recorder| { + let mut step = recorder.step( + ObservationKind::Generation, + generation_name(&identity.provider, &identity.model), + ); + step.set_model(identity.model.clone()); + step.set_metadata("provider", serde_json::json!(identity.provider)); + set_generation_parameters(&mut step, provider, controls); + step.set_metadata("iteration", serde_json::json!(iteration)); + step.set_metadata("tool_count", serde_json::json!(tool_count)); + step.set_input(serde_json::Value::Array( + messages.iter().map(ChatMessage::to_openai_value).collect(), + )); + step + }) +} + +/// Extract a provider's terminal reason from a raw streaming event. +#[must_use] +pub fn finish_reason(raw: &serde_json::Value) -> Option { + raw.pointer("/choices/0/finish_reason") + .or_else(|| raw.get("stop_reason")) + .or_else(|| raw.pointer("/response/status")) + .and_then(serde_json::Value::as_str) + .map(str::to_string) +} + +/// Extract separately reported reasoning-token usage from a raw event. +#[must_use] +pub fn reasoning_tokens(raw: &serde_json::Value) -> Option { + [ + "/usage/completion_tokens_details/reasoning_tokens", + "/usage/output_tokens_details/reasoning_tokens", + "/response/usage/output_tokens_details/reasoning_tokens", + "/choices/0/usage/completion_tokens_details/reasoning_tokens", + ] + .iter() + .find_map(|pointer| raw.pointer(pointer).and_then(serde_json::Value::as_u64)) + .map(|tokens| u32::try_from(tokens).unwrap_or(u32::MAX)) +} + +/// Close the trace root for every return path from an agent loop. +pub fn finish_turn( + recorder: Option<&TurnRecorder>, + result: &Result, +) { + let Some(recorder) = recorder else { + return; + }; + match result { + Ok(run) => { + recorder.set_output(serde_json::Value::String(run.text.clone())); + recorder.set_metadata("iterations", serde_json::json!(run.iterations)); + recorder.set_metadata("tool_calls", serde_json::json!(run.tool_calls_made)); + recorder.finish(); + }, + Err(error) => recorder.finish_with_error(error.to_string()), + } +} + +/// Close a tool observation with the exact content persisted for the model. +pub fn finish_tool_step( + step: Option, + persisted_output: &str, + success: bool, + error: Option<&str>, + persisted_result: &serde_json::Value, +) { + let Some(mut step) = step else { + return; + }; + step.set_output(serde_json::Value::String(persisted_output.to_string())); + let failure = error + .map(str::to_string) + .or_else(|| tool_result_failure(persisted_result)); + if !success || failure.is_some() { + step.fail(failure.unwrap_or_else(|| "tool execution failed".to_string())); + } + step.finish(); +} + /// The observation kind a tool call should be recorded as. /// /// Retrieval tools are reported as `RETRIEVER` so backends that special-case @@ -179,9 +291,51 @@ pub fn tool_observation_kind(tool_name: &str) -> ObservationKind { } #[cfg(test)] +#[allow(clippy::expect_used)] mod tests { + use std::{ + sync::{Arc, Mutex}, + time::Duration, + }; + + use moltis_observability::{Event, ObservationSink, RecorderSettings, RedactionPolicy}; + use super::*; + #[derive(Default)] + struct CollectingSink { + events: Mutex>, + } + + #[async_trait::async_trait] + impl ObservationSink for CollectingSink { + fn name(&self) -> &str { + "agent-test" + } + + fn record(&self, event: Event) { + self.events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(event); + } + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + Ok(()) + } + } + + fn test_recorder(sink: Arc) -> TurnRecorder { + TurnRecorder::begin_with_sink(sink, "agent-run", TraceScope::default(), RecorderSettings { + redaction: RedactionPolicy::default(), + capture_input: true, + capture_output: true, + capture_tool_io: true, + sample_rate: 1.0, + }) + .expect("sampled recorder") + } + fn binding() -> ChannelBinding { ChannelBinding { surface: Some("chat".into()), @@ -323,4 +477,108 @@ mod tests { "anthropic/claude-opus-4" ); } + + #[test] + fn finish_reason_supports_chat_anthropic_and_responses_shapes() { + assert_eq!( + finish_reason(&serde_json::json!({"choices": [{"finish_reason": "tool_calls"}]})) + .as_deref(), + Some("tool_calls") + ); + assert_eq!( + finish_reason(&serde_json::json!({"stop_reason": "end_turn"})).as_deref(), + Some("end_turn") + ); + assert_eq!( + finish_reason(&serde_json::json!({"response": {"status": "completed"}})).as_deref(), + Some("completed") + ); + } + + #[test] + fn reasoning_tokens_supports_chat_and_responses_usage() { + assert_eq!( + reasoning_tokens(&serde_json::json!({ + "usage": {"completion_tokens_details": {"reasoning_tokens": 42}} + })), + Some(42) + ); + assert_eq!( + reasoning_tokens(&serde_json::json!({ + "response": {"usage": {"output_tokens_details": {"reasoning_tokens": 84}}} + })), + Some(84) + ); + } + + #[test] + fn collecting_sink_receives_successful_and_failed_turn_completions() { + let sink = Arc::new(CollectingSink::default()); + let successful = test_recorder(Arc::clone(&sink)); + finish_turn( + Some(&successful), + &Ok(AgentRunResult { + text: "finished".into(), + iterations: 2, + tool_calls_made: 1, + usage: Usage::default(), + request_usage: Usage::default(), + raw_llm_responses: Vec::new(), + }), + ); + + let failed = test_recorder(Arc::clone(&sink)); + finish_turn( + Some(&failed), + &Err(AgentRunError::Other(anyhow::anyhow!("provider failed"))), + ); + + let traces = sink + .events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .iter() + .filter_map(|event| match event { + Event::Trace(trace) => Some((**trace).clone()), + _ => None, + }) + .collect::>(); + assert!(traces.iter().any(|trace| { + trace.output == Some(serde_json::Value::String("finished".into())) + && trace.metadata.get("iterations") == Some(&serde_json::json!(2)) + })); + assert!(traces.iter().any(|trace| { + trace.metadata.get("error") == Some(&serde_json::json!("provider failed")) + })); + } + + #[test] + fn collecting_sink_gets_persisted_tool_output_and_logical_failure() { + let sink = Arc::new(CollectingSink::default()); + let recorder = test_recorder(Arc::clone(&sink)); + let mut step = recorder.step(ObservationKind::Tool, "browser"); + step.set_input(serde_json::json!({"url": "https://example.com"})); + let persisted = serde_json::json!({"result": {"success": false, "secret": "[removed]"}}); + let output = persisted.to_string(); + + finish_tool_step(Some(step), &output, true, None, &persisted); + + let events = sink + .events + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let tool = events.iter().find_map(|event| match event { + Event::ObservationEnd(observation) if observation.kind == ObservationKind::Tool => { + Some(observation) + }, + _ => None, + }); + let tool = tool.expect("completed tool observation"); + assert_eq!(tool.output, Some(serde_json::Value::String(output))); + assert_eq!(tool.level, moltis_observability::Level::Error); + assert_eq!( + tool.status_message.as_deref(), + Some("tool returned success: false") + ); + } } diff --git a/crates/agents/src/runner/non_streaming.rs b/crates/agents/src/runner/non_streaming.rs index 56e026ad6..f21b1bcef 100644 --- a/crates/agents/src/runner/non_streaming.rs +++ b/crates/agents/src/runner/non_streaming.rs @@ -10,7 +10,10 @@ use { use moltis_common::hooks::{HookAction, HookPayload, HookRegistry}; use crate::{ - model::{AgentToolControls, ChatMessage, LlmProvider, ToolCall, ToolChoice, UserContent}, + model::{ + AgentToolControls, ChatMessage, LlmProvider, ProviderAttemptEvent, ToolCall, ToolChoice, + UserContent, + }, response_sanitizer::recover_tool_calls_from_content, tool_arg_validator::{coerce_scalar_args, validate_tool_args}, tool_loop_detector::ToolCallFingerprint, @@ -154,6 +157,10 @@ pub async fn run_agent_loop_with_context_and_limits( .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); + let trace_correlation_key = tool_context + .as_ref() + .and_then(|ctx| ctx.get("_trace_correlation_key")) + .and_then(|value| value.as_str()); let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); @@ -162,6 +169,7 @@ pub async fn run_agent_loop_with_context_and_limits( // traces the observation taxonomy exists to represent. let turn_recorder = Arc::new(instrumentation::begin_turn( &session_key_for_hooks, + trace_correlation_key, channel_for_hooks.as_ref(), provider.name(), provider.id(), @@ -171,6 +179,7 @@ pub async fn run_agent_loop_with_context_and_limits( instrumentation::release(&config.instrumentation), )); + let result: std::result::Result = async { dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -302,23 +311,48 @@ pub async fn run_agent_loop_with_context_and_limits( cb(RunnerEvent::Thinking); } - let mut generation_step = turn_recorder.as_ref().as_ref().map(|recorder| { - let mut step = recorder.step( - moltis_observability::ObservationKind::Generation, - instrumentation::generation_name(provider.name(), provider.id()), - ); - step.set_model(provider.id()); - step.set_metadata("iteration", serde_json::json!(iterations)); - step.set_input(serde_json::Value::Array( - messages.iter().map(ChatMessage::to_openai_value).collect(), - )); - step - }); + let mut generation_step = None; + let completion_result = { + let mut on_attempt = |event| match event { + ProviderAttemptEvent::Started(identity) => { + if let Some(recorder) = turn_recorder.as_ref().as_ref() { + recorder.set_metadata( + "provider", + serde_json::Value::String(identity.provider.clone()), + ); + recorder.set_metadata( + "model", + serde_json::Value::String(identity.model.clone()), + ); + } + generation_step = instrumentation::begin_generation_step( + turn_recorder.as_ref().as_ref(), + &identity, + provider.as_ref(), + &tool_controls, + iterations, + schemas_for_api.len(), + &messages, + ); + }, + ProviderAttemptEvent::Failed { error, .. } => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + step.finish(); + } + }, + }; - let mut response = match provider - .complete_with_options(&messages, &schemas_for_api, &tool_controls) - .await - { + provider + .complete_with_options_tracked( + &messages, + &schemas_for_api, + &tool_controls, + &mut on_attempt, + ) + .await + }; + let mut response = match completion_result { Ok(r) => r, Err(e) => { let msg = e.to_string(); @@ -364,9 +398,13 @@ pub async fn run_agent_loop_with_context_and_limits( if let Some(mut step) = generation_step.take() { step.set_usage(instrumentation::to_token_usage(&response.usage)); - if let Some(text) = response.text.clone() { - step.set_output(serde_json::Value::String(text)); - } + step.set_output(serde_json::json!({ + "text": response.text, + "tool_calls": response.tool_calls.iter().map(|call| serde_json::json!({ + "id": call.id, + "name": call.name, + })).collect::>(), + })); if !response.tool_calls.is_empty() { step.set_metadata("tool_calls", serde_json::json!(response.tool_calls.len())); } @@ -587,6 +625,7 @@ pub async fn run_agent_loop_with_context_and_limits( .tool_calls .iter() .map(|tc| { + let tool_turn_recorder = Arc::clone(&turn_recorder); let sanitized = sanitize_tool_name(&tc.name); if *sanitized != tc.name { debug!(original = %tc.name, sanitized = %sanitized, "sanitized mangled tool name"); @@ -668,14 +707,27 @@ pub async fn run_agent_loop_with_context_and_limits( } async move { + let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { + recorder.step( + instrumentation::tool_observation_kind(&tc_name), + tc_name.clone(), + ) + }); if let Some(err_msg) = validation_error { + if let Some(step) = tool_step.as_mut() { + step.set_input(args); + } return ( false, serde_json::json!({ "error": err_msg.clone() }), Some(err_msg), true, + tool_step, ); } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } // Run BeforeToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::BeforeToolCall { @@ -693,6 +745,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ); }, Ok(HookAction::ModifyPayload(v)) => { @@ -712,6 +765,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str.clone() }), Some(err_str), false, + tool_step, ); } } @@ -723,27 +777,19 @@ pub async fn run_agent_loop_with_context_and_limits( } } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } + if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { - // Check if the result indicates a logical failure - // (e.g., BrowserResponse with success: false) - let has_error = val.get("error").is_some() - || val.get("success") == Some(&serde_json::json!(false)); - let error_msg = if has_error { - val.get("error") - .and_then(|e| e.as_str()) - .map(String::from) - } else { - None - }; - // Dispatch AfterToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: !has_error, + success: true, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -752,12 +798,13 @@ pub async fn run_agent_loop_with_context_and_limits( } } - if has_error { - // Tool executed but returned an error in the result - (false, serde_json::json!({ "result": val }), error_msg, false) - } else { - (true, serde_json::json!({ "result": val }), None, false) - } + ( + true, + serde_json::json!({ "result": val }), + None, + false, + tool_step, + ) }, Err(e) => { let err_str = e.to_string(); @@ -779,6 +826,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) }, } @@ -789,6 +837,7 @@ pub async fn run_agent_loop_with_context_and_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } @@ -808,7 +857,8 @@ pub async fn run_agent_loop_with_context_and_limits( // stale intervention — the reset() abandons it cleanly. // 2. A batch that races through both escalation stages must still // deliver the stage-1 nudge first, not skip straight to strip. - for (tc, (success, mut result, error, rejected)) in response.tool_calls.iter().zip(results) + for (tc, (success, mut result, error, rejected, mut tool_step)) in + response.tool_calls.iter().zip(results) { if success { info!(tool = %tc.name, id = %tc.id, "tool execution succeeded"); @@ -848,7 +898,7 @@ pub async fn run_agent_loop_with_context_and_limits( id: tc.id.clone(), name: tc.name.clone(), success, - error, + error: error.clone(), result: if success { result.get("result").cloned() } else { @@ -893,6 +943,13 @@ pub async fn run_agent_loop_with_context_and_limits( // multimodal content in tool results. Images are stripped but the UI // still receives them via ToolCallEnd event. let tool_result_str = sanitize_tool_result(&result.to_string(), max_tool_result_bytes); + instrumentation::finish_tool_step( + tool_step.take(), + &tool_result_str, + success, + error.as_deref(), + &result, + ); debug!( tool = %tc.name, id = %tc.id, @@ -912,6 +969,11 @@ pub async fn run_agent_loop_with_context_and_limits( on_event, ); } + } + .await; + + instrumentation::finish_turn(turn_recorder.as_ref().as_ref(), &result); + result } /// Convenience wrapper matching the old stub signature. diff --git a/crates/agents/src/runner/streaming.rs b/crates/agents/src/runner/streaming.rs index 478de87fb..7aae1afa6 100644 --- a/crates/agents/src/runner/streaming.rs +++ b/crates/agents/src/runner/streaming.rs @@ -16,8 +16,9 @@ use moltis_common::hooks::{HookAction, HookPayload, HookRegistry}; use crate::{ model::{ - AgentToolControls, ChatMessage, LlmProvider, StreamEvent, ToolCall, ToolChoice, Usage, - UserContent, decode_tool_call_arguments_from_str, push_capped_provider_raw_event, + AgentToolControls, ChatMessage, LlmProvider, ProviderAttemptEvent, ProviderIdentity, + StreamEvent, ToolCall, ToolChoice, TrackedStreamEvent, Usage, UserContent, + decode_tool_call_arguments_from_str, push_capped_provider_raw_event, }, response_sanitizer::{clean_response, recover_tool_calls_from_content}, tool_arg_validator::{coerce_scalar_args, validate_tool_args}, @@ -144,6 +145,10 @@ pub async fn run_agent_loop_streaming_with_limits( .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); + let trace_correlation_key = tool_context + .as_ref() + .and_then(|ctx| ctx.get("_trace_correlation_key")) + .and_then(|value| value.as_str()); let channel_for_hooks = channel_binding_from_tool_context(&session_key_for_hooks, tool_context.as_ref()); @@ -151,6 +156,7 @@ pub async fn run_agent_loop_streaming_with_limits( // not sampled, in which case every call below is skipped. let turn_recorder = Arc::new(instrumentation::begin_turn( &session_key_for_hooks, + trace_correlation_key, channel_for_hooks.as_ref(), provider.name(), provider.id(), @@ -160,6 +166,7 @@ pub async fn run_agent_loop_streaming_with_limits( instrumentation::release(&config.instrumentation), )); + let result: std::result::Result = async { dispatch_before_agent_start_hook( hook_registry.as_ref(), &session_key_for_hooks, @@ -302,21 +309,10 @@ pub async fn run_agent_loop_streaming_with_limits( // Use streaming API. #[cfg(feature = "metrics")] let iter_start = std::time::Instant::now(); - let mut generation_step = turn_recorder.as_ref().as_ref().map(|recorder| { - let mut step = recorder.step( - moltis_observability::ObservationKind::Generation, - instrumentation::generation_name(provider.name(), provider.id()), - ); - step.set_model(provider.id()); - step.set_metadata("iteration", serde_json::json!(iterations)); - step.set_metadata("tool_count", serde_json::json!(schemas_for_api.len())); - step.set_input(serde_json::Value::Array( - messages.iter().map(ChatMessage::to_openai_value).collect(), - )); - step - }); + let mut generation_step = None; + let mut selected_identity: Option = None; - let mut stream = provider.stream_with_tools_and_options( + let mut stream = provider.stream_with_tools_and_options_tracked( messages.clone(), schemas_for_api.clone(), tool_controls.clone(), @@ -337,10 +333,46 @@ pub async fn run_agent_loop_streaming_with_limits( std::collections::HashMap::new(); let mut request_usage = Usage::default(); let mut stream_error: Option = None; + let mut finish_reason: Option = None; + let mut reasoning_tokens: u32 = 0; while let Some(event) = stream.next().await { match event { - StreamEvent::Delta(text) => { + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) => { + if let Some(recorder) = turn_recorder.as_ref().as_ref() { + recorder.set_metadata( + "provider", + serde_json::Value::String(identity.provider.clone()), + ); + recorder.set_metadata( + "model", + serde_json::Value::String(identity.model.clone()), + ); + } + generation_step = instrumentation::begin_generation_step( + turn_recorder.as_ref().as_ref(), + &identity, + provider.as_ref(), + &tool_controls, + iterations, + schemas_for_api.len(), + &messages, + ); + selected_identity = Some(identity); + }, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { error, .. }) => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); + step.finish(); + } + request_usage = Usage::default(); + finish_reason = None; + reasoning_tokens = 0; + }, + TrackedStreamEvent::Event(StreamEvent::Delta(text)) => { if let Some(step) = generation_step.as_mut() { step.mark_first_token(); } @@ -350,21 +382,33 @@ pub async fn run_agent_loop_streaming_with_limits( cb(RunnerEvent::FinalText(text)); } }, - StreamEvent::ProviderRaw(raw) => { + TrackedStreamEvent::Event(StreamEvent::ProviderRaw(raw)) => { + if let Some(reason) = instrumentation::finish_reason(&raw) { + finish_reason = Some(reason); + } + if let Some(tokens) = instrumentation::reasoning_tokens(&raw) { + reasoning_tokens = tokens; + } push_capped_provider_raw_event(&mut raw_llm_responses, raw); }, - StreamEvent::ReasoningDelta(text) => { + TrackedStreamEvent::Event(StreamEvent::ReasoningDelta(text)) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } accumulated_reasoning.push_str(&text); if let Some(cb) = on_event { cb(RunnerEvent::ThinkingText(accumulated_reasoning.clone())); } }, - StreamEvent::ToolCallStart { + TrackedStreamEvent::Event(StreamEvent::ToolCallStart { id, name, index, metadata, - } => { + }) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } let vec_pos = tool_calls.len(); debug!(tool = %name, id = %id, stream_index = index, vec_pos, "tool call started in stream"); tool_calls.push(ToolCall { @@ -377,17 +421,23 @@ pub async fn run_agent_loop_streaming_with_limits( stream_idx_to_vec_pos.insert(index, vec_pos); tool_call_args.insert(index, String::new()); }, - StreamEvent::ToolCallArgumentsDelta { index, delta } => { + TrackedStreamEvent::Event(StreamEvent::ToolCallArgumentsDelta { + index, + delta, + }) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } if let Some(args) = tool_call_args.get_mut(&index) { args.push_str(&delta); } }, - StreamEvent::ToolCallComplete { index } => { + TrackedStreamEvent::Event(StreamEvent::ToolCallComplete { index }) => { // Arguments are finalized after stream completes. // Just log for now - we'll parse accumulated args later. debug!(index, "tool call arguments complete"); }, - StreamEvent::Done(usage) => { + TrackedStreamEvent::Event(StreamEvent::Done(usage)) => { request_usage = usage.clone(); debug!( input_tokens = request_usage.input_tokens, @@ -399,8 +449,12 @@ pub async fn run_agent_loop_streaming_with_limits( #[cfg(feature = "metrics")] { - let provider_name = provider.name().to_string(); - let model_id = provider.id().to_string(); + let provider_name = selected_identity + .as_ref() + .map_or_else(|| provider.name().to_string(), |id| id.provider.clone()); + let model_id = selected_identity + .as_ref() + .map_or_else(|| provider.id().to_string(), |id| id.model.clone()); let duration = iter_start.elapsed().as_secs_f64(); counter!( llm_metrics::COMPLETIONS_TOTAL, @@ -440,7 +494,7 @@ pub async fn run_agent_loop_streaming_with_limits( .record(duration); } }, - StreamEvent::Error(msg) => { + TrackedStreamEvent::Event(StreamEvent::Error(msg)) => { stream_error = Some(msg); break; }, @@ -458,7 +512,9 @@ pub async fn run_agent_loop_streaming_with_limits( // ending as a span with no error on it. if let Some(mut step) = generation_step.take() { step.fail(err.clone()); - step.set_usage(instrumentation::to_token_usage(&request_usage)); + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); step.finish(); } if is_context_window_error(&err) { @@ -492,13 +548,23 @@ pub async fn run_agent_loop_streaming_with_limits( } if let Some(mut step) = generation_step.take() { - step.set_usage(instrumentation::to_token_usage(&request_usage)); - if !accumulated_text.is_empty() { - step.set_output(serde_json::Value::String(accumulated_text.clone())); - } + let mut usage = instrumentation::to_token_usage(&request_usage); + usage.reasoning = reasoning_tokens; + step.set_usage(usage); + step.set_output(serde_json::json!({ + "text": accumulated_text, + "reasoning": accumulated_reasoning, + "tool_calls": tool_calls.iter().map(|call| serde_json::json!({ + "id": call.id, + "name": call.name, + })).collect::>(), + })); if !tool_calls.is_empty() { step.set_metadata("tool_calls", serde_json::json!(tool_calls.len())); } + if let Some(reason) = finish_reason { + step.set_metadata("finish_reason", serde_json::Value::String(reason)); + } step.finish(); } @@ -685,12 +751,6 @@ pub async fn run_agent_loop_streaming_with_limits( tool_calls = total_tool_calls, "streaming agent loop complete — returning text" ); - if let Some(recorder) = turn_recorder.as_ref() { - recorder.set_output(serde_json::Value::String(final_text.clone())); - recorder.set_metadata("iterations", serde_json::json!(iterations)); - recorder.set_metadata("tool_calls", serde_json::json!(total_tool_calls)); - recorder.finish(); - } return Ok(finish_agent_run( final_text, iterations, @@ -821,14 +881,27 @@ pub async fn run_agent_loop_streaming_with_limits( } async move { + let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { + recorder.step( + instrumentation::tool_observation_kind(&tc_name), + tc_name.clone(), + ) + }); if let Some(err_msg) = validation_error { + if let Some(step) = tool_step.as_mut() { + step.set_input(args); + } return ( false, serde_json::json!({ "error": err_msg.clone() }), Some(err_msg), true, + tool_step, ); } + if let Some(step) = tool_step.as_mut() { + step.set_input(args.clone()); + } // Run BeforeToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::BeforeToolCall { @@ -846,6 +919,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ); } Ok(HookAction::ModifyPayload(v)) => { @@ -865,6 +939,7 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str.clone() }), Some(err_str), false, + tool_step, ); } } @@ -876,35 +951,18 @@ pub async fn run_agent_loop_streaming_with_limits( } } - let mut tool_step = tool_turn_recorder.as_ref().as_ref().map(|recorder| { - let mut step = recorder.step( - instrumentation::tool_observation_kind(&tc_name), - tc_name.clone(), - ); + if let Some(step) = tool_step.as_mut() { step.set_input(args.clone()); - step - }); + } if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { - // Check if the result indicates a logical failure - // (e.g., BrowserResponse with success: false) - let has_error = val.get("error").is_some() - || val.get("success") == Some(&serde_json::json!(false)); - let error_msg = if has_error { - val.get("error") - .and_then(|e| e.as_str()) - .map(String::from) - } else { - None - }; - if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: !has_error, + success: true, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -913,26 +971,16 @@ pub async fn run_agent_loop_streaming_with_limits( } } - if let Some(mut step) = tool_step.take() { - step.set_output(val.clone()); - if let Some(message) = error_msg.clone() { - step.fail(message); - } - step.finish(); - } - - if has_error { - (false, serde_json::json!({ "result": val }), error_msg, false) - } else { - (true, serde_json::json!({ "result": val }), None, false) - } + ( + true, + serde_json::json!({ "result": val }), + None, + false, + tool_step, + ) } Err(e) => { let err_str = e.to_string(); - if let Some(mut step) = tool_step.take() { - step.fail(err_str.clone()); - step.finish(); - } if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), @@ -950,20 +998,18 @@ pub async fn run_agent_loop_streaming_with_limits( serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } } else { let err_str = format!("unknown tool: {tc_name}"); - if let Some(mut step) = tool_step.take() { - step.fail(err_str.clone()); - step.finish(); - } ( false, serde_json::json!({ "error": err_str }), Some(err_str), false, + tool_step, ) } } @@ -977,7 +1023,9 @@ pub async fn run_agent_loop_streaming_with_limits( // Intervention is derived from the detector's post-batch state // via `consume_pending_action()` below — see the non-streaming // path for the rationale (issue #658). - for (tc, (success, mut result, error, rejected)) in tool_calls.iter().zip(results) { + for (tc, (success, mut result, error, rejected, mut tool_step)) in + tool_calls.iter().zip(results) + { if success { info!(tool = %tc.name, id = %tc.id, "tool execution succeeded"); trace!(tool = %tc.name, result = %result, "tool result"); @@ -1014,7 +1062,7 @@ pub async fn run_agent_loop_streaming_with_limits( id: tc.id.clone(), name: tc.name.clone(), success, - error, + error: error.clone(), result: if success { result.get("result").cloned() } else { @@ -1059,6 +1107,13 @@ pub async fn run_agent_loop_streaming_with_limits( // multimodal content in tool results. Images are stripped but the UI // still receives them via ToolCallEnd event. let tool_result_str = sanitize_tool_result(&result.to_string(), max_tool_result_bytes); + instrumentation::finish_tool_step( + tool_step.take(), + &tool_result_str, + success, + error.as_deref(), + &result, + ); debug!( tool = %tc.name, id = %tc.id, @@ -1092,4 +1147,9 @@ pub async fn run_agent_loop_streaming_with_limits( } } } + } + .await; + + instrumentation::finish_turn(turn_recorder.as_ref().as_ref(), &result); + result } diff --git a/crates/agents/src/runner/tool_result.rs b/crates/agents/src/runner/tool_result.rs index a7fc8ada3..d92df1aaa 100644 --- a/crates/agents/src/runner/tool_result.rs +++ b/crates/agents/src/runner/tool_result.rs @@ -103,6 +103,43 @@ pub fn sanitize_tool_result(input: &str, max_bytes: usize) -> String { result } +/// Return a stable failure message for tool results that encode logical errors. +#[must_use] +pub fn tool_result_failure(result: &serde_json::Value) -> Option { + let payload = result.get("result").unwrap_or(result); + let error = payload.get("error").filter(|value| !value.is_null()); + if let Some(error) = error { + return Some( + error + .as_str() + .map_or_else(|| error.to_string(), str::to_string), + ); + } + (payload.get("success") == Some(&serde_json::Value::Bool(false))) + .then(|| "tool returned success: false".to_string()) +} + +#[cfg(test)] +mod failure_tests { + use super::tool_result_failure; + + #[test] + fn success_false_without_error_has_a_failure_message() { + assert_eq!( + tool_result_failure(&serde_json::json!({"success": false})).as_deref(), + Some("tool returned success: false") + ); + } + + #[test] + fn wrapped_tool_errors_are_detected() { + assert_eq!( + tool_result_failure(&serde_json::json!({"result": {"error": "denied"}})).as_deref(), + Some("denied") + ); + } +} + // ── Multimodal tool result helpers ───────────────────────────────────────── /// Image extracted from a tool result for multimodal handling. diff --git a/crates/channels/src/feedback.rs b/crates/channels/src/feedback.rs index b0c3880d5..b374c185f 100644 --- a/crates/channels/src/feedback.rs +++ b/crates/channels/src/feedback.rs @@ -10,8 +10,8 @@ use std::sync::{Arc, RwLock}; use { moltis_config::FeedbackSettings, moltis_observability::{ - FeedbackSignal, FeedbackVocabulary, TraceId, exporters::langfuse::LangfuseClient, - feedback_score, feedback_score_id, + FeedbackSignal, FeedbackVocabulary, ScoreDeleteRecord, TraceId, + exporters::langfuse::LangfuseClient, feedback_score, feedback_score_id, }, tracing::{debug, warn}, }; @@ -164,6 +164,9 @@ impl FeedbackService { let Some(signal) = signal else { return FeedbackOutcome::NotFeedback; }; + if langfuse.is_none() { + return FeedbackOutcome::Disabled; + } let link = match links.lookup(channel, account_id, chat_id, message_id).await { Ok(Some(link)) => link, @@ -198,13 +201,9 @@ impl FeedbackService { } let score_id = feedback_score_id(&trace_id, Some(&scoped_user)); - let Some(client) = langfuse else { - // Nothing to retract against; the score sink has no delete path. - return FeedbackOutcome::Retracted; - }; - if let Err(error) = client.delete_score(&score_id).await { - warn!(%error, channel, "failed to retract reaction feedback"); - } + moltis_observability::record(moltis_observability::Event::ScoreDelete(Box::new( + ScoreDeleteRecord::new(trace_id, score_id), + ))); FeedbackOutcome::Retracted } @@ -296,6 +295,19 @@ mod tests { (service, links) } + fn langfuse_client() -> Arc { + Arc::new(LangfuseClient::new( + moltis_observability::exporters::langfuse::LangfuseConfig { + host: "https://cloud.langfuse.com".into(), + public_key: "pk-test".into(), + secret_key: "sk-test".to_string().into(), + environment: Some("test".into()), + release: None, + timeout: std::time::Duration::from_secs(1), + }, + )) + } + /// Collects scores emitted through the global sink. /// /// The sink is process-wide, so the tests that assert on emitted scores @@ -381,6 +393,7 @@ mod tests { let (service, links) = service(true); link_reply(&links, "42", "trace-1").await; + let langfuse = langfuse_client(); let outcome = service .on_reaction( @@ -391,7 +404,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + Some(&langfuse), ) .await; @@ -433,6 +446,7 @@ mod tests { #[tokio::test] async fn a_reaction_on_an_unlinked_message_is_ignored() { let (service, _links) = service(true); + let langfuse = langfuse_client(); let outcome = service .on_reaction( "telegram", @@ -442,7 +456,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + Some(&langfuse), ) .await; @@ -451,8 +465,12 @@ mod tests { #[tokio::test] async fn removing_a_reaction_retracts_rather_than_scoring_again() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); let (service, links) = service(true); link_reply(&links, "42", "trace-1").await; + let langfuse = langfuse_client(); let outcome = service .on_reaction( @@ -463,11 +481,38 @@ mod tests { "\u{1f44d}", "99", false, - None, + Some(&langfuse), ) .await; assert_eq!(outcome, FeedbackOutcome::Retracted); + let events = sink.events.lock().unwrap_or_else(|e| e.into_inner()); + assert!(matches!(events.as_slice(), [ + moltis_observability::Event::ScoreDelete(_) + ])); + drop(events); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn score_feedback_is_disabled_without_langfuse() { + let (service, links) = service(true); + link_reply(&links, "42", "trace-1").await; + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "42", + "\u{1f44d}", + "99", + true, + None, + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Disabled); } #[tokio::test] diff --git a/crates/chat/src/agent_loop.rs b/crates/chat/src/agent_loop.rs index dd0a75453..fa828ecef 100644 --- a/crates/chat/src/agent_loop.rs +++ b/crates/chat/src/agent_loop.rs @@ -169,6 +169,7 @@ pub(crate) struct ChannelStreamDispatcher { delivered: Arc)>>>, feedback: Option>, session_key: String, + trace_correlation_key: String, started: bool, sent_final_delta: bool, } @@ -177,6 +178,7 @@ impl ChannelStreamDispatcher { pub(crate) async fn for_session( state: &Arc, session_key: &str, + trace_correlation_key: &str, ) -> Option { let outbound = state.channel_stream_outbound()?; let targets: Vec = state @@ -196,6 +198,7 @@ impl ChannelStreamDispatcher { delivered: Arc::new(Mutex::new(Vec::new())), feedback: state.feedback(), session_key: session_key.to_string(), + trace_correlation_key: trace_correlation_key.to_string(), started: false, sent_final_delta: false, }; @@ -323,6 +326,7 @@ impl ChannelStreamDispatcher { &target, &message_ids, &self.session_key, + &self.trace_correlation_key, ) .await; } @@ -530,6 +534,7 @@ pub(crate) async fn run_explicit_shell_command( deliver_channel_replies( state, session_key, + run_id, &final_text, ReplyMedium::Text, &streamed_target_keys, @@ -706,6 +711,7 @@ mod tests { delivered: Arc::new(Mutex::new(Vec::new())), feedback: None, session_key: "session".into(), + trace_correlation_key: "run".into(), started: false, sent_final_delta: false, } @@ -803,6 +809,7 @@ mod tests { let mut dispatcher = dispatcher_with(outbound); dispatcher.feedback = Some(feedback); dispatcher.session_key = session_key.to_string(); + dispatcher.trace_correlation_key = session_key.to_string(); dispatcher.send_delta("final").await; dispatcher.finish().await; diff --git a/crates/chat/src/channel_feedback.rs b/crates/chat/src/channel_feedback.rs index cc4617b2d..01d9bba3a 100644 --- a/crates/chat/src/channel_feedback.rs +++ b/crates/chat/src/channel_feedback.rs @@ -22,7 +22,7 @@ pub(crate) async fn record_web_reply_trace( let Some(feedback) = state.feedback() else { return; }; - let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + let Some(trace_id) = moltis_observability::recent_trace(run_id) else { return; }; feedback @@ -53,6 +53,7 @@ pub(crate) async fn record_reply_trace( target: &moltis_channels::ChannelReplyTarget, message_ids: &[String], session_key: &str, + trace_correlation_key: &str, ) { if message_ids.is_empty() { return; @@ -60,7 +61,7 @@ pub(crate) async fn record_reply_trace( let Some(feedback) = feedback else { return; }; - let Some(trace_id) = moltis_observability::recent_trace(session_key) else { + let Some(trace_id) = moltis_observability::recent_trace(trace_correlation_key) else { return; }; feedback @@ -150,6 +151,7 @@ mod tests { &target, &["901".to_string(), "902".to_string()], session_key, + session_key, ) .await; @@ -167,6 +169,13 @@ mod tests { #[tokio::test] async fn nothing_is_recorded_without_a_feedback_service() { - record_reply_trace(None, &telegram_topic_target(), &["1".to_string()], "none").await; + record_reply_trace( + None, + &telegram_topic_target(), + &["1".to_string()], + "none", + "none", + ) + .await; } } diff --git a/crates/chat/src/channel_reply_delivery.rs b/crates/chat/src/channel_reply_delivery.rs index 9aa561933..e3171bd10 100644 --- a/crates/chat/src/channel_reply_delivery.rs +++ b/crates/chat/src/channel_reply_delivery.rs @@ -33,6 +33,7 @@ async fn deliver_logbook_follow_up( to: &str, logbook_html: &str, session_key: &str, + trace_correlation_key: &str, ) { if logbook_html.is_empty() { return; @@ -47,6 +48,7 @@ async fn deliver_logbook_follow_up( target, &message_ids, session_key, + trace_correlation_key, ) .await; }, @@ -78,6 +80,7 @@ pub(crate) async fn deliver_text_reply( logbook_html: &str, reply_to: Option<&str>, session_key: &str, + trace_correlation_key: &str, ) { let result = if logbook_html.is_empty() { outbound @@ -101,6 +104,7 @@ pub(crate) async fn deliver_text_reply( target, &message_ids, session_key, + trace_correlation_key, ) .await; }, @@ -119,6 +123,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( outbound: Arc, targets: Vec, session_key: &str, + trace_correlation_key: &str, text: &str, state: Arc, desired_reply_medium: ReplyMedium, @@ -126,6 +131,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( streamed_target_keys: &HashSet, ) { let session_key = session_key.to_string(); + let trace_correlation_key = trace_correlation_key.to_string(); let text = text.to_string(); let logbook_html = format_logbook_html(&status_log); // Resolved once rather than per target: it is the same service either way. @@ -136,6 +142,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( let state = Arc::clone(&state); let feedback = feedback.clone(); let session_key = session_key.clone(); + let trace_correlation_key = trace_correlation_key.clone(); let text = text.clone(); let logbook_html = logbook_html.clone(); // Text was already delivered via edit-in-place streaming — skip text @@ -176,6 +183,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &to, &logbook_html, &session_key, + &trace_correlation_key, ) .await; } else { @@ -211,6 +219,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &to, &logbook_html, &session_key, + &trace_correlation_key, ) .await; } else { @@ -240,6 +249,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &logbook_html, None, &session_key, + &trace_correlation_key, ) .await; } @@ -255,6 +265,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &to, &logbook_html, &session_key, + &trace_correlation_key, ) .await; }, @@ -268,6 +279,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &logbook_html, reply_to, &session_key, + &trace_correlation_key, ) .await; }, @@ -296,6 +308,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &to, &logbook_html, &session_key, + &trace_correlation_key, ) .await; }, @@ -309,6 +322,7 @@ pub(crate) async fn deliver_channel_replies_to_targets( &logbook_html, reply_to, &session_key, + &trace_correlation_key, ) .await; }, @@ -469,6 +483,7 @@ mod tests { "chan", "
log
", "session", + "run", ) .await; @@ -481,7 +496,16 @@ mod tests { let recorder = Arc::new(ReportingOutbound::default()); let outbound: Arc = recorder.clone(); - deliver_logbook_follow_up(&outbound, None, &reply_target(), "chan", "", "session").await; + deliver_logbook_follow_up( + &outbound, + None, + &reply_target(), + "chan", + "", + "session", + "run", + ) + .await; let calls = recorder.calls.lock().unwrap_or_else(|e| e.into_inner()); assert!(calls.is_empty(), "expected no send, got {calls:?}"); @@ -523,6 +547,7 @@ mod tests { "
log
", None, "session", + "run", ) .await; @@ -544,6 +569,7 @@ mod tests { "", None, "session", + "run", ) .await; diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 290302c97..da80cca2a 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -57,6 +57,7 @@ pub(crate) async fn send_chat_push_notification( pub(crate) async fn deliver_channel_replies( state: &Arc, session_key: &str, + trace_correlation_key: &str, text: &str, desired_reply_medium: ReplyMedium, streamed_target_keys: &HashSet, @@ -136,6 +137,7 @@ pub(crate) async fn deliver_channel_replies( &logbook_html, state.feedback(), session_key, + trace_correlation_key, ) .await; } @@ -154,6 +156,7 @@ pub(crate) async fn deliver_channel_replies( outbound, targets, session_key, + trace_correlation_key, text, Arc::clone(state), desired_reply_medium, @@ -193,6 +196,7 @@ async fn send_channel_logbook_follow_up_to_targets( logbook_html: &str, feedback: Option>, session_key: &str, + trace_correlation_key: &str, ) { if targets.is_empty() || logbook_html.is_empty() { return; @@ -200,12 +204,14 @@ async fn send_channel_logbook_follow_up_to_targets( let html = logbook_html.to_string(); let session_key = session_key.to_string(); + let trace_correlation_key = trace_correlation_key.to_string(); let mut tasks = Vec::with_capacity(targets.len()); for target in targets { let outbound = Arc::clone(&outbound); let html = html.clone(); let feedback = feedback.clone(); let session_key = session_key.clone(); + let trace_correlation_key = trace_correlation_key.clone(); let to = target.outbound_to().into_owned(); tasks.push(tokio::spawn(async move { match outbound @@ -218,6 +224,7 @@ async fn send_channel_logbook_follow_up_to_targets( &target, &message_ids, &session_key, + &trace_correlation_key, ) .await; }, diff --git a/crates/chat/src/run_with_tools.rs b/crates/chat/src/run_with_tools.rs index a668a719a..0ea6f8d0b 100644 --- a/crates/chat/src/run_with_tools.rs +++ b/crates/chat/src/run_with_tools.rs @@ -278,9 +278,10 @@ pub(crate) async fn run_with_tools( let provider_name_for_events = provider_name.to_string(); let active_partial_for_events = active_partial_assistant.as_ref().map(Arc::clone); let (on_event, mut event_rx) = ordered_runner_event_callback(); - let channel_stream_dispatcher = ChannelStreamDispatcher::for_session(state, session_key) - .await - .map(|dispatcher| Arc::new(Mutex::new(dispatcher))); + let channel_stream_dispatcher = + ChannelStreamDispatcher::for_session(state, session_key, run_id) + .await + .map(|dispatcher| Arc::new(Mutex::new(dispatcher))); let channel_stream_for_events = channel_stream_dispatcher.as_ref().map(Arc::clone); let event_forwarder = tokio::spawn(async move { // Track tool call arguments from ToolCallStart so they can be persisted in ToolCallEnd. @@ -894,6 +895,7 @@ pub(crate) async fn run_with_tools( conn_id.as_deref(), runtime_context, ); + tool_context["_trace_correlation_key"] = serde_json::json!(run_id); if let Some(controls) = tool_controls { if let Some(active_tools) = controls.active_tools { tool_context["active_tools"] = serde_json::json!(active_tools); @@ -1217,6 +1219,7 @@ pub(crate) async fn run_with_tools( deliver_channel_replies( state, session_key, + run_id, &display_text, desired_reply_medium, &streamed_target_keys, diff --git a/crates/chat/src/streaming.rs b/crates/chat/src/streaming.rs index 631954706..7881903ec 100644 --- a/crates/chat/src/streaming.rs +++ b/crates/chat/src/streaming.rs @@ -17,7 +17,8 @@ use { moltis_agents::{ ChatMessage, UserContent, model::{ - StreamEvent, push_capped_provider_raw_event, + AgentToolControls, ProviderAttemptEvent, ProviderIdentity, StreamEvent, + TrackedStreamEvent, push_capped_provider_raw_event, values_to_chat_messages_with_tool_result_limit, }, prompt::{PromptRuntimeContext, build_system_prompt_minimal_runtime_details}, @@ -34,7 +35,7 @@ use crate::{ chat_error::parse_chat_error, message::apply_voice_reply_suffix, models::DisabledModelsStore, - prompt::prompt_build_limits_from_config, + prompt::{channel_binding_from_runtime_context, prompt_build_limits_from_config}, runtime::ChatRuntime, service::ActiveAssistantDraft, types::*, @@ -226,326 +227,446 @@ pub(crate) async fn run_streaming( let mut rate_limit_retries_remaining: u8 = STREAM_RATE_LIMIT_MAX_RETRIES; let mut rate_limit_backoff_ms: Option = None; let mut channel_stream_dispatcher = - ChannelStreamDispatcher::for_session(state, session_key).await; + ChannelStreamDispatcher::for_session(state, session_key, run_id).await; + let turn_recorder = moltis_agents::runner::instrumentation::begin_turn( + session_key, + Some(run_id), + channel_binding_from_runtime_context(runtime_context).as_ref(), + provider_name, + model_id, + user_content, + moltis_agents::runner::instrumentation::recorder_settings(&persona.config.instrumentation), + persona.config.instrumentation.environment.clone(), + moltis_agents::runner::instrumentation::release(&persona.config.instrumentation), + ); 'attempts: loop { #[cfg(feature = "metrics")] let stream_start = Instant::now(); - let mut stream = provider.stream(messages.clone()); + let mut generation_step: Option = None; + let mut selected_identity = ProviderIdentity::new(provider_name, model_id); + let mut stream = provider.stream_with_tools_and_options_tracked( + messages.clone(), + Vec::new(), + AgentToolControls::default(), + ); let mut accumulated = String::new(); let mut accumulated_reasoning = String::new(); let mut raw_llm_responses: Vec = Vec::new(); while let Some(event) = stream.next().await { match event { - StreamEvent::Delta(delta) => { - accumulated.push_str(&delta); - if let Some(ref map) = active_partial_assistant - && let Some(draft) = map.write().await.get_mut(session_key) - { - draft.append_text(&delta); + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity)) => { + if let Some(mut previous) = generation_step.take() { + previous.fail("provider attempt superseded before completion"); + previous.finish(); } - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.send_delta(&delta).await; + selected_identity = identity; + if let Some(recorder) = turn_recorder.as_ref() { + recorder.set_metadata( + "provider", + Value::String(selected_identity.provider.clone()), + ); + recorder + .set_metadata("model", Value::String(selected_identity.model.clone())); + let mut step = recorder.step( + moltis_observability::ObservationKind::Generation, + moltis_agents::runner::instrumentation::generation_name( + &selected_identity.provider, + &selected_identity.model, + ), + ); + step.set_model(selected_identity.model.clone()); + step.set_metadata( + "provider", + Value::String(selected_identity.provider.clone()), + ); + step.set_input(Value::Array( + messages.iter().map(ChatMessage::to_openai_value).collect(), + )); + generation_step = Some(step); } - broadcast( - state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "delta", - "text": delta, - }), - BroadcastOpts::default(), - ) - .await; }, - StreamEvent::ReasoningDelta(delta) => { - accumulated_reasoning.push_str(&delta); - if let Some(ref map) = active_partial_assistant - && let Some(draft) = map.write().await.get_mut(session_key) - { - draft.set_reasoning(&accumulated_reasoning); + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { error, .. }) => { + if let Some(mut step) = generation_step.take() { + step.fail(error); + step.finish(); } - broadcast( - state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "thinking_text", - "text": accumulated_reasoning.clone(), - }), - BroadcastOpts::default(), - ) - .await; }, - StreamEvent::ProviderRaw(raw) => { - push_capped_provider_raw_event(&mut raw_llm_responses, raw); - }, - StreamEvent::Done(usage) => { - clear_unsupported_model(state, model_store, model_id).await; - - // Record streaming completion metrics (mirroring provider_chain.rs) - #[cfg(feature = "metrics")] - { - let duration = stream_start.elapsed().as_secs_f64(); - counter!( - llm_metrics::COMPLETIONS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(1); - counter!( - llm_metrics::INPUT_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.input_tokens)); - counter!( - llm_metrics::OUTPUT_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.output_tokens)); - counter!( - llm_metrics::CACHE_READ_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() - ) - .increment(u64::from(usage.cache_read_tokens)); - counter!( - llm_metrics::CACHE_WRITE_TOKENS_TOTAL, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() + TrackedStreamEvent::Event(event) => match event { + StreamEvent::Delta(delta) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } + accumulated.push_str(&delta); + if let Some(ref map) = active_partial_assistant + && let Some(draft) = map.write().await.get_mut(session_key) + { + draft.append_text(&delta); + } + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.send_delta(&delta).await; + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "delta", + "text": delta, + }), + BroadcastOpts::default(), ) - .increment(u64::from(usage.cache_write_tokens)); - histogram!( - llm_metrics::COMPLETION_DURATION_SECONDS, - labels::PROVIDER => provider_name.to_string(), - labels::MODEL => model_id.to_string() + .await; + }, + StreamEvent::ReasoningDelta(delta) => { + if let Some(step) = generation_step.as_mut() { + step.mark_first_token(); + } + accumulated_reasoning.push_str(&delta); + if let Some(ref map) = active_partial_assistant + && let Some(draft) = map.write().await.get_mut(session_key) + { + draft.set_reasoning(&accumulated_reasoning); + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "thinking_text", + "text": accumulated_reasoning.clone(), + }), + BroadcastOpts::default(), ) - .record(duration); - } + .await; + }, + StreamEvent::ProviderRaw(raw) => { + push_capped_provider_raw_event(&mut raw_llm_responses, raw); + }, + StreamEvent::Done(usage) => { + clear_unsupported_model(state, model_store, &selected_identity.model).await; + + // Record streaming completion metrics (mirroring provider_chain.rs) + #[cfg(feature = "metrics")] + { + let duration = stream_start.elapsed().as_secs_f64(); + counter!( + llm_metrics::COMPLETIONS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(1); + counter!( + llm_metrics::INPUT_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.input_tokens)); + counter!( + llm_metrics::OUTPUT_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.output_tokens)); + counter!( + llm_metrics::CACHE_READ_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.cache_read_tokens)); + counter!( + llm_metrics::CACHE_WRITE_TOKENS_TOTAL, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .increment(u64::from(usage.cache_write_tokens)); + histogram!( + llm_metrics::COMPLETION_DURATION_SECONDS, + labels::PROVIDER => selected_identity.provider.clone(), + labels::MODEL => selected_identity.model.clone() + ) + .record(duration); + } - let is_silent = accumulated.trim().is_empty(); - let reasoning = { - let trimmed = accumulated_reasoning.trim(); - (!trimmed.is_empty()).then(|| trimmed.to_string()) - }; - let streamed_target_keys = - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - dispatcher.completed_target_keys().await - } else { - HashSet::new() + let is_silent = accumulated.trim().is_empty(); + let reasoning = { + let trimmed = accumulated_reasoning.trim(); + (!trimmed.is_empty()).then(|| trimmed.to_string()) }; - - info!( - run_id, - input_tokens = usage.input_tokens, - output_tokens = usage.output_tokens, - response = %accumulated, - silent = is_silent, - "chat stream done" - ); - - // Detect provider failures: silent stream with zero tokens - // means the LLM never produced output (e.g. network_error). - if is_silent && usage.output_tokens == 0 { - warn!( + let streamed_target_keys = + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.finish().await; + dispatcher.completed_target_keys().await + } else { + HashSet::new() + }; + + info!( run_id, - "empty stream with zero tokens — treating as provider error" - ); - let error_obj = parse_chat_error( - "The provider returned an empty response (possible network error). Please try again.", - Some(provider_name), + input_tokens = usage.input_tokens, + output_tokens = usage.output_tokens, + response = %accumulated, + silent = is_silent, + "chat stream done" ); - deliver_channel_error(state, session_key, &error_obj).await; - let error_payload = ChatErrorBroadcast { - run_id: run_id.to_string(), - session_key: session_key.to_string(), - state: "error", - error: error_obj, - seq: client_seq, - }; - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; - return None; - } - let assistant_message_index = user_message_index + 1; - - // Generate & persist TTS audio for voice-medium web UI replies. - let mut audio_warning: Option = None; - let audio_path = if !is_silent && desired_reply_medium == ReplyMedium::Voice { - match generate_tts_audio(state, session_key, &accumulated).await { - Ok(bytes) => { - let filename = format!("{run_id}.ogg"); - if let Some(store) = session_store { - match store.save_media(session_key, &filename, &bytes).await { - Ok(path) => Some(path), - Err(e) => { - let warning = format!( - "TTS audio generated but failed to save: {e}" - ); - warn!(run_id, error = %warning, "failed to save TTS audio to media dir"); - audio_warning = Some(warning); - None - }, - } - } else { - audio_warning = Some( + // Detect provider failures: silent stream with zero tokens + // means the LLM never produced output (e.g. network_error). + if is_silent && usage.output_tokens == 0 { + warn!( + run_id, + "empty stream with zero tokens — treating as provider error" + ); + let error_obj = parse_chat_error( + "The provider returned an empty response (possible network error). Please try again.", + Some(&selected_identity.provider), + ); + let message = + "provider returned an empty response with zero output tokens"; + if let Some(mut step) = generation_step.take() { + step.set_usage( + moltis_agents::runner::instrumentation::to_token_usage(&usage), + ); + step.fail(message); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(message); + } + deliver_channel_error(state, session_key, &error_obj).await; + let error_payload = ChatErrorBroadcast { + run_id: run_id.to_string(), + session_key: session_key.to_string(), + state: "error", + error: error_obj, + seq: client_seq, + }; + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&error_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + return None; + } + + if let Some(mut step) = generation_step.take() { + step.set_usage(moltis_agents::runner::instrumentation::to_token_usage( + &usage, + )); + step.set_output(Value::String(accumulated.clone())); + if !accumulated_reasoning.is_empty() { + step.set_output_metadata( + "reasoning", + Value::String(accumulated_reasoning.clone()), + ); + } + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.set_output(Value::String(accumulated.clone())); + recorder.finish(); + } + + let assistant_message_index = user_message_index + 1; + + // Generate & persist TTS audio for voice-medium web UI replies. + let mut audio_warning: Option = None; + let audio_path = if !is_silent && desired_reply_medium == ReplyMedium::Voice + { + match generate_tts_audio(state, session_key, &accumulated).await { + Ok(bytes) => { + let filename = format!("{run_id}.ogg"); + if let Some(store) = session_store { + match store.save_media(session_key, &filename, &bytes).await + { + Ok(path) => Some(path), + Err(e) => { + let warning = format!( + "TTS audio generated but failed to save: {e}" + ); + warn!(run_id, error = %warning, "failed to save TTS audio to media dir"); + audio_warning = Some(warning); + None + }, + } + } else { + audio_warning = Some( "TTS audio generated but session media storage is unavailable" .to_string(), ); + None + } + }, + Err(error) => { + let error = error.to_string(); + warn!(run_id, error = %error, "voice reply generation skipped"); + audio_warning = Some(error); None - } - }, - Err(error) => { - let error = error.to_string(); - warn!(run_id, error = %error, "voice reply generation skipped"); - audio_warning = Some(error); - None - }, - } - } else { - None - }; - - let final_payload = build_chat_final_broadcast( - run_id, - session_key, - accumulated.clone(), - provider.id().to_string(), - provider_name.to_string(), - UsageSnapshot::new(usage.clone(), Some(usage.clone())), - run_started.elapsed().as_millis() as u64, - assistant_message_index, - desired_reply_medium, - None, - None, - audio_path.clone(), - audio_warning, - reasoning.clone(), - client_seq, - ); - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&final_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; - - if !is_silent { - // Send push notification when chat response completes - #[cfg(feature = "push-notifications")] - { - tracing::info!("push: checking push notification"); - send_chat_push_notification(state, session_key, &accumulated).await; - } - deliver_channel_replies( - state, + }, + } + } else { + None + }; + + let final_payload = build_chat_final_broadcast( + run_id, session_key, - &accumulated, + accumulated.clone(), + selected_identity.model.clone(), + selected_identity.provider.clone(), + UsageSnapshot::new(usage.clone(), Some(usage.clone())), + run_started.elapsed().as_millis() as u64, + assistant_message_index, desired_reply_medium, - &streamed_target_keys, - ) - .await; - } - let llm_api_response = - (!raw_llm_responses.is_empty()).then_some(Value::Array(raw_llm_responses)); - return Some(build_assistant_turn_output( - accumulated, - UsageSnapshot::new(usage.clone(), Some(usage)), - run_started.elapsed().as_millis() as u64, - audio_path, - reasoning, - llm_api_response, - )); - }, - StreamEvent::Error(msg) => { - let error_obj = parse_chat_error(&msg, Some(provider_name)); - let has_no_streamed_content = accumulated.trim().is_empty() - && accumulated_reasoning.trim().is_empty() - && raw_llm_responses.is_empty(); - if has_no_streamed_content - && let Some(delay_ms) = next_stream_retry_delay_ms( - &msg, - &error_obj, - &mut server_retries_remaining, - &mut rate_limit_retries_remaining, - &mut rate_limit_backoff_ms, - ) - { - warn!( - run_id, - error = %msg, - delay_ms, - server_retries_remaining, - rate_limit_retries_remaining, - "chat stream transient error, retrying after delay" + None, + None, + audio_path.clone(), + audio_warning, + reasoning.clone(), + client_seq, ); - if error_obj.get("type").and_then(Value::as_str) - == Some("rate_limit_exceeded") - { - send_retry_status_to_channels( + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&final_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + + if !is_silent { + // Send push notification when chat response completes + #[cfg(feature = "push-notifications")] + { + tracing::info!("push: checking push notification"); + send_chat_push_notification(state, session_key, &accumulated).await; + } + deliver_channel_replies( state, session_key, + run_id, + &accumulated, + desired_reply_medium, + &streamed_target_keys, + ) + .await; + } + let llm_api_response = (!raw_llm_responses.is_empty()) + .then_some(Value::Array(raw_llm_responses)); + return Some(build_assistant_turn_output( + accumulated, + UsageSnapshot::new(usage.clone(), Some(usage)), + run_started.elapsed().as_millis() as u64, + audio_path, + reasoning, + llm_api_response, + )); + }, + StreamEvent::Error(msg) => { + let error_obj = parse_chat_error(&msg, Some(&selected_identity.provider)); + let has_no_streamed_content = accumulated.trim().is_empty() + && accumulated_reasoning.trim().is_empty() + && raw_llm_responses.is_empty(); + if has_no_streamed_content + && let Some(delay_ms) = next_stream_retry_delay_ms( + &msg, &error_obj, - Duration::from_millis(delay_ms), + &mut server_retries_remaining, + &mut rate_limit_retries_remaining, + &mut rate_limit_backoff_ms, + ) + { + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } + warn!( + run_id, + error = %msg, + delay_ms, + server_retries_remaining, + rate_limit_retries_remaining, + "chat stream transient error, retrying after delay" + ); + if error_obj.get("type").and_then(Value::as_str) + == Some("rate_limit_exceeded") + { + send_retry_status_to_channels( + state, + session_key, + &error_obj, + Duration::from_millis(delay_ms), + ) + .await; + } + broadcast( + state, + "chat", + serde_json::json!({ + "runId": run_id, + "sessionKey": session_key, + "state": "retrying", + "error": error_obj, + "retryAfterMs": delay_ms, + "seq": client_seq, + }), + BroadcastOpts::default(), ) .await; + tokio::time::sleep(Duration::from_millis(delay_ms)).await; + continue 'attempts; } - broadcast( + + warn!(run_id, error = %msg, "chat stream error"); + if let Some(mut step) = generation_step.take() { + step.fail(msg.clone()); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(msg.clone()); + } + if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { + dispatcher.finish().await; + } + state.set_run_error(run_id, msg.clone()).await; + mark_unsupported_model( state, - "chat", - serde_json::json!({ - "runId": run_id, - "sessionKey": session_key, - "state": "retrying", - "error": error_obj, - "retryAfterMs": delay_ms, - "seq": client_seq, - }), - BroadcastOpts::default(), + model_store, + &selected_identity.model, + &selected_identity.provider, + &error_obj, ) .await; - tokio::time::sleep(Duration::from_millis(delay_ms)).await; - continue 'attempts; - } - - warn!(run_id, error = %msg, "chat stream error"); - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - } - state.set_run_error(run_id, msg.clone()).await; - mark_unsupported_model(state, model_store, model_id, provider_name, &error_obj) - .await; - deliver_channel_error(state, session_key, &error_obj).await; - let error_payload = ChatErrorBroadcast { - run_id: run_id.to_string(), - session_key: session_key.to_string(), - state: "error", - error: error_obj, - seq: client_seq, - }; - #[allow(clippy::unwrap_used)] // serializing known-valid struct - let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; - return None; + deliver_channel_error(state, session_key, &error_obj).await; + let error_payload = ChatErrorBroadcast { + run_id: run_id.to_string(), + session_key: session_key.to_string(), + state: "error", + error: error_obj, + seq: client_seq, + }; + #[allow(clippy::unwrap_used)] // serializing known-valid struct + let payload_val = serde_json::to_value(&error_payload).unwrap(); + terminal_runs.write().await.insert(run_id.to_string()); + broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + return None; + }, + // Tool events not expected in stream-only mode. + StreamEvent::ToolCallStart { .. } + | StreamEvent::ToolCallArgumentsDelta { .. } + | StreamEvent::ToolCallComplete { .. } => {}, }, - // Tool events not expected in stream-only mode. - StreamEvent::ToolCallStart { .. } - | StreamEvent::ToolCallArgumentsDelta { .. } - | StreamEvent::ToolCallComplete { .. } => {}, } } // Stream ended unexpectedly without Done/Error. + let message = "provider stream ended without a terminal event"; + if let Some(mut step) = generation_step.take() { + step.fail(message); + step.finish(); + } + if let Some(recorder) = turn_recorder.as_ref() { + recorder.finish_with_error(message); + } return None; } } diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs index 67d2afc17..5aace9a23 100644 --- a/crates/config/src/schema/instrumentation.rs +++ b/crates/config/src/schema/instrumentation.rs @@ -129,14 +129,14 @@ pub struct InstrumentationConfig { /// the built-in list; it cannot be narrowed. #[serde(default)] pub redact: Vec, - /// Bounded export queue depth. Events are dropped once this fills rather - /// than blocking the agent loop. + /// Nonzero bounded export queue depth. Events are dropped once this fills + /// rather than blocking the agent loop. #[serde(default = "default_queue_capacity")] pub queue_capacity: usize, - /// Maximum time an event waits before being flushed. + /// Nonzero maximum time an event waits before being flushed. #[serde(default = "default_flush_interval_ms")] pub flush_interval_ms: u64, - /// Maximum estimated batch size in bytes before a forced flush. + /// Nonzero maximum estimated batch size in bytes before a forced flush. #[serde(default = "default_max_batch_bytes")] pub max_batch_bytes: usize, /// Langfuse backend. @@ -220,7 +220,7 @@ pub struct LangfuseSettings { /// credentials to appear. #[serde(default = "default_true")] pub capture_tool_io: bool, - /// Per-request timeout in seconds. + /// Nonzero per-request timeout in seconds. #[serde(default = "default_timeout_secs")] pub timeout_secs: u64, } @@ -266,7 +266,7 @@ pub struct OtlpSettings { /// in an APM's index and often a compliance question. #[serde(default)] pub emit_user_id: bool, - /// Per-request timeout in seconds. + /// Nonzero per-request timeout in seconds. #[serde(default = "default_timeout_secs")] pub timeout_secs: u64, } @@ -314,7 +314,7 @@ pub struct DatadogSettings { /// How much conversation content this backend receives. #[serde(default)] pub content: ContentCaptureMode, - /// Per-request timeout in seconds. + /// Nonzero per-request timeout in seconds. #[serde(default = "default_timeout_secs")] pub timeout_secs: u64, } diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index cf7c74a62..467789239 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -573,47 +573,53 @@ port = {port} # Port number (auto-generated for this i # INSTRUMENTATION (Langfuse / OpenTelemetry / Datadog) # ══════════════════════════════════════════════════════════════════════════════ # -# Exports agent runs — LLM calls, tool calls, retrievals — to an external -# backend. Disabled by default: enabling it sends data about your -# conversations to a third party. See docs/src/instrumentation.md. +# Exports completed agent runs — LLM calls, tool calls and retrievals — to an +# external backend. Observations are immutable and sent once, after completion. +# Disabled by default: enabling it sends conversation data to a third party. +# See docs/src/instrumentation.md. # # Backends deliberately receive different data. Langfuse gets the full -# conversation because its cost, session and evaluation features are built on -# it. OTLP and Datadog get operational shape only — latency, errors, model, -# token counts — because prompt bodies in an APM mean unbounded span size, -# cardinality pressure, per-byte billing, and conversation content in a system -# nobody scoped for it. +# conversation, token usage and session context for LLM observability and cost +# inference. Prompt Management, datasets, evaluators and media uploads are not +# integrated. OTLP and Datadog receive operational shape only. # [instrumentation] # enabled = false # Master switch, gates every backend # environment = "production" # Reported to every backend # sample_rate = 1.0 # Fraction of turns traced (0.0-1.0) # redact = ["customer_ref"] # Extra keys to redact; extends the defaults -# queue_capacity = 10000 # Events are dropped, never blocking a turn +# queue_capacity = 10000 # Must be nonzero; full queues drop events +# flush_interval_ms = 5000 # Must be nonzero +# max_batch_bytes = 3000000 # Must be nonzero # [instrumentation.langfuse] # enabled = false # host = "https://cloud.langfuse.com" # Or a self-hosted URL # public_key = "pk-lf-..." -# secret_key = "sk-lf-..." +# Prefer MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY in the process environment. +# A secret_key config value is also accepted. # capture_input = true # Turn and LLM inputs # capture_output = true # Turn and LLM outputs # capture_tool_io = true # Tool arguments and results +# timeout_secs = 10 # Must be nonzero # [instrumentation.otlp] # Grafana Tempo/Alloy, Honeycomb, a collector # enabled = false # endpoint = "http://localhost:4318/v1/traces" # content = "metadata_only" # "full" | "metadata_only" | "none" # emit_user_id = false # High-cardinality in an APM index +# timeout_secs = 10 # Must be nonzero # [instrumentation.datadog] # Via the Datadog Agent's OTLP intake # enabled = false # endpoint = "http://localhost:4318/v1/traces" # service = "moltis" # content = "metadata_only" +# timeout_secs = 10 # Must be nonzero # Reaction feedback. A thumbs up/down on a reply in Telegram, Discord or Slack -# becomes a "user-feedback" score on the trace that produced it. Lists accept +# becomes a BOOLEAN "user-feedback" score through Langfuse's dedicated Scores +# API. Score creates/replacements and deletions retain queue order. Lists accept # raw emoji or shortcodes; empty means the built-in vocabulary. # [instrumentation.feedback] # enabled = true diff --git a/crates/config/src/validate/semantic.rs b/crates/config/src/validate/semantic.rs index 276c2bc51..5097be71c 100644 --- a/crates/config/src/validate/semantic.rs +++ b/crates/config/src/validate/semantic.rs @@ -409,6 +409,8 @@ pub(super) fn check_semantic_warnings(config: &MoltisConfig, diagnostics: &mut V }); } + validate_instrumentation(&config.instrumentation, diagnostics); + for (name, server) in &config.mcp.servers { if server.request_timeout_secs == Some(0) { diagnostics.push(Diagnostic { @@ -987,6 +989,109 @@ pub(super) fn check_semantic_warnings(config: &MoltisConfig, diagnostics: &mut V } } +fn validate_instrumentation( + config: &crate::schema::InstrumentationConfig, + diagnostics: &mut Vec, +) { + if !config.sample_rate.is_finite() { + diagnostics.push(Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: "instrumentation.sample_rate".into(), + message: "instrumentation.sample_rate must be finite and between 0.0 and 1.0".into(), + }); + } else if !(0.0..=1.0).contains(&config.sample_rate) { + diagnostics.push(Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: "instrumentation.sample_rate".into(), + message: format!( + "instrumentation.sample_rate must be between 0.0 and 1.0 (got {})", + config.sample_rate + ), + }); + } + + let zero_values = [ + (config.queue_capacity == 0, "instrumentation.queue_capacity"), + ( + config.flush_interval_ms == 0, + "instrumentation.flush_interval_ms", + ), + ( + config.max_batch_bytes == 0, + "instrumentation.max_batch_bytes", + ), + ( + config.langfuse.timeout_secs == 0, + "instrumentation.langfuse.timeout_secs", + ), + ( + config.otlp.timeout_secs == 0, + "instrumentation.otlp.timeout_secs", + ), + ( + config.datadog.timeout_secs == 0, + "instrumentation.datadog.timeout_secs", + ), + ]; + diagnostics.extend( + zero_values + .into_iter() + .filter(|(is_zero, _)| *is_zero) + .map(|(_, path)| Diagnostic { + severity: Severity::Error, + category: "invalid-value", + path: path.into(), + message: format!("{path} must be at least 1"), + }), + ); + + let unusually_large = [ + ( + config.queue_capacity > 1_000_000, + "instrumentation.queue_capacity", + "a queue above 1,000,000 events can consume excessive memory", + ), + ( + config.flush_interval_ms > 300_000, + "instrumentation.flush_interval_ms", + "an interval above 5 minutes delays delivery and increases shutdown loss risk", + ), + ( + config.max_batch_bytes > 100_000_000, + "instrumentation.max_batch_bytes", + "a batch above 100 MB can exceed collector and proxy request limits", + ), + ( + config.langfuse.timeout_secs > 300, + "instrumentation.langfuse.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ( + config.otlp.timeout_secs > 300, + "instrumentation.otlp.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ( + config.datadog.timeout_secs > 300, + "instrumentation.datadog.timeout_secs", + "a request timeout above 5 minutes can delay retries and shutdown", + ), + ]; + diagnostics.extend( + unusually_large + .into_iter() + .filter(|(is_large, ..)| *is_large) + .map(|(_, path, message)| Diagnostic { + severity: Severity::Warning, + category: "invalid-value", + path: path.into(), + message: message.into(), + }), + ); +} + /// Validate a `context_window` override value (optional field). fn validate_context_window(value: Option, path: &str, diagnostics: &mut Vec) { let Some(cw) = value else { diff --git a/crates/config/src/validate/tests.rs b/crates/config/src/validate/tests.rs index ab741b24d..acdbe0fb4 100644 --- a/crates/config/src/validate/tests.rs +++ b/crates/config/src/validate/tests.rs @@ -13,6 +13,8 @@ mod channels; mod common; #[path = "tests/defaults.rs"] mod defaults; +#[path = "tests/instrumentation.rs"] +mod instrumentation; #[path = "tests/memory.rs"] mod memory; #[path = "tests/providers.rs"] diff --git a/crates/config/src/validate/tests/instrumentation.rs b/crates/config/src/validate/tests/instrumentation.rs new file mode 100644 index 000000000..9f2ca9537 --- /dev/null +++ b/crates/config/src/validate/tests/instrumentation.rs @@ -0,0 +1,100 @@ +use std::collections::BTreeSet; + +use super::*; + +fn diagnostics_for(toml: &str, severity: Severity) -> BTreeSet { + validate_toml_str(toml) + .diagnostics + .into_iter() + .filter(|diagnostic| diagnostic.severity == severity) + .map(|diagnostic| diagnostic.path) + .filter(|path| path.starts_with("instrumentation.")) + .collect() +} + +#[test] +fn sample_rate_must_be_finite_and_in_range() { + for sample_rate in ["nan", "inf", "-inf", "-0.01", "1.01"] { + let result = + validate_toml_str(&format!("[instrumentation]\nsample_rate = {sample_rate}\n")); + let diagnostic = result.diagnostics.iter().find(|diagnostic| { + diagnostic.severity == Severity::Error + && diagnostic.path == "instrumentation.sample_rate" + }); + + assert!( + diagnostic.is_some(), + "sample_rate {sample_rate} should be rejected: {:?}", + result.diagnostics + ); + } + + for sample_rate in ["0.0", "0.25", "1.0"] { + let errors = diagnostics_for( + &format!("[instrumentation]\nsample_rate = {sample_rate}\n"), + Severity::Error, + ); + assert!( + !errors.contains("instrumentation.sample_rate"), + "sample_rate {sample_rate} should be accepted" + ); + } +} + +#[test] +fn batching_and_backend_timeouts_must_be_nonzero() { + let errors = diagnostics_for( + r#" +[instrumentation] +queue_capacity = 0 +flush_interval_ms = 0 +max_batch_bytes = 0 + +[instrumentation.langfuse] +timeout_secs = 0 + +[instrumentation.otlp] +timeout_secs = 0 + +[instrumentation.datadog] +timeout_secs = 0 +"#, + Severity::Error, + ); + + assert_eq!( + errors, + BTreeSet::from([ + "instrumentation.datadog.timeout_secs".to_string(), + "instrumentation.flush_interval_ms".to_string(), + "instrumentation.langfuse.timeout_secs".to_string(), + "instrumentation.max_batch_bytes".to_string(), + "instrumentation.otlp.timeout_secs".to_string(), + "instrumentation.queue_capacity".to_string(), + ]) + ); +} + +#[test] +fn operationally_extreme_delivery_settings_are_warned() { + let warnings = diagnostics_for( + r#" +[instrumentation] +queue_capacity = 1000001 +flush_interval_ms = 300001 +max_batch_bytes = 100000001 + +[instrumentation.langfuse] +timeout_secs = 301 + +[instrumentation.otlp] +timeout_secs = 301 + +[instrumentation.datadog] +timeout_secs = 301 +"#, + Severity::Warning, + ); + + assert_eq!(warnings.len(), 6, "unexpected warnings: {warnings:?}"); +} diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs index b53ad7a87..cf56bbda6 100644 --- a/crates/gateway/src/methods/services/feedback.rs +++ b/crates/gateway/src/methods/services/feedback.rs @@ -25,7 +25,9 @@ pub(super) fn register(reg: &mut MethodRegistry) { Ok(serde_json::json!({ // The control is only useful when something is collecting // the result, so the UI hides it unless both are on. - "enabled": config.enabled && config.feedback.enabled, + "enabled": config.enabled + && config.feedback.enabled + && ctx.state.instrumentation.langfuse().is_some(), "instrumentation_active": ctx.state.instrumentation.status().active, "retention_days": config.feedback.link_retention_days, })) diff --git a/crates/gateway/src/methods/services/instrumentation.rs b/crates/gateway/src/methods/services/instrumentation.rs index 0313277a4..ea3ad2541 100644 --- a/crates/gateway/src/methods/services/instrumentation.rs +++ b/crates/gateway/src/methods/services/instrumentation.rs @@ -19,11 +19,14 @@ pub(super) fn register(reg: &mut MethodRegistry) { "active": status.active, "backends": status.backends, "skipped": status.skipped, + "delivery": status.delivery, "config": { "enabled": config.enabled, "environment": config.environment, "sample_rate": config.sample_rate, "queue_capacity": config.queue_capacity, + "flush_interval_ms": config.flush_interval_ms, + "max_batch_bytes": config.max_batch_bytes, "langfuse": { "enabled": config.langfuse.enabled, "host": config.langfuse.host, @@ -32,12 +35,14 @@ pub(super) fn register(reg: &mut MethodRegistry) { "capture_input": config.langfuse.capture_input, "capture_output": config.langfuse.capture_output, "capture_tool_io": config.langfuse.capture_tool_io, + "timeout_secs": config.langfuse.timeout_secs, }, "otlp": { "enabled": config.otlp.enabled, "endpoint": config.otlp.endpoint, "content": config.otlp.content, "emit_user_id": config.otlp.emit_user_id, + "timeout_secs": config.otlp.timeout_secs, }, "datadog": { "enabled": config.datadog.enabled, @@ -45,6 +50,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { "service": config.datadog.service, "api_key_set": config.datadog.api_key.is_some(), "content": config.datadog.content, + "timeout_secs": config.datadog.timeout_secs, }, }, })) @@ -82,12 +88,13 @@ pub(super) fn register(reg: &mut MethodRegistry) { }, // OTLP collectors have no standard health endpoint, and // POSTing a probe span would pollute the operator's traces - // with fake data. Sink counters are the honest signal. + // with fake data. Live delivery counters are the honest + // signal once the exporter has received real events. other => Ok(serde_json::json!({ "ok": false, "error": format!( - "no connection test available for `{other}`; check the \ - delivery counters after the next agent run instead" + "no connection test available for `{other}`; check its \ + delivery status after the next agent run" ), })), } @@ -95,3 +102,125 @@ pub(super) fn register(reg: &mut MethodRegistry) { }), ); } + +#[cfg(test)] +mod tests { + use { + super::*, + crate::{ + auth::{AuthMode, ResolvedAuth}, + methods::MethodContext, + services::GatewayServices, + state::GatewayState, + }, + moltis_config::{InstrumentationConfig, LangfuseSettings}, + }; + + async fn dispatch( + state: std::sync::Arc, + method: &str, + params: serde_json::Value, + ) -> serde_json::Value { + let mut registry = MethodRegistry::default(); + register(&mut registry); + let response = registry + .dispatch(MethodContext { + request_id: "test".into(), + method: method.into(), + params, + client_conn_id: "conn-1".into(), + client_role: "operator".into(), + client_scopes: vec!["operator.read".into(), "operator.write".into()], + state, + channel: None, + }) + .await; + + assert!(response.ok, "method failed: {:?}", response.error); + response + .payload + .unwrap_or_else(|| panic!("{method} returned no payload")) + } + + fn gateway_state() -> std::sync::Arc { + GatewayState::new( + ResolvedAuth { + mode: AuthMode::Token, + token: None, + password: None, + }, + GatewayServices::noop(), + ) + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn status_preserves_the_latest_skipped_backend_failure() { + let state = gateway_state(); + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + ..Default::default() + }, + ..Default::default() + }; + state.instrumentation.apply(&config, "test"); + + let payload = dispatch(state, "instrumentation.status", serde_json::json!({})).await; + + assert_eq!(payload["active"], false); + assert_eq!(payload["skipped"][0]["name"], "langfuse"); + assert_eq!(payload["config"]["flush_interval_ms"], 5_000); + assert_eq!(payload["config"]["max_batch_bytes"], 3_000_000); + assert_eq!(payload["config"]["langfuse"]["timeout_secs"], 10); + assert!( + payload["skipped"][0]["reason"] + .as_str() + .is_some_and(|reason| reason.contains("public_key")) + ); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn unsupported_connection_test_points_to_live_delivery_health() { + let payload = dispatch( + gateway_state(), + "instrumentation.test", + serde_json::json!({ "backend": "otlp" }), + ) + .await; + + assert_eq!(payload["ok"], false); + let error = payload["error"].as_str().unwrap_or_default(); + assert!(error.contains("delivery status")); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn status_surfaces_each_exporter_delivery_path() { + let state = gateway_state(); + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + public_key: "pk-test".into(), + secret_key: Some(secrecy::Secret::new("sk-test".to_string())), + ..Default::default() + }, + ..Default::default() + }; + state.instrumentation.apply(&config, "test"); + + let payload = dispatch(state, "instrumentation.status", serde_json::json!({})).await; + + assert_eq!(payload["delivery"][0]["name"], "langfuse"); + assert_eq!(payload["delivery"][0]["accepted"], 0); + assert_eq!(payload["delivery"][0]["delivered"], 0); + assert_eq!(payload["delivery"][0]["dropped_queue_full"], 0); + assert_eq!(payload["delivery"][0]["dropped_failed"], 0); + assert_eq!(payload["delivery"][0]["retries"], 0); + assert_eq!(payload["delivery"][1]["name"], "langfuse-scores"); + moltis_observability::clear_global_sink(); + } +} diff --git a/crates/gateway/src/server/instrumentation.rs b/crates/gateway/src/server/instrumentation.rs index 6bf5e07a6..ead3303cb 100644 --- a/crates/gateway/src/server/instrumentation.rs +++ b/crates/gateway/src/server/instrumentation.rs @@ -4,12 +4,16 @@ //! process-wide sink, which the agent runner then discovers without any //! plumbing through its call signatures. -use std::sync::{Arc, RwLock}; +use std::{ + sync::{Arc, RwLock}, + time::Duration, +}; use { moltis_config::InstrumentationConfig, moltis_observability::{ - BuiltInstrumentation, SkippedBackend, exporters::langfuse::LangfuseClient, + BuiltInstrumentation, ObservationSink, SinkStatsSnapshot, SkippedBackend, + exporters::langfuse::LangfuseClient, }, tracing::{info, warn}, }; @@ -17,11 +21,13 @@ use { /// Live instrumentation state, surfaced by the `instrumentation.*` RPC methods. #[derive(Default)] pub struct InstrumentationState { - inner: RwLock>, + inner: RwLock, } -/// What is currently running. -struct ActiveInstrumentation { +/// The latest applied outcome, including failures when no sink could start. +#[derive(Default)] +struct InstrumentationSnapshot { + sink: Option>, backends: Vec, skipped: Vec, langfuse: Option>, @@ -38,41 +44,31 @@ pub struct InstrumentationStatus { /// Surfaced rather than only logged: a silently disabled exporter is /// indistinguishable from a broken one. pub skipped: Vec, + /// Live delivery health for each exporter path. + pub delivery: Vec, } impl InstrumentationState { - /// Build from config and install the sink. Replaces any previous setup, so - /// the settings UI can reconfigure without a restart. + /// Build from config and install the sink. Replaces any previous setup and + /// flushes the replaced sink in the background; this synchronous method + /// never waits on I/O. pub fn apply(&self, config: &InstrumentationConfig, release: &str) -> InstrumentationStatus { let outcome = moltis_observability::build(config, release); - - let status = match &outcome.built { - Some(built) => InstrumentationStatus { - active: true, - backends: built.backends.clone(), - skipped: outcome.skipped.clone(), - }, - None => InstrumentationStatus { - active: false, - backends: Vec::new(), - skipped: outcome.skipped.clone(), - }, - }; - - match outcome.built { + let snapshot = match outcome.built { Some(BuiltInstrumentation { sink, langfuse, backends, .. }) => { - moltis_observability::set_global_sink(sink); + moltis_observability::set_global_sink(Arc::clone(&sink)); info!(backends = ?backends, "agent instrumentation active"); - self.store(Some(ActiveInstrumentation { + InstrumentationSnapshot { + sink: Some(sink), backends, skipped: outcome.skipped, langfuse, - })); + } }, None => { // Tear down rather than leaving a stale sink installed, so @@ -84,8 +80,20 @@ impl InstrumentationState { "instrumentation configured but no backend could start" ); } - self.store(None); + InstrumentationSnapshot { + skipped: outcome.skipped, + ..Default::default() + } }, + }; + + let status = snapshot.status(); + if let Some(old_sink) = self.replace(snapshot) { + tokio::spawn(async move { + if let Err(error) = old_sink.flush(Duration::from_secs(5)).await { + warn!(%error, "replaced instrumentation sink flush failed"); + } + }); } status @@ -98,18 +106,7 @@ impl InstrumentationState { .inner .read() .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.as_ref().map_or_else( - || InstrumentationStatus { - active: false, - backends: Vec::new(), - skipped: Vec::new(), - }, - |active| InstrumentationStatus { - active: true, - backends: active.backends.clone(), - skipped: active.skipped.clone(), - }, - ) + guard.status() } /// The Langfuse client, when that backend is running. @@ -119,12 +116,19 @@ impl InstrumentationState { .inner .read() .unwrap_or_else(std::sync::PoisonError::into_inner); - guard.as_ref().and_then(|a| a.langfuse.clone()) + guard.langfuse.clone() } /// Flush every backend, for a clean shutdown. - pub async fn flush(&self, timeout: std::time::Duration) { - let Some(sink) = moltis_observability::global_sink() else { + pub async fn flush(&self, timeout: Duration) { + let sink = { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.sink.clone() + }; + let Some(sink) = sink else { return; }; if let Err(error) = sink.flush(timeout).await { @@ -132,17 +136,46 @@ impl InstrumentationState { } } - fn store(&self, value: Option) { + /// Stop accepting new events and flush the currently installed sink. + pub async fn shutdown(&self, timeout: Duration) { + moltis_observability::clear_global_sink(); + let started = tokio::time::Instant::now(); + if !moltis_observability::wait_for_active_turns(timeout).await { + warn!("instrumentation shutdown timed out waiting for active turns"); + return; + } + self.flush(timeout.saturating_sub(started.elapsed())).await; + } + + fn replace(&self, value: InstrumentationSnapshot) -> Option> { match self.inner.write() { - Ok(mut guard) => *guard = value, - Err(poisoned) => *poisoned.into_inner() = value, + Ok(mut guard) => std::mem::replace(&mut *guard, value).sink, + Err(poisoned) => std::mem::replace(&mut *poisoned.into_inner(), value).sink, + } + } +} + +impl InstrumentationSnapshot { + fn status(&self) -> InstrumentationStatus { + InstrumentationStatus { + active: self.sink.is_some(), + backends: self.backends.clone(), + skipped: self.skipped.clone(), + delivery: self + .sink + .as_ref() + .map_or_else(Vec::new, |sink| sink.delivery_stats()), } } } #[cfg(test)] +#[allow(clippy::expect_used)] mod tests { - use {moltis_config::LangfuseSettings, secrecy::Secret}; + use { + async_trait::async_trait, moltis_config::LangfuseSettings, secrecy::Secret, + std::sync::Mutex, tokio::sync::oneshot, + }; use super::*; @@ -160,7 +193,33 @@ mod tests { } } + struct FlushTrackingSink { + flushed: Mutex>>, + } + + #[async_trait] + impl ObservationSink for FlushTrackingSink { + fn name(&self) -> &str { + "tracking" + } + + fn record(&self, _event: moltis_observability::Event) {} + + async fn flush(&self, _timeout: Duration) -> anyhow::Result<()> { + if let Some(flushed) = self + .flushed + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = flushed.send(()); + } + Ok(()) + } + } + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] async fn disabled_config_reports_inactive_and_installs_no_sink() { let state = InstrumentationState::default(); let status = state.apply(&InstrumentationConfig::default(), "test"); @@ -173,12 +232,22 @@ mod tests { } #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] async fn applying_a_valid_config_installs_the_sink() { let state = InstrumentationState::default(); let status = state.apply(&valid_langfuse_config(), "20260726.01"); assert!(status.active); assert_eq!(status.backends, vec!["langfuse"]); + assert_eq!( + status + .delivery + .iter() + .map(|stats| stats.name.as_str()) + .collect::>(), + vec!["langfuse", "langfuse-scores"] + ); + assert!(status.delivery.iter().all(|stats| stats.accepted == 0)); assert!(moltis_observability::is_enabled()); assert!(state.langfuse().is_some()); @@ -186,6 +255,7 @@ mod tests { } #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] async fn reapplying_a_disabled_config_tears_the_sink_down() { let state = InstrumentationState::default(); state.apply(&valid_langfuse_config(), "test"); @@ -201,6 +271,7 @@ mod tests { } #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] async fn skipped_backends_are_reported_not_just_logged() { let mut config = valid_langfuse_config(); config.langfuse.public_key = String::new(); @@ -213,6 +284,9 @@ mod tests { assert_eq!(status.skipped[0].name, "langfuse"); assert!(status.skipped[0].reason.contains("public_key")); + let current = state.status(); + assert_eq!(current.skipped, status.skipped); + moltis_observability::clear_global_sink(); } @@ -226,9 +300,95 @@ mod tests { } #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] async fn flush_without_a_sink_is_a_no_op() { moltis_observability::clear_global_sink(); let state = InstrumentationState::default(); - state.flush(std::time::Duration::from_millis(10)).await; + state.flush(Duration::from_millis(10)).await; + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn reconfiguration_flushes_the_replaced_sink_without_blocking_apply() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let old_sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&old_sink)); + state.replace(InstrumentationSnapshot { + sink: Some(old_sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + + let status = state.apply(&InstrumentationConfig::default(), "test"); + + assert!(!status.active); + tokio::time::timeout(Duration::from_secs(1), flushed_rx) + .await + .expect("replaced sink flush should be scheduled") + .expect("flush notification should be sent"); + assert!(!moltis_observability::is_enabled()); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_stops_new_events_and_flushes_the_active_sink() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&sink)); + state.replace(InstrumentationSnapshot { + sink: Some(sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + + state.shutdown(Duration::from_secs(1)).await; + + flushed_rx + .await + .expect("active sink should be flushed during shutdown"); + assert!(!moltis_observability::is_enabled()); + } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_waits_for_active_turns_before_flushing() { + let (flushed_tx, flushed_rx) = oneshot::channel(); + let sink: Arc = Arc::new(FlushTrackingSink { + flushed: Mutex::new(Some(flushed_tx)), + }); + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&sink)); + state.replace(InstrumentationSnapshot { + sink: Some(sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + let recorder = moltis_observability::TurnRecorder::begin( + "active", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + + let shutdown = state.shutdown(Duration::from_secs(1)); + tokio::pin!(shutdown); + assert!( + tokio::time::timeout(Duration::from_millis(10), &mut shutdown) + .await + .is_err(), + "shutdown must wait for the active recorder" + ); + drop(recorder); + shutdown.await; + + flushed_rx + .await + .expect("sink should flush after the turn closes"); } } diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index 9a08028a5..016395c8c 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -409,9 +409,9 @@ pub(super) async fn complete_startup( ); // ── Agent instrumentation ───────────────────────────────────────────── - // Applied on the state's own instance so the RPC handlers and the shutdown - // flush observe the same backends that were installed here. Runs before - // any agent can be invoked, so the first turn is traced. + // Applied on the state's own instance so RPC and lifecycle callers observe + // the same backends that were installed here. Runs before any agent can be + // invoked, so the first turn is traced. { let status = state .instrumentation diff --git a/crates/httpd/src/server/runtime.rs b/crates/httpd/src/server/runtime.rs index 0ce615668..8da43d8b8 100644 --- a/crates/httpd/src/server/runtime.rs +++ b/crates/httpd/src/server/runtime.rs @@ -1371,6 +1371,7 @@ pub async fn start_gateway( // - SIGINT (ctrl-c): immediate exit { let browser_for_shutdown = Arc::clone(&banner.browser_for_lifecycle); + let instrumentation_for_shutdown = Arc::clone(&state.instrumentation); #[cfg(feature = "tailscale")] let reset_tailscale_on_exit = banner.tailscale_mode != TailscaleMode::Off && banner.tailscale_reset_on_exit; @@ -1406,19 +1407,18 @@ pub async fn start_gateway( } "SIGINT" }; - if signal_name == "SIGINT" { - info!("received SIGINT, exiting immediately"); + info!("received SIGINT, flushing instrumentation before exit"); + instrumentation_for_shutdown + .shutdown(std::time::Duration::from_secs(1)) + .await; std::process::exit(0); } - info!(signal = signal_name, "starting graceful shutdown"); - #[cfg(feature = "mdns")] if let Some(ref daemon) = _mdns_daemon { moltis_gateway::mdns::shutdown(daemon); } - #[cfg(feature = "tailscale")] if reset_tailscale_on_exit { info!("shutting down tailscale {ts_mode}"); @@ -1427,8 +1427,8 @@ pub async fn start_gateway( warn!("failed to reset tailscale on exit: {e}"); } } - let shutdown_grace = std::time::Duration::from_secs(5); + instrumentation_for_shutdown.shutdown(shutdown_grace).await; info!( grace_secs = shutdown_grace.as_secs(), "shutting down browser pool" diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs index 4df972865..e773cd175 100644 --- a/crates/observability/src/builder.rs +++ b/crates/observability/src/builder.rs @@ -312,7 +312,10 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { - use secrecy::Secret; + use { + crate::{Event, ScoreRecord, ScoreValue, TraceId}, + secrecy::Secret, + }; use super::*; @@ -424,6 +427,15 @@ mod tests { // Langfuse contributes two sinks: traces over OTLP and scores over the // ingestion API. It still reports as one backend to the operator. assert_eq!(built.sink.name(), "langfuse+langfuse-scores+otlp+datadog"); + assert_eq!( + built + .sink + .delivery_stats() + .into_iter() + .map(|stats| stats.name) + .collect::>(), + vec!["langfuse", "langfuse-scores", "otlp", "datadog"] + ); } #[tokio::test] @@ -438,6 +450,17 @@ mod tests { let built = build(&config, "test").built.expect("should build"); assert_eq!(built.sink.name(), "langfuse+langfuse-scores"); + + built.sink.record(Event::Score(Box::new(ScoreRecord::new( + TraceId::generate(), + "quality", + ScoreValue::Numeric(1.0), + )))); + let delivery = built.sink.delivery_stats(); + assert_eq!(delivery[0].name, "langfuse"); + assert_eq!(delivery[0].accepted, 0); + assert_eq!(delivery[1].name, "langfuse-scores"); + assert_eq!(delivery[1].accepted, 1); } #[tokio::test] diff --git a/crates/observability/src/exporters/langfuse/config.rs b/crates/observability/src/exporters/langfuse/config.rs index 1c606e6bc..2ae3c9743 100644 --- a/crates/observability/src/exporters/langfuse/config.rs +++ b/crates/observability/src/exporters/langfuse/config.rs @@ -96,6 +96,13 @@ impl LangfuseConfig { headers } + /// Auth headers required by Langfuse's v4 OTLP ingestion contract. + fn otlp_auth_headers(&self) -> BTreeMap { + let mut headers = self.auth_headers(); + headers.insert("X-Langfuse-Ingestion-Version".to_string(), "4".to_string()); + headers + } + /// Build the OTLP transport that carries traces to Langfuse. /// /// The profile is fixed to [`ExportProfile::langfuse`]: content capture is @@ -107,7 +114,7 @@ impl LangfuseConfig { OtlpTransport::new(OtlpConfig { name: "langfuse".to_string(), endpoint: self.url(OTEL_TRACES_PATH), - headers: self.auth_headers(), + headers: self.otlp_auth_headers(), timeout: self.timeout, service_name: "moltis".to_string(), service_version, @@ -120,7 +127,14 @@ impl LangfuseConfig { #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { - use super::*; + use { + super::*, + crate::{model::Event, runtime::Transport}, + wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{header, method, path}, + }, + }; fn config(host: String) -> LangfuseConfig { LangfuseConfig { @@ -177,4 +191,35 @@ mod tests { "https://cloud.langfuse.com/api/public/otel/v1/traces" ); } + + #[test] + fn otlp_headers_request_ingestion_version_four() { + let headers = config("https://cloud.langfuse.com".into()).otlp_auth_headers(); + assert_eq!( + headers + .get("X-Langfuse-Ingestion-Version") + .map(String::as_str), + Some("4") + ); + } + + #[tokio::test] + async fn ingestion_version_four_is_sent_on_the_wire() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(OTEL_TRACES_PATH)) + .and(header("x-langfuse-ingestion-version", "4")) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&server) + .await; + let transport = config(server.uri()).build_transport("test".into()); + let mut trace = crate::model::TraceRecord::new("turn"); + trace.finish(); + + transport + .send(&[Event::Trace(Box::new(trace))]) + .await + .expect("OTLP request accepted"); + } } diff --git a/crates/observability/src/exporters/langfuse/scores.rs b/crates/observability/src/exporters/langfuse/scores.rs index f62649caa..d2e126e55 100644 --- a/crates/observability/src/exporters/langfuse/scores.rs +++ b/crates/observability/src/exporters/langfuse/scores.rs @@ -2,7 +2,7 @@ //! //! Scores are the one part of the model OTLP cannot carry, so they take a //! separate path: a dedicated sink that filters the fanout down to -//! [`Event::Score`] and posts batches to the ingestion API. +//! [`Event::Score`] and posts stable requests to the Scores API. //! //! Without this sink `TurnRecorder::score` would emit into a void — the OTLP //! mapping drops score events by design, so the Langfuse trace transport alone @@ -17,37 +17,33 @@ use { }; use { - super::{ - client::LangfuseClient, - config::{INGESTION_PATH, SCORES_PATH}, - }, + super::{client::LangfuseClient, config::SCORES_PATH}, crate::{ - model::{Event, ScoreRecord, ScoreValue}, + model::{Event, ScoreDeleteRecord, ScoreRecord, ScoreValue}, runtime::{BatchConfig, BatchSink, Transport, TransportError}, sink::ObservationSink, }, }; -/// A `score-create` event in the ingestion batch envelope. +/// Value shape accepted by the dedicated Scores API. BOOLEAN uses 0/1 on this +/// API even though the recorder models it as a real boolean. #[derive(Debug, Serialize)] -struct IngestionEvent<'a> { - id: String, - timestamp: String, - #[serde(rename = "type")] - kind: &'static str, - body: ScoreBody<'a>, +#[serde(untagged)] +enum ScoreApiValue<'a> { + Numeric(f64), + Text(&'a str), } -/// Langfuse `ScoreBody`. +/// Langfuse create-score request. #[derive(Debug, Serialize)] struct ScoreBody<'a> { id: &'a str, #[serde(rename = "traceId")] - trace_id: &'a str, + trace_id: String, #[serde(rename = "observationId", skip_serializing_if = "Option::is_none")] - observation_id: Option<&'a str>, + observation_id: Option, name: &'a str, - value: &'a ScoreValue, + value: ScoreApiValue<'a>, #[serde(rename = "dataType")] data_type: &'static str, #[serde(skip_serializing_if = "Option::is_none")] @@ -56,10 +52,34 @@ struct ScoreBody<'a> { environment: Option<&'a str>, } -/// The ingestion request envelope. -#[derive(Debug, Serialize)] -struct IngestionRequest<'a> { - batch: Vec>, +impl<'a> ScoreBody<'a> { + fn new(score: &'a ScoreRecord, default_environment: Option<&'a str>) -> Self { + let (value, data_type) = match &score.value { + ScoreValue::Numeric(value) => (ScoreApiValue::Numeric(*value), "NUMERIC"), + ScoreValue::Categorical(value) => (ScoreApiValue::Text(value), "CATEGORICAL"), + ScoreValue::Boolean(value) => ( + ScoreApiValue::Numeric(if *value { + 1.0 + } else { + 0.0 + }), + "BOOLEAN", + ), + }; + Self { + id: &score.id, + trace_id: score.trace_id.as_otel_hex(), + observation_id: score + .observation_id + .as_ref() + .map(crate::model::ObservationId::as_otel_hex), + name: &score.name, + value, + data_type, + comment: score.comment.as_deref(), + environment: score.environment.as_deref().or(default_environment), + } + } } impl LangfuseClient { @@ -71,65 +91,45 @@ impl LangfuseClient { return Ok(()); } - let now = time::OffsetDateTime::now_utc() - .format(&time::format_description::well_known::Rfc3339) - .unwrap_or_default(); - - let batch: Vec> = scores - .iter() - .map(|score| IngestionEvent { - id: uuid::Uuid::new_v4().to_string(), - timestamp: now.clone(), - kind: "score-create", - body: ScoreBody { - id: &score.id, - trace_id: &score.trace_id.0, - observation_id: score.observation_id.as_ref().map(|o| o.0.as_str()), - name: &score.name, - value: &score.value, - data_type: match score.value { - ScoreValue::Numeric(_) => "NUMERIC", - ScoreValue::Categorical(_) => "CATEGORICAL", - }, - comment: score.comment.as_deref(), - environment: score - .environment - .as_deref() - .or(self.config().environment.as_deref()), - }, - }) - .collect(); - - let response = self - .post(INGESTION_PATH) - .json(&IngestionRequest { batch }) - .send() - .await?; + self.submit_scores_for_transport(scores) + .await + .map_err(anyhow::Error::new)?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - return Err(anyhow::anyhow!( - "Langfuse rejected scores: HTTP {status}: {}", - body.chars().take(512).collect::() - )); - } + debug!(count = scores.len(), "submitted scores to Langfuse"); + Ok(()) + } - // The ingestion endpoint reports per-event outcomes in a 207, so a 2xx - // alone does not mean every score landed. - let body: serde_json::Value = response.json().await.unwrap_or_default(); - if let Some(errors) = body.get("errors").and_then(|e| e.as_array()) - && !errors.is_empty() - { - return Err(anyhow::anyhow!( - "Langfuse rejected {} of {} scores: {}", - errors.len(), - scores.len(), - serde_json::to_string(errors).unwrap_or_default() - )); + async fn submit_scores_for_transport( + &self, + scores: &[ScoreRecord], + ) -> Result<(), TransportError> { + for score in scores { + let body = ScoreBody::new(score, self.config().environment.as_deref()); + let response = self + .post(SCORES_PATH) + .json(&body) + .send() + .await + .map_err(|error| TransportError::Retryable(error.to_string()))?; + + let status = response.status(); + if status.is_success() { + continue; + } + let response_body = response.text().await.unwrap_or_default(); + let message = format!( + "Langfuse rejected score {}: HTTP {status}: {}", + score.id, + response_body.chars().take(512).collect::() + ); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status == reqwest::StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + return Err(TransportError::Retryable(message)); + } + return Err(TransportError::Fatal(message)); } - - debug!(count = scores.len(), "submitted scores to Langfuse"); Ok(()) } @@ -156,9 +156,38 @@ impl LangfuseClient { body.chars().take(512).collect::() )) } + + async fn delete_score_for_transport( + &self, + deletion: &ScoreDeleteRecord, + ) -> Result<(), TransportError> { + let response = self + .delete(&format!("{SCORES_PATH}/{}", deletion.score_id)) + .send() + .await + .map_err(|error| TransportError::Retryable(error.to_string()))?; + let status = response.status(); + if status.is_success() || status == reqwest::StatusCode::NOT_FOUND { + return Ok(()); + } + let body = response.text().await.unwrap_or_default(); + let message = format!( + "Langfuse rejected score deletion {}: HTTP {status}: {}", + deletion.score_id, + body.chars().take(512).collect::() + ); + if status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status == reqwest::StatusCode::REQUEST_TIMEOUT + || status.is_server_error() + { + Err(TransportError::Retryable(message)) + } else { + Err(TransportError::Fatal(message)) + } + } } -/// Batch transport that posts scores to the ingestion API. +/// Batch transport that posts scores to the dedicated Scores API. pub struct ScoreTransport { client: Arc, } @@ -178,24 +207,22 @@ impl Transport for ScoreTransport { } async fn send(&self, batch: &[Event]) -> Result<(), TransportError> { - let scores: Vec = batch - .iter() - .filter_map(|event| match event { - Event::Score(score) => Some((**score).clone()), - _ => None, - }) - .collect(); - - if scores.is_empty() { - return Ok(()); + for event in batch { + match event { + Event::Score(score) => { + self.client + .submit_scores_for_transport(std::slice::from_ref(score.as_ref())) + .await?; + }, + Event::ScoreDelete(deletion) => { + self.client + .delete_score_for_transport(deletion.as_ref()) + .await?; + }, + _ => {}, + } } - - self.client.submit_scores(&scores).await.map_err(|error| { - // A rejected score is worth retrying: the common causes are - // transient (the trace has not landed yet, a 5xx) rather than the - // payload being permanently invalid. - TransportError::Retryable(error.to_string()) - }) + Ok(()) } } @@ -211,7 +238,10 @@ pub struct ScoreSink { impl ScoreSink { /// Spawn a score sink over `client`. #[must_use] - pub fn spawn(client: Arc, config: BatchConfig) -> Self { + pub fn spawn(client: Arc, mut config: BatchConfig) -> Self { + // Each score is a distinct HTTP mutation. Single-event batches keep + // retries and delivery statistics exact without sacrificing queue order. + config.max_batch_events = 1; Self { inner: BatchSink::spawn(Arc::new(ScoreTransport::new(client)), config), } @@ -225,7 +255,7 @@ impl ObservationSink for ScoreSink { } fn record(&self, event: Event) { - if matches!(event, Event::Score(_)) { + if matches!(event, Event::Score(_) | Event::ScoreDelete(_)) { self.inner.record(event); } } @@ -237,6 +267,10 @@ impl ObservationSink for ScoreSink { } Ok(()) } + + fn delivery_stats(&self) -> Vec { + self.inner.delivery_stats() + } } #[cfg(test)] @@ -246,7 +280,7 @@ mod tests { std::time::Duration, wiremock::{ Mock, MockServer, ResponseTemplate, - matchers::{method, path}, + matchers::{body_json, method, path}, }, }; @@ -271,23 +305,39 @@ mod tests { } fn score(name: &str) -> ScoreRecord { - ScoreRecord::new(TraceId("trace-1".into()), name, ScoreValue::Numeric(1.0)) + ScoreRecord::new( + TraceId("01234567-89ab-cdef-0123-456789abcdef".into()), + name, + ScoreValue::Numeric(1.0), + ) } #[tokio::test] - async fn scores_post_as_score_create_events() { + async fn scores_post_to_the_dedicated_api_with_wire_ids() { let server = MockServer::start().await; + let expected = serde_json::json!({ + "id": "score-1", + "traceId": "0123456789abcdef0123456789abcdef", + "observationId": "fedcba9876543210", + "name": "user-feedback", + "value": 1.0, + "dataType": "NUMERIC", + "comment": "helpful", + "environment": "production" + }); Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [{ "id": "1", "status": 201 }], "errors": [] + .and(path(SCORES_PATH)) + .and(body_json(expected)) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "id": "score-1" }))) .expect(1) .mount(&server) .await; let mut score = score("user-feedback"); - score.observation_id = Some(ObservationId("obs-1".into())); + score.id = "score-1".into(); + score.observation_id = Some(ObservationId("fedcba98-7654-3210-fedc-ba9876543210".into())); score.comment = Some("helpful".into()); LangfuseClient::new(config(server.uri())) @@ -297,25 +347,29 @@ mod tests { } #[tokio::test] - async fn partial_rejection_in_a_207_is_surfaced_as_an_error() { + async fn boolean_scores_use_the_boolean_data_type() { let server = MockServer::start().await; - // A 2xx alone does not mean the data landed: the ingestion endpoint - // reports per-event outcomes in the body. Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [], - "errors": [{ "id": "1", "status": 400, "message": "invalid value" }] + .and(path(SCORES_PATH)) + .and(body_json(serde_json::json!({ + "id": "boolean-score", + "traceId": "0123456789abcdef0123456789abcdef", + "name": "user-feedback", + "value": 0.0, + "dataType": "BOOLEAN", + "environment": "production" }))) + .respond_with(ResponseTemplate::new(200)) .mount(&server) .await; - let error = LangfuseClient::new(config(server.uri())) - .submit_scores(&[score("user-feedback")]) + let mut score = score("user-feedback"); + score.id = "boolean-score".into(); + score.value = ScoreValue::Boolean(false); + LangfuseClient::new(config(server.uri())) + .submit_scores(&[score]) .await - .expect_err("partial failure should surface"); - - assert!(error.to_string().contains("rejected 1 of 1"), "{error}"); + .expect("boolean score should be accepted"); } #[tokio::test] @@ -346,6 +400,10 @@ mod tests { serde_json::to_value(&categorical).expect("serializable"), serde_json::json!("helpful") ); + assert_eq!( + serde_json::to_value(ScoreValue::Boolean(true)).expect("serializable"), + serde_json::json!(true) + ); } #[tokio::test] @@ -368,10 +426,8 @@ mod tests { async fn the_sink_drops_trace_events_before_they_reach_the_queue() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path(INGESTION_PATH)) - .respond_with(ResponseTemplate::new(207).set_body_json(serde_json::json!({ - "successes": [], "errors": [] - }))) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(200)) .expect(1) .mount(&server) .await; @@ -443,7 +499,7 @@ mod tests { async fn a_rejected_batch_is_retryable_rather_than_dropped() { let server = MockServer::start().await; Mock::given(method("POST")) - .and(path(INGESTION_PATH)) + .and(path(SCORES_PATH)) .respond_with(ResponseTemplate::new(503)) .mount(&server) .await; @@ -458,4 +514,67 @@ mod tests { "a transient rejection must not discard user feedback, got {error:?}" ); } + + #[tokio::test] + async fn invalid_scores_are_fatal() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(400)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let batch = vec![Event::Score(Box::new(score("user-feedback")))]; + + let error = transport.send(&batch).await.expect_err("400 should fail"); + assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + } + + #[test] + fn retry_payload_is_stable() { + let mut score = score("user-feedback"); + score.id = "stable-score".into(); + score.value = ScoreValue::Boolean(true); + let first = + serde_json::to_vec(&ScoreBody::new(&score, Some("production"))).expect("serializable"); + let second = + serde_json::to_vec(&ScoreBody::new(&score, Some("production"))).expect("serializable"); + assert_eq!(first, second); + } + + #[tokio::test] + async fn create_and_delete_are_delivered_in_queue_order() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path(SCORES_PATH)) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + Mock::given(method("DELETE")) + .and(path(format!("{SCORES_PATH}/ordered-score"))) + .respond_with(ResponseTemplate::new(200)) + .mount(&server) + .await; + + let client = Arc::new(LangfuseClient::new(config(server.uri()))); + let transport = ScoreTransport::new(client); + let mut created = score("user-feedback"); + created.id = "ordered-score".into(); + let deleted = ScoreDeleteRecord::new(created.trace_id.clone(), created.id.clone()); + + transport + .send(&[ + Event::Score(Box::new(created)), + Event::ScoreDelete(Box::new(deleted)), + ]) + .await + .expect("ordered mutations should succeed"); + + let requests = server.received_requests().await.expect("request log"); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].method.as_str(), "POST"); + assert_eq!(requests[1].method.as_str(), "DELETE"); + } } diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs index 56a79268c..e9eb5345c 100644 --- a/crates/observability/src/exporters/otlp/mapping.rs +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -4,6 +4,8 @@ //! same agent run yields a content-rich payload for Langfuse and a lean //! operational span for Datadog or Grafana. See `crate::profile` for why. +use std::collections::HashSet; + use time::OffsetDateTime; use { @@ -47,11 +49,15 @@ fn clamp(text: String, max: usize) -> String { if max == 0 || text.len() <= max { return text; } - let mut end = max; + const SUFFIX: &str = "..."; + if max <= SUFFIX.len() { + return SUFFIX[..max].to_string(); + } + let mut end = max - SUFFIX.len(); while end > 0 && !text.is_char_boundary(end) { end -= 1; } - format!("{}…", &text[..end]) + format!("{}{SUFFIX}", &text[..end]) } /// Serialize a value for an attribute slot that expects JSON text. @@ -59,9 +65,36 @@ fn json_attr(key: &str, value: &serde_json::Value, max: usize) -> Option None, serde_json::Value::String(s) => Some(wire::KeyValue::string(key, clamp(s.clone(), max))), - other => serde_json::to_string(other) - .ok() - .map(|s| wire::KeyValue::string(key, clamp(s, max))), + other => bounded_json(other, max).map(|s| wire::KeyValue::string(key, s)), + } +} + +/// Serialize structured JSON without ever truncating it into invalid syntax. +fn bounded_json(value: &serde_json::Value, max: usize) -> Option { + let serialized = serde_json::to_string(value).ok()?; + if max == 0 || serialized.len() <= max { + return Some(serialized); + } + + let marker = serde_json::json!({ + "moltis_truncated": true, + "original_bytes": serialized.len(), + }) + .to_string(); + if marker.len() <= max { + return Some(marker); + } + (max >= 4).then(|| "null".to_string()) +} + +fn push_structured_attr( + out: &mut Vec, + key: &str, + value: &serde_json::Value, + max: usize, +) { + if let Some(serialized) = bounded_json(value, max) { + out.push(wire::KeyValue::string(key, serialized)); } } @@ -119,16 +152,36 @@ fn push_scope_attrs(out: &mut Vec, scope: &TraceScope, profile: /// Build the span representing a trace's root. /// -/// Langfuse reconstructs the trace record from its spans, so trace-level input, -/// output and metadata ride on a zero-duration root span. +/// Langfuse reconstructs the trace from its spans, so the completed root is a +/// real AGENT observation and also carries the trace-level compatibility attrs. #[must_use] pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { let profile = &ctx.profile; let mut attributes = Vec::new(); + let error_message = trace + .metadata + .get("error") + .and_then(serde_json::Value::as_str); if profile.emits_langfuse_attrs() { attributes.push(wire::KeyValue::string(attr::TRACE_NAME, trace.name.clone())); - attributes.push(wire::KeyValue::string(attr::OBSERVATION_TYPE, "SPAN")); + attributes.push(wire::KeyValue::string(attr::OBSERVATION_TYPE, "agent")); + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_LEVEL, + if error_message.is_some() { + Level::Error.as_str() + } else { + Level::Default.as_str() + }, + )); + if let Some(message) = error_message { + attributes.push(wire::KeyValue::string( + attr::OBSERVATION_STATUS_MESSAGE, + clamp(message.to_string(), profile.max_attribute_bytes), + )); + } + } else { + attributes.push(wire::KeyValue::string(attr::GEN_AI_OPERATION_NAME, "agent")); } push_scope_attrs(&mut attributes, &trace.scope, profile); @@ -136,12 +189,24 @@ pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { if let Some(input) = &trace.input && let Some(kv) = json_attr(attr::TRACE_INPUT, input, profile.max_attribute_bytes) { - attributes.push(kv); + attributes.push(kv.clone()); + if let Some(observation) = + json_attr(attr::OBSERVATION_INPUT, input, profile.max_attribute_bytes) + { + attributes.push(observation); + } } if let Some(output) = &trace.output && let Some(kv) = json_attr(attr::TRACE_OUTPUT, output, profile.max_attribute_bytes) { - attributes.push(kv); + attributes.push(kv.clone()); + if let Some(observation) = json_attr( + attr::OBSERVATION_OUTPUT, + output, + profile.max_attribute_bytes, + ) { + attributes.push(observation); + } } } else if profile.emits_content_metadata() { // Size without content: enough to spot a runaway prompt in an APM @@ -161,13 +226,20 @@ pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { } if profile.emits_langfuse_attrs() { - if !trace.metadata.is_empty() - && let Ok(meta) = serde_json::to_string(&trace.metadata) - { - attributes.push(wire::KeyValue::string( + if !trace.metadata.is_empty() { + let metadata = serde_json::to_value(&trace.metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, attr::TRACE_METADATA, - clamp(meta, profile.max_attribute_bytes), - )); + &metadata, + profile.max_attribute_bytes, + ); + push_structured_attr( + &mut attributes, + attr::OBSERVATION_METADATA, + &metadata, + profile.max_attribute_bytes, + ); } if trace.public { attributes.push(wire::KeyValue::bool(attr::TRACE_PUBLIC, true)); @@ -184,10 +256,12 @@ pub fn trace_to_span(trace: &TraceRecord, ctx: &ExportContext) -> wire::Span { parent_span_id: None, name: trace.name.clone(), kind: wire::SPAN_KIND_INTERNAL, - end_time_unix_nano: start.clone(), + end_time_unix_nano: unix_nanos(trace.end_time.unwrap_or(trace.timestamp)), start_time_unix_nano: start, attributes, - status: wire::Status::unset(), + status: error_message.map_or_else(wire::Status::unset, |message| { + wire::Status::error(Some(message.to_string())) + }), } } @@ -198,6 +272,18 @@ pub fn observation_to_span(obs: &ObservationRecord, ctx: &ExportContext) -> wire let mut attributes = Vec::new(); if profile.emits_langfuse_attrs() { + if let Some(trace_name) = &obs.trace_name { + attributes.push(wire::KeyValue::string(attr::TRACE_NAME, trace_name.clone())); + } + if !obs.trace_metadata.is_empty() { + let metadata = serde_json::to_value(&obs.trace_metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, + attr::TRACE_METADATA, + &metadata, + profile.max_attribute_bytes, + ); + } attributes.push(wire::KeyValue::string( attr::OBSERVATION_TYPE, obs.kind.as_str(), @@ -211,7 +297,7 @@ pub fn observation_to_span(obs: &ObservationRecord, ctx: &ExportContext) -> wire // operation name is the GenAI-conventional way to express it. attributes.push(wire::KeyValue::string( attr::GEN_AI_OPERATION_NAME, - obs.kind.as_str().to_lowercase(), + obs.kind.as_str(), )); } @@ -236,17 +322,21 @@ pub fn observation_to_span(obs: &ObservationRecord, ctx: &ExportContext) -> wire if profile.emits_langfuse_attrs() { push_prompt_attrs(&mut attributes, obs); - if !obs.metadata.is_empty() - && let Ok(meta) = serde_json::to_string(&obs.metadata) - { - attributes.push(wire::KeyValue::string( + if !obs.metadata.is_empty() { + let metadata = serde_json::to_value(&obs.metadata).unwrap_or_default(); + push_structured_attr( + &mut attributes, attr::OBSERVATION_METADATA, - clamp(meta, profile.max_attribute_bytes), - )); + &metadata, + profile.max_attribute_bytes, + ); } } - if obs.kind == crate::model::ObservationKind::Tool { + if matches!( + obs.kind, + crate::model::ObservationKind::Tool | crate::model::ObservationKind::Retriever + ) { attributes.push(wire::KeyValue::string( attr::GEN_AI_TOOL_NAME, obs.name.clone(), @@ -351,13 +441,14 @@ fn push_model_attrs( if obs.model_parameters.is_empty() { return; } - if profile.emits_langfuse_attrs() - && let Ok(params) = serde_json::to_string(&obs.model_parameters) - { - out.push(wire::KeyValue::string( + if profile.emits_langfuse_attrs() { + let parameters = serde_json::to_value(&obs.model_parameters).unwrap_or_default(); + push_structured_attr( + out, attr::OBSERVATION_MODEL_PARAMETERS, - clamp(params, profile.max_attribute_bytes), - )); + ¶meters, + profile.max_attribute_bytes, + ); } // Promote the two parameters GenAI consumers actually chart. if let Some(temp) = obs @@ -406,12 +497,13 @@ fn push_usage_attrs( // Only Langfuse prices the cache buckets; elsewhere this is an // opaque JSON blob nothing can chart. let details = usage.to_usage_details(); - if let Ok(json) = serde_json::to_string(&details) { - out.push(wire::KeyValue::string( - attr::OBSERVATION_USAGE_DETAILS, - json, - )); - } + let details = serde_json::to_value(details).unwrap_or_default(); + push_structured_attr( + out, + attr::OBSERVATION_USAGE_DETAILS, + &details, + profile.max_attribute_bytes, + ); } out.push(wire::KeyValue::int( attr::GEN_AI_USAGE_INPUT_TOKENS, @@ -423,12 +515,9 @@ fn push_usage_attrs( )); } - if profile.emits_langfuse_attrs() - && !obs.cost_details.is_empty() - && let Ok(json) = serde_json::to_string(&obs.cost_details) - { - out.push(wire::KeyValue::string(attr::OBSERVATION_COST_DETAILS, json)); - } + // Keep locally computed costs inside Moltis. Langfuse maintains versioned + // model definitions and pricing tiers; sending our static table would take + // precedence over that inference and make stale prices authoritative. } /// Managed-prompt linkage. Langfuse-only: no other backend models it. @@ -472,19 +561,18 @@ pub fn resource_attributes(ctx: &ExportContext) -> Vec { /// delivered through the backend's scoring API instead. #[must_use] pub fn batch_to_request(events: &[Event], ctx: &ExportContext) -> wire::ExportTraceServiceRequest { + let mut exported_ids = HashSet::new(); let spans: Vec = events .iter() .filter_map(|event| match event { - Event::Trace(trace) => Some(trace_to_span(trace, ctx)), - // Backends that treat spans as immutable get only the completion, - // or the same step would appear twice in the trace. - Event::ObservationStart(obs) => ctx - .profile - .emits_partial_spans() - .then(|| observation_to_span(obs, ctx)), - Event::ObservationEnd(obs) => Some(observation_to_span(obs, ctx)), - Event::Score(_) => None, + Event::Trace(trace) if trace.end_time.is_some() => Some(trace_to_span(trace, ctx)), + Event::ObservationEnd(obs) if obs.end_time.is_some() => { + Some(observation_to_span(obs, ctx)) + }, + Event::Trace(_) | Event::ObservationEnd(_) => None, + Event::Score(_) | Event::ScoreDelete(_) => None, }) + .filter(|span| exported_ids.insert((span.trace_id.clone(), span.span_id.clone()))) .collect(); wire::ExportTraceServiceRequest { @@ -677,7 +765,7 @@ mod tests { assert!(attr_value(&otel, attr::OBSERVATION_USAGE_DETAILS).is_none()); let lf = observation_to_span(&obs, &langfuse_ctx()); - assert!(attr_value(&lf, attr::OBSERVATION_COST_DETAILS).is_some()); + assert!(attr_value(&lf, attr::OBSERVATION_COST_DETAILS).is_none()); } #[test] @@ -748,7 +836,8 @@ mod tests { fn clamping_does_not_split_multibyte_characters() { // Slicing blindly at a byte offset would panic on this input. let clamped = clamp("ααααααααα".to_string(), 5); - assert!(clamped.ends_with('…')); + assert!(clamped.ends_with("...")); + assert!(clamped.len() <= 5); } // ── Shared structural behaviour ───────────────────────────────────── @@ -761,7 +850,7 @@ mod tests { assert_eq!( attr_value(&span, attr::OBSERVATION_TYPE), - Some(json!({ "stringValue": "TOOL" })) + Some(json!({ "stringValue": "tool" })) ); } @@ -887,6 +976,48 @@ mod tests { assert!(first.parent_span_id.is_none()); } + #[test] + fn completed_trace_is_a_duration_bearing_agent_observation() { + let mut trace = TraceRecord::new("turn"); + trace.timestamp = OffsetDateTime::UNIX_EPOCH; + trace.end_time = Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)); + trace.input = Some(json!({ "role": "user", "content": "hello" })); + trace.output = Some(json!({ "role": "assistant", "content": "hi" })); + let span = trace_to_span(&trace, &langfuse_ctx()); + + assert_eq!(span.start_time_unix_nano, "0"); + assert_eq!(span.end_time_unix_nano, "2000000000"); + assert_eq!( + attr_value(&span, attr::OBSERVATION_TYPE), + Some(json!({ "stringValue": "agent" })) + ); + assert!(attr_value(&span, attr::OBSERVATION_INPUT).is_some()); + assert!(attr_value(&span, attr::OBSERVATION_OUTPUT).is_some()); + // Trace attrs are retained because Langfuse builds its trace view from + // them independently of the root observation view. + assert!(attr_value(&span, attr::TRACE_INPUT).is_some()); + assert!(attr_value(&span, attr::TRACE_OUTPUT).is_some()); + } + + #[test] + fn child_spans_carry_filterable_trace_context() { + let mut obs = generation(); + obs.trace_name = Some("support-agent".into()); + obs.trace_metadata.insert("tenant".into(), json!("acme")); + let span = observation_to_span(&obs, &langfuse_ctx()); + + assert_eq!( + attr_value(&span, attr::TRACE_NAME), + Some(json!({ "stringValue": "support-agent" })) + ); + let metadata_attr = attr_value(&span, attr::TRACE_METADATA).expect("trace metadata"); + let metadata = metadata_attr["stringValue"] + .as_str() + .map(|value| serde_json::from_str::(value).expect("valid json")) + .expect("trace metadata string"); + assert_eq!(metadata["tenant"], "acme"); + } + #[test] fn string_inputs_are_not_double_encoded() { let mut trace = TraceRecord::new("turn"); @@ -961,14 +1092,13 @@ mod tests { } #[test] - fn in_progress_spans_are_dropped_for_immutable_span_backends() { + fn in_progress_spans_are_dropped_for_all_immutable_span_backends() { let mut obs = generation(); obs.end_time = None; - let events = vec![Event::ObservationStart(Box::new(obs))]; + let events = vec![Event::ObservationEnd(Box::new(obs))]; - // Langfuse upserts by id, so the live view is worth the extra span. let lf = batch_to_request(&events, &langfuse_ctx()); - assert_eq!(lf.resource_spans[0].scope_spans[0].spans.len(), 1); + assert!(lf.resource_spans[0].scope_spans[0].spans.is_empty()); // Tempo and Datadog would render this and the later completion as two // separate spans with the same id. @@ -999,7 +1129,9 @@ mod tests { #[test] fn batch_carries_resource_and_scope_identity() { - let events = vec![Event::Trace(Box::new(TraceRecord::new("turn")))]; + let mut trace = TraceRecord::new("turn"); + trace.finish(); + let events = vec![Event::Trace(Box::new(trace))]; let request = batch_to_request(&events, &langfuse_ctx()); let json = serde_json::to_value(&request).expect("serializable"); @@ -1025,4 +1157,44 @@ mod tests { // far-future timestamp in collectors. assert_eq!(span.start_time_unix_nano, "0"); } + + #[test] + fn incomplete_roots_and_duplicate_completed_ids_are_filtered() { + let incomplete = TraceRecord::new("turn"); + let mut complete = incomplete.clone(); + complete.finish(); + let observation = generation(); + let events = vec![ + Event::Trace(Box::new(incomplete)), + Event::Trace(Box::new(complete.clone())), + Event::Trace(Box::new(complete)), + Event::ObservationEnd(Box::new(observation.clone())), + Event::ObservationEnd(Box::new(observation)), + ]; + let request = batch_to_request(&events, &langfuse_ctx()); + let spans = &request.resource_spans[0].scope_spans[0].spans; + let ids: HashSet<_> = spans.iter().map(|span| span.span_id.as_str()).collect(); + + assert_eq!(spans.len(), 2); + assert_eq!(ids.len(), spans.len(), "each wire id must occur once"); + } + + #[test] + fn oversized_structured_attributes_remain_valid_json_within_the_limit() { + let mut obs = generation(); + obs.output = Some(json!({ "items": ["x".repeat(500), "y".repeat(500)] })); + obs.metadata.insert("large".into(), json!("z".repeat(500))); + let profile = ExportProfile { + max_attribute_bytes: 80, + ..ExportProfile::langfuse() + }; + let span = observation_to_span(&obs, &ctx(profile)); + + for key in [attr::OBSERVATION_OUTPUT, attr::OBSERVATION_METADATA] { + let attribute = attr_value(&span, key).expect("attribute present"); + let value = attribute["stringValue"].as_str().expect("string value"); + assert!(value.len() <= 80); + serde_json::from_str::(value).expect("valid bounded json"); + } + } } diff --git a/crates/observability/src/exporters/otlp/mod.rs b/crates/observability/src/exporters/otlp/mod.rs index 3590aa11a..0b5184270 100644 --- a/crates/observability/src/exporters/otlp/mod.rs +++ b/crates/observability/src/exporters/otlp/mod.rs @@ -189,6 +189,10 @@ impl Transport for OtlpTransport { let body = body.chars().take(512).collect::(); Err(Self::classify(status, body)) } + + fn accepts(&self, event: &Event) -> bool { + !matches!(event, Event::Score(_) | Event::ScoreDelete(_)) + } } #[cfg(test)] diff --git a/crates/observability/src/feedback.rs b/crates/observability/src/feedback.rs index 09e6157a9..c8c04a99b 100644 --- a/crates/observability/src/feedback.rs +++ b/crates/observability/src/feedback.rs @@ -31,15 +31,12 @@ pub enum FeedbackSignal { } impl FeedbackSignal { - /// Numeric value carried to the backend. - /// - /// Langfuse treats a 0/1 numeric score as a boolean for charting, which is - /// what a thumb is. + /// Boolean value carried to the backend. #[must_use] - pub const fn score_value(self) -> f64 { + pub const fn score_value(self) -> bool { match self { - Self::Positive => 1.0, - Self::Negative => 0.0, + Self::Positive => true, + Self::Negative => false, } } } @@ -187,7 +184,7 @@ pub fn feedback_score( trace_id: trace_id.clone(), observation_id: None, name: USER_FEEDBACK_SCORE.to_string(), - value: ScoreValue::Numeric(signal.score_value()), + value: ScoreValue::Boolean(signal.score_value()), comment, environment, } @@ -298,9 +295,9 @@ mod tests { } #[test] - fn signals_map_to_boolean_friendly_numbers() { - assert!((FeedbackSignal::Positive.score_value() - 1.0).abs() < f64::EPSILON); - assert!(FeedbackSignal::Negative.score_value().abs() < f64::EPSILON); + fn signals_map_to_booleans() { + assert!(FeedbackSignal::Positive.score_value()); + assert!(!FeedbackSignal::Negative.score_value()); } #[test] @@ -367,7 +364,7 @@ mod tests { ); assert_eq!(score.name, USER_FEEDBACK_SCORE); - assert_eq!(score.value, ScoreValue::Numeric(1.0)); + assert_eq!(score.value, ScoreValue::Boolean(true)); assert_eq!(score.comment.as_deref(), Some("great answer")); assert_eq!(score.environment.as_deref(), Some("production")); } diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs index 91f47ab69..4922801a9 100644 --- a/crates/observability/src/lib.rs +++ b/crates/observability/src/lib.rs @@ -7,7 +7,8 @@ //! //! * **Langfuse** gets the full conversation — prompts, completions, tool //! arguments and results — plus the observation taxonomy, cache-aware token -//! usage, cost, and managed-prompt linkage. +//! usage, and reaction feedback. Langfuse infers cost from the model and +//! usage details. //! * **Grafana, Datadog, Honeycomb** get operational shape only: latency, //! errors, model, token counts. No conversation content, no per-user //! cardinality. @@ -32,13 +33,13 @@ pub use { FeedbackSignal, FeedbackVocabulary, USER_FEEDBACK_SCORE, feedback_score, feedback_score_id, }, model::{ - Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, - TokenUsage, TraceId, TraceRecord, TraceScope, + Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreDeleteRecord, + ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, }, pricing::{ModelPrice, cost_details, price_for, total_cost}, profile::{ContentCapture, ExportProfile, Vocabulary}, recent::{recent_trace, remember_trace}, - recorder::{RecorderSettings, StepGuard, TurnRecorder}, + recorder::{RecorderSettings, StepGuard, TurnRecorder, wait_for_active_turns}, redact::RedactionPolicy, runtime::{BatchConfig, BatchSink, SinkStatsSnapshot, Transport, TransportError}, sink::{ diff --git a/crates/observability/src/model.rs b/crates/observability/src/model.rs index 5b9e99ccf..6095a6143 100644 --- a/crates/observability/src/model.rs +++ b/crates/observability/src/model.rs @@ -87,8 +87,8 @@ fn fnv1a64(bytes: &[u8]) -> u64 { // ── Taxonomy ──────────────────────────────────────────────────────────────── -/// Observation kind. Mirrors Langfuse's `ObservationType` enum verbatim so the -/// exporter never has to guess at a mapping. +/// Observation kind. Mirrors Langfuse's observation taxonomy so the exporter +/// never has to guess at a mapping. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub enum ObservationKind { @@ -115,20 +115,20 @@ pub enum ObservationKind { } impl ObservationKind { - /// Langfuse wire representation. + /// Langfuse OTLP wire representation. #[must_use] pub const fn as_str(self) -> &'static str { match self { - Self::Span => "SPAN", - Self::Event => "EVENT", - Self::Generation => "GENERATION", - Self::Agent => "AGENT", - Self::Tool => "TOOL", - Self::Chain => "CHAIN", - Self::Retriever => "RETRIEVER", - Self::Evaluator => "EVALUATOR", - Self::Embedding => "EMBEDDING", - Self::Guardrail => "GUARDRAIL", + Self::Span => "span", + Self::Event => "event", + Self::Generation => "generation", + Self::Agent => "agent", + Self::Tool => "tool", + Self::Chain => "chain", + Self::Retriever => "retriever", + Self::Evaluator => "evaluator", + Self::Embedding => "embedding", + Self::Guardrail => "guardrail", } } @@ -303,6 +303,9 @@ pub struct TraceRecord { /// Wall-clock start. #[serde(with = "time::serde::rfc3339")] pub timestamp: OffsetDateTime, + /// Wall-clock end; `None` until the turn is complete. + #[serde(with = "time::serde::rfc3339::option")] + pub end_time: Option, /// Trace-scoped attributes. pub scope: TraceScope, /// Structured context. @@ -323,6 +326,7 @@ impl TraceRecord { id: TraceId::generate(), name: name.into(), timestamp: OffsetDateTime::now_utc(), + end_time: None, scope: TraceScope::default(), metadata: Metadata::new(), input: None, @@ -330,6 +334,11 @@ impl TraceRecord { public: false, } } + + /// Stamp the trace end time as now. + pub fn finish(&mut self) { + self.end_time = Some(OffsetDateTime::now_utc()); + } } /// A single observation within a trace. @@ -345,6 +354,10 @@ pub struct ObservationRecord { pub kind: ObservationKind, /// Human-readable name. pub name: String, + /// Owning trace's display name, copied for stateless OTLP export. + pub trace_name: Option, + /// Owning trace's filterable metadata, copied for stateless OTLP export. + pub trace_metadata: Metadata, /// Trace-scoped attributes, copied from the owning trace. pub scope: TraceScope, /// Wall-clock start. @@ -390,6 +403,8 @@ impl ObservationRecord { parent_id: None, kind, name: name.into(), + trace_name: None, + trace_metadata: Metadata::new(), scope: TraceScope::default(), start_time: OffsetDateTime::now_utc(), end_time: None, @@ -422,6 +437,14 @@ impl ObservationRecord { self } + /// Copy trace identity and filterable metadata onto this observation. + #[must_use] + pub fn with_trace_context(mut self, name: String, metadata: Metadata) -> Self { + self.trace_name = Some(name); + self.trace_metadata = metadata; + self + } + /// Stamp the end time as now. pub fn finish(&mut self) { self.end_time = Some(OffsetDateTime::now_utc()); @@ -438,10 +461,12 @@ impl ObservationRecord { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum ScoreValue { - /// Numeric score. Also carries boolean scores as 0.0/1.0. + /// Numeric score. Numeric(f64), /// Categorical score, e.g. "helpful". Categorical(String), + /// Boolean score, e.g. end-user approval. + Boolean(bool), } /// A quality score attached to a trace or observation. @@ -463,6 +488,27 @@ pub struct ScoreRecord { pub environment: Option, } +/// Delete a previously recorded score through the same ordered queue used for +/// score creation. Keeping both mutations on one transport prevents an older +/// queued create from racing a direct deletion and recreating the score. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScoreDeleteRecord { + /// Trace retained for sink routing and diagnostics. + pub trace_id: TraceId, + /// Stable Langfuse score id to delete. + pub score_id: String, +} + +impl ScoreDeleteRecord { + #[must_use] + pub fn new(trace_id: TraceId, score_id: impl Into) -> Self { + Self { + trace_id, + score_id: score_id.into(), + } + } +} + impl ScoreRecord { /// Build a score with a generated id. #[must_use] @@ -482,14 +528,14 @@ impl ScoreRecord { /// A single unit of work handed to a sink. #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Event { - /// Create or update a trace. + /// A trace completed. Trace(Box), - /// An observation began. Emitted eagerly so long runs are visible live. - ObservationStart(Box), /// An observation finished. ObservationEnd(Box), /// A score was produced. Score(Box), + /// A score was retracted. + ScoreDelete(Box), } impl Event { @@ -498,8 +544,9 @@ impl Event { pub fn trace_id(&self) -> &TraceId { match self { Self::Trace(t) => &t.id, - Self::ObservationStart(o) | Self::ObservationEnd(o) => &o.trace_id, + Self::ObservationEnd(o) => &o.trace_id, Self::Score(s) => &s.trace_id, + Self::ScoreDelete(s) => &s.trace_id, } } } diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs index 8d53899bc..ad3959cce 100644 --- a/crates/observability/src/profile.rs +++ b/crates/observability/src/profile.rs @@ -73,14 +73,6 @@ pub struct ExportProfile { /// Langfuse prices these; an APM generally has no model price table, so /// the same numbers are better served from the Prometheus endpoint. pub emit_usage: bool, - /// Whether to export the in-progress span emitted when a step opens. - /// - /// Langfuse upserts observations by id, so the eager start makes a - /// long-running turn visible while it is still executing and the closing - /// update fills in model, usage and output. A plain OTLP collector treats - /// spans as immutable: it would render the start and the completion as two - /// separate spans with the same id, duplicating every step in the trace. - pub emit_partial_spans: bool, /// Ceiling on any single exported attribute value, in bytes. pub max_attribute_bytes: usize, } @@ -96,7 +88,6 @@ impl ExportProfile { emit_session_id: true, emit_tags: true, emit_usage: true, - emit_partial_spans: true, max_attribute_bytes: 32_768, } } @@ -113,7 +104,6 @@ impl ExportProfile { emit_session_id: true, emit_tags: true, emit_usage: true, - emit_partial_spans: false, max_attribute_bytes: 4_096, } } @@ -130,7 +120,6 @@ impl ExportProfile { emit_session_id: true, emit_tags: false, emit_usage: true, - emit_partial_spans: false, max_attribute_bytes: 4_096, } } @@ -147,12 +136,6 @@ impl ExportProfile { self.content.allows_bodies() } - /// Whether in-progress spans should be exported. - #[must_use] - pub const fn emits_partial_spans(&self) -> bool { - self.emit_partial_spans - } - /// Whether content-derived structural metadata (lengths, counts) may be /// written. Suppressed only under [`ContentCapture::None`]. #[must_use] @@ -221,16 +204,6 @@ mod tests { assert!(!ExportProfile::default().emits_bodies()); } - #[test] - fn only_langfuse_receives_in_progress_spans() { - // Langfuse upserts by observation id, so a start span becomes the - // live view. A plain OTLP collector treats spans as immutable and - // would render start and completion as two duplicate spans. - assert!(ExportProfile::langfuse().emits_partial_spans()); - assert!(!ExportProfile::otel_generic().emits_partial_spans()); - assert!(!ExportProfile::datadog().emits_partial_spans()); - } - #[test] fn langfuse_allows_larger_attributes_than_an_apm() { assert!( diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index ba32ed345..936872cdf 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -8,16 +8,19 @@ //! Every entry point is a no-op when no sink is installed, and the constructor //! returns `None` in that case so callers pay nothing beyond one atomic read. -use std::sync::{Arc, Mutex}; +use std::sync::{ + Arc, LazyLock, Mutex, + atomic::{AtomicUsize, Ordering}, +}; -use time::OffsetDateTime; +use {time::OffsetDateTime, tokio::sync::Notify}; use crate::{ model::{ Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, }, - redact::RedactionPolicy, + redact::{REDACTED, RedactionPolicy}, sink::{self, ObservationSink}, }; @@ -71,7 +74,44 @@ pub struct TurnRecorder { /// The run's root observation, parent of every step. root_id: ObservationId, /// Trace record, retained so the closing update carries the final output. - trace: Mutex, + trace: Arc>, + /// Present only for turns opened through the process-wide sink. + _active_turn: Option, +} + +static ACTIVE_TURNS: AtomicUsize = AtomicUsize::new(0); +static ACTIVE_TURNS_CHANGED: LazyLock = LazyLock::new(Notify::new); + +struct ActiveTurnGuard; + +impl ActiveTurnGuard { + fn new() -> Self { + ACTIVE_TURNS.fetch_add(1, Ordering::AcqRel); + Self + } +} + +impl Drop for ActiveTurnGuard { + fn drop(&mut self) { + ACTIVE_TURNS.fetch_sub(1, Ordering::AcqRel); + ACTIVE_TURNS_CHANGED.notify_waiters(); + } +} + +/// Wait until all turns opened through the process-wide sink have closed. +/// +/// Returns `false` when the deadline expires first. +pub async fn wait_for_active_turns(timeout: std::time::Duration) -> bool { + let deadline = tokio::time::Instant::now() + timeout; + loop { + let changed = ACTIVE_TURNS_CHANGED.notified(); + if ACTIVE_TURNS.load(Ordering::Acquire) == 0 { + return true; + } + if tokio::time::timeout_at(deadline, changed).await.is_err() { + return false; + } + } } impl TurnRecorder { @@ -83,7 +123,13 @@ impl TurnRecorder { scope: TraceScope, settings: RecorderSettings, ) -> Option { - Self::begin_with_sink(sink::global_sink()?, name, scope, settings) + Self::begin_inner( + sink::global_sink()?, + name, + scope, + settings, + Some(ActiveTurnGuard::new()), + ) } /// Begin recording a turn against an explicit sink. @@ -98,6 +144,16 @@ impl TurnRecorder { name: impl Into, scope: TraceScope, settings: RecorderSettings, + ) -> Option { + Self::begin_inner(sink, name, scope, settings, None) + } + + fn begin_inner( + sink: Arc, + name: impl Into, + scope: TraceScope, + settings: RecorderSettings, + active_turn: Option, ) -> Option { if !settings.sampled() { return None; @@ -117,9 +173,9 @@ impl TurnRecorder { trace_id: trace_id.clone(), scope, root_id, - trace: Mutex::new(trace.clone()), + trace: Arc::new(Mutex::new(trace)), + _active_turn: active_turn, }; - recorder.sink.record(Event::Trace(Box::new(trace))); Some(recorder) } @@ -149,8 +205,10 @@ impl TurnRecorder { /// Attach a metadata entry to the trace. pub fn set_metadata(&self, key: impl Into, value: serde_json::Value) { + let key = key.into(); + let value = self.redact_metadata_value(&key, value); self.with_trace(|trace| { - trace.metadata.insert(key.into(), value); + trace.metadata.insert(key, value); }); } @@ -173,14 +231,10 @@ impl TurnRecorder { .with_parent(parent.or_else(|| Some(self.root_id.clone()))) .with_scope(self.scope.clone()); - // Emit the start eagerly so a long-running turn is visible in the - // backend while it is still executing, rather than only at the end. - self.sink - .record(Event::ObservationStart(Box::new(record.clone()))); - StepGuard { sink: Arc::clone(&self.sink), settings: self.settings.clone(), + trace: Arc::clone(&self.trace), record: Some(record), } } @@ -188,7 +242,7 @@ impl TurnRecorder { /// Record a score against this turn. pub fn score(&self, name: impl Into, value: ScoreValue, comment: Option) { let mut score = ScoreRecord::new(self.trace_id.clone(), name, value); - score.comment = comment; + score.comment = comment.map(|value| self.settings.redaction.redact_str(&value)); score.environment = self.scope.environment.clone(); self.sink.record(Event::Score(Box::new(score))); } @@ -198,7 +252,17 @@ impl TurnRecorder { /// Takes `&self` rather than `self` so the recorder can be shared across /// the concurrently-executing tool futures in the agent loop. pub fn finish(&self) { - let trace = self.snapshot_trace(); + let trace = { + let mut guard = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if guard.end_time.is_some() { + return; + } + guard.end_time = Some(OffsetDateTime::now_utc()); + guard.clone() + }; self.sink.record(Event::Trace(Box::new(trace))); } @@ -216,11 +280,12 @@ impl TurnRecorder { f(&mut guard); } - fn snapshot_trace(&self) -> TraceRecord { - self.trace - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .clone() + fn redact_metadata_value(&self, key: &str, value: serde_json::Value) -> serde_json::Value { + if self.settings.redaction.is_sensitive_key(key) { + serde_json::Value::String(REDACTED.to_string()) + } else { + self.settings.redaction.redact(&value) + } } } @@ -229,6 +294,7 @@ impl TurnRecorder { pub struct StepGuard { sink: Arc, settings: RecorderSettings, + trace: Arc>, /// `None` once the guard has emitted, so drop does not emit twice. record: Option, } @@ -327,8 +393,21 @@ impl StepGuard { /// Attach a metadata entry. pub fn set_metadata(&mut self, key: impl Into, value: serde_json::Value) { + let key = key.into(); + let value = if self.settings.redaction.is_sensitive_key(&key) { + serde_json::Value::String(REDACTED.to_string()) + } else { + self.settings.redaction.redact(&value) + }; if let Some(record) = self.record.as_mut() { - record.metadata.insert(key.into(), value); + record.metadata.insert(key, value); + } + } + + /// Attach output-derived metadata while honoring the output capture switch. + pub fn set_output_metadata(&mut self, key: impl Into, value: serde_json::Value) { + if self.capture_output_allowed() { + self.set_metadata(key, value); } } @@ -336,14 +415,14 @@ impl StepGuard { pub fn set_level(&mut self, level: Level, message: Option) { if let Some(record) = self.record.as_mut() { record.level = level; - record.status_message = message; + record.status_message = message.map(|value| self.settings.redaction.redact_str(&value)); } } /// Mark the step failed. The span is still emitted on drop. pub fn fail(&mut self, message: impl Into) { if let Some(record) = self.record.as_mut() { - record.fail(message); + record.fail(self.settings.redaction.redact_str(&message.into())); } } @@ -356,6 +435,14 @@ impl StepGuard { let Some(mut record) = self.record.take() else { return; }; + { + let trace = self + .trace + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + record.trace_name = Some(trace.name.clone()); + record.trace_metadata = trace.metadata.clone(); + } record.finish(); self.sink.record(Event::ObservationEnd(Box::new(record))); } @@ -364,14 +451,18 @@ impl StepGuard { /// switch, since they are the most likely place for credentials to appear. fn capture_input_allowed(&self) -> bool { match self.record.as_ref().map(|r| r.kind) { - Some(ObservationKind::Tool) => self.settings.capture_tool_io, + Some(ObservationKind::Tool | ObservationKind::Retriever) => { + self.settings.capture_tool_io + }, _ => self.settings.capture_input, } } fn capture_output_allowed(&self) -> bool { match self.record.as_ref().map(|r| r.kind) { - Some(ObservationKind::Tool) => self.settings.capture_tool_io, + Some(ObservationKind::Tool | ObservationKind::Retriever) => { + self.settings.capture_tool_io + }, _ => self.settings.capture_output, } } @@ -379,12 +470,19 @@ impl StepGuard { impl Drop for StepGuard { fn drop(&mut self) { - // An early return through `?` must still close the span; otherwise the - // backend shows a step that never ended. + if let Some(record) = self.record.as_mut() { + record.fail("observation cancelled before completion"); + } self.emit(); } } +impl Drop for TurnRecorder { + fn drop(&mut self) { + self.finish_with_error("agent run cancelled before completion"); + } +} + #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { @@ -451,11 +549,8 @@ mod tests { } } - /// Serialises tests that touch the process-wide sink registry. - static GLOBAL_LOCK: StdMutex<()> = StdMutex::new(()); - fn with_sink(f: impl FnOnce(Arc) -> R) -> R { - let _guard = GLOBAL_LOCK + let _guard = sink::GLOBAL_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); let collected = CollectingSink::new(); @@ -478,7 +573,7 @@ mod tests { #[test] fn returns_none_when_no_sink_is_installed() { - let _guard = GLOBAL_LOCK + let _guard = sink::GLOBAL_TEST_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); sink::clear_global_sink(); @@ -556,19 +651,13 @@ mod tests { } #[test] - fn a_start_event_is_emitted_before_the_step_completes() { + fn no_observation_event_is_emitted_before_the_step_completes() { with_sink(|collected| { let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) .expect("sink installed"); let step = recorder.step(ObservationKind::Generation, "llm"); - // A multi-minute turn should be visible while it is still running. - let starts = collected - .events() - .into_iter() - .filter(|e| matches!(e, Event::ObservationStart(_))) - .count(); - assert_eq!(starts, 1); + assert!(collected.events().is_empty()); step.finish(); recorder.finish(); @@ -588,9 +677,36 @@ mod tests { let observations = collected.observations(); assert_eq!(observations.len(), 1, "dropped step must still be emitted"); assert!(observations[0].end_time.is_some()); + assert_eq!(observations[0].level, Level::Error); + assert_eq!( + observations[0].status_message.as_deref(), + Some("observation cancelled before completion") + ); }); } + #[test] + fn dropping_an_unfinished_turn_emits_an_error_root() { + let collected = CollectingSink::new(); + let recorder = TurnRecorder::begin_with_sink( + collected.clone(), + "turn", + scope(), + RecorderSettings::default(), + ) + .expect("sampled"); + + drop(recorder); + + let traces = collected.traces(); + assert_eq!(traces.len(), 1); + assert_eq!( + traces[0].metadata["error"], + json!("agent run cancelled before completion") + ); + assert!(traces[0].end_time.is_some()); + } + #[test] fn finishing_a_step_does_not_emit_it_twice() { with_sink(|collected| { @@ -655,6 +771,27 @@ mod tests { }); } + #[test] + fn output_capture_switch_suppresses_output_derived_metadata() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_output: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + let mut step = recorder.step(ObservationKind::Generation, "llm"); + step.set_output_metadata("reasoning", json!("private reasoning")); + step.finish(); + recorder.finish(); + + assert!( + !collected.observations()[0] + .metadata + .contains_key("reasoning") + ); + }); + } + #[test] fn tool_io_switch_is_independent_of_the_generation_switches() { with_sink(|collected| { @@ -691,6 +828,28 @@ mod tests { }); } + #[test] + fn retriever_io_uses_the_tool_capture_switch() { + with_sink(|collected| { + let settings = RecorderSettings { + capture_input: true, + capture_output: true, + capture_tool_io: false, + ..Default::default() + }; + let recorder = TurnRecorder::begin("turn", scope(), settings).expect("sink installed"); + let mut retriever = recorder.step(ObservationKind::Retriever, "memory-search"); + retriever.set_input(json!({ "query": "secret" })); + retriever.set_output(json!({ "matches": ["private"] })); + retriever.finish(); + recorder.finish(); + + let observed = &collected.observations()[0]; + assert!(observed.input.is_none()); + assert!(observed.output.is_none()); + }); + } + #[test] fn first_token_marker_records_only_the_first_call() { with_sink(|collected| { @@ -738,6 +897,38 @@ mod tests { }); } + #[test] + fn metadata_status_and_score_comments_are_redacted() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_metadata("api_key", json!("raw-trace-secret")); + recorder.set_metadata("context", json!({ "password": "raw-password" })); + + let mut step = recorder.step(ObservationKind::Tool, "exec"); + step.set_metadata("access_token", json!("raw-step-secret")); + step.set_level( + Level::Warning, + Some("Bearer abcdefghijklmnopqrstuvwxyz".into()), + ); + step.finish(); + recorder.score( + "review", + ScoreValue::Boolean(true), + Some("sk-live-abcdefghijkl".into()), + ); + recorder.finish(); + + let rendered = serde_json::to_string(&collected.events()).expect("serializable"); + assert!(!rendered.contains("raw-trace-secret")); + assert!(!rendered.contains("raw-password")); + assert!(!rendered.contains("raw-step-secret")); + assert!(!rendered.contains("Bearer abcdefghijklmnopqrstuvwxyz")); + assert!(!rendered.contains("sk-live-abcdefghijkl")); + assert!(rendered.contains(REDACTED)); + }); + } + #[test] fn usage_is_priced_when_the_model_is_known() { with_sink(|collected| { @@ -833,6 +1024,29 @@ mod tests { }); } + #[test] + fn trace_and_observation_are_emitted_once_after_completion() { + with_sink(|collected| { + let recorder = TurnRecorder::begin("turn", scope(), RecorderSettings::default()) + .expect("sink installed"); + recorder.set_metadata("tenant", json!("acme")); + recorder.step(ObservationKind::Generation, "llm").finish(); + recorder.finish(); + recorder.finish(); + + let traces = collected.traces(); + assert_eq!(traces.len(), 1); + assert!(traces[0].end_time.is_some()); + let observations = collected.observations(); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].trace_name.as_deref(), Some("turn")); + assert_eq!( + observations[0].trace_metadata.get("tenant"), + Some(&json!("acme")) + ); + }); + } + #[test] fn finish_with_error_records_the_failure_on_the_trace() { with_sink(|collected| { diff --git a/crates/observability/src/redact.rs b/crates/observability/src/redact.rs index e8be39b48..76497c89c 100644 --- a/crates/observability/src/redact.rs +++ b/crates/observability/src/redact.rs @@ -146,7 +146,14 @@ impl RedactionPolicy { if trimmed.len() < 8 { return false; } - SECRET_PREFIXES.iter().any(|p| trimmed.starts_with(p)) + SECRET_PREFIXES.iter().any(|prefix| { + trimmed.match_indices(prefix).any(|(index, _)| { + index == 0 + || trimmed[..index].chars().next_back().is_some_and(|c| { + c.is_whitespace() || matches!(c, '=' | ':' | '\'' | '"' | '`') + }) + }) + }) } /// Redact a JSON value in place-equivalent fashion, returning a clean copy. @@ -255,6 +262,19 @@ mod tests { assert_eq!(out["note"], json!("the deploy finished successfully")); } + #[test] + fn redacts_credentials_embedded_in_shell_output_and_commands() { + let policy = RedactionPolicy::default(); + for value in [ + "OPENAI_API_KEY=sk-abcdefghijklmnop", + "curl -H 'Authorization: Bearer abcdefghijklmnop' https://example.com", + "request failed: xoxb-example", + ] { + assert_eq!(policy.redact(&json!(value)), json!(REDACTED), "{value}"); + } + assert!(!policy.looks_like_secret("ordinary task-sk-result text")); + } + #[test] fn redacts_pem_private_key_blocks() { let policy = RedactionPolicy::default(); diff --git a/crates/observability/src/runtime.rs b/crates/observability/src/runtime.rs index 766059be1..9469a8bc9 100644 --- a/crates/observability/src/runtime.rs +++ b/crates/observability/src/runtime.rs @@ -10,7 +10,7 @@ use std::{ sync::{ - Arc, + Arc, RwLock, atomic::{AtomicU64, Ordering}, }, time::Duration, @@ -44,6 +44,12 @@ pub trait Transport: Send + Sync + 'static { /// Deliver a batch. Called from the background task only. async fn send(&self, batch: &[Event]) -> Result<(), TransportError>; + /// Whether this transport can represent `event`. This runs on the recording + /// hot path and must be fast and non-blocking. + fn accepts(&self, _event: &Event) -> bool { + true + } + /// Approximate serialized size of `event`, used for batch sizing. fn estimate_bytes(&self, event: &Event) -> usize { serde_json::to_vec(event).map(|v| v.len()).unwrap_or(1024) @@ -88,6 +94,7 @@ impl Default for BatchConfig { /// Counters describing sink health, surfaced in the settings UI. #[derive(Debug, Default)] pub struct SinkStats { + name: String, /// Events accepted into the queue. pub accepted: AtomicU64, /// Events dropped because the queue was full. @@ -98,11 +105,20 @@ pub struct SinkStats { pub batches_sent: AtomicU64, /// Batches that failed permanently. pub batches_failed: AtomicU64, + /// Events delivered successfully. + pub delivered: AtomicU64, + /// Retry attempts made after retryable failures. + pub retries: AtomicU64, + last_success_at: RwLock>, + last_error: RwLock>, + last_error_at: RwLock>, } /// Point-in-time snapshot of [`SinkStats`]. -#[derive(Debug, Clone, Copy, serde::Serialize)] +#[derive(Debug, Clone, serde::Serialize)] pub struct SinkStatsSnapshot { + /// Sink whose counters are represented by this snapshot. + pub name: String, /// Events accepted into the queue. pub accepted: u64, /// Events dropped because the queue was full. @@ -113,20 +129,73 @@ pub struct SinkStatsSnapshot { pub batches_sent: u64, /// Batches that failed permanently. pub batches_failed: u64, + /// Events delivered successfully. + pub delivered: u64, + /// Retry attempts made after retryable failures. + pub retries: u64, + /// RFC 3339 timestamp of the latest successful delivery. + pub last_success_at: Option, + /// Most recent delivery error. + pub last_error: Option, + /// RFC 3339 timestamp of the most recent delivery error. + pub last_error_at: Option, } impl SinkStats { + fn new(name: String) -> Self { + Self { + name, + ..Default::default() + } + } + /// Read every counter. #[must_use] pub fn snapshot(&self) -> SinkStatsSnapshot { SinkStatsSnapshot { + name: self.name.clone(), accepted: self.accepted.load(Ordering::Relaxed), dropped_queue_full: self.dropped_queue_full.load(Ordering::Relaxed), dropped_failed: self.dropped_failed.load(Ordering::Relaxed), batches_sent: self.batches_sent.load(Ordering::Relaxed), batches_failed: self.batches_failed.load(Ordering::Relaxed), + delivered: self.delivered.load(Ordering::Relaxed), + retries: self.retries.load(Ordering::Relaxed), + last_success_at: read_state(&self.last_success_at), + last_error: read_state(&self.last_error), + last_error_at: read_state(&self.last_error_at), } } + + fn record_success(&self, count: u64) { + self.delivered.fetch_add(count, Ordering::Relaxed); + write_state(&self.last_success_at, now_rfc3339()); + } + + fn record_error(&self, reason: String) { + write_state(&self.last_error, Some(reason)); + write_state(&self.last_error_at, now_rfc3339()); + } +} + +fn read_state(state: &RwLock>) -> Option { + state + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() +} + +fn write_state(state: &RwLock>, value: Option) { + match state.write() { + Ok(mut guard) => *guard = value, + Err(poisoned) => *poisoned.into_inner() = value, + } +} + +fn now_rfc3339() -> Option { + time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .ok() } /// Queue message: either an event or a flush barrier. @@ -138,6 +207,7 @@ enum Message { /// An [`ObservationSink`] that batches into a [`Transport`]. pub struct BatchSink { name: String, + transport: Arc, tx: mpsc::Sender, stats: Arc, } @@ -147,8 +217,8 @@ impl BatchSink { #[must_use] pub fn spawn(transport: Arc, config: BatchConfig) -> Self { let (tx, rx) = mpsc::channel(config.queue_capacity); - let stats = Arc::new(SinkStats::default()); let name = transport.name().to_string(); + let stats = Arc::new(SinkStats::new(name.clone())); tokio::spawn(export_loop( rx, @@ -157,7 +227,12 @@ impl BatchSink { Arc::clone(&stats), )); - Self { name, tx, stats } + Self { + name, + transport, + tx, + stats, + } } /// Health counters for this sink. @@ -174,6 +249,10 @@ impl ObservationSink for BatchSink { } fn record(&self, event: Event) { + if !self.transport.accepts(&event) { + return; + } + // `try_send` never awaits: a saturated queue drops the event rather // than applying backpressure to the caller's turn. match self.tx.try_send(Message::Event(Box::new(event))) { @@ -210,19 +289,24 @@ impl ObservationSink for BatchSink { } async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { - let (ack_tx, ack_rx) = oneshot::channel(); - if self.tx.send(Message::Flush(ack_tx)).await.is_err() { - return Err(anyhow::anyhow!("export task for {} has stopped", self.name)); - } - match tokio::time::timeout(timeout, ack_rx).await { - Ok(Ok(())) => Ok(()), - Ok(Err(_)) => Err(anyhow::anyhow!( - "export task for {} dropped the flush barrier", - self.name - )), + let flush = async { + let (ack_tx, ack_rx) = oneshot::channel(); + if self.tx.send(Message::Flush(ack_tx)).await.is_err() { + return Err(anyhow::anyhow!("export task for {} has stopped", self.name)); + } + ack_rx.await.map_err(|_| { + anyhow::anyhow!("export task for {} dropped the flush barrier", self.name) + }) + }; + match tokio::time::timeout(timeout, flush).await { + Ok(result) => result, Err(_) => Err(anyhow::anyhow!("flush of {} timed out", self.name)), } } + + fn delivery_stats(&self) -> Vec { + vec![self.stats()] + } } /// Drain the queue, batching by count, size and time. @@ -294,6 +378,7 @@ async fn deliver( match transport.send(batch).await { Ok(()) => { stats.batches_sent.fetch_add(1, Ordering::Relaxed); + stats.record_success(count); trace!(sink = transport.name(), events = count, "batch delivered"); break; }, @@ -307,6 +392,7 @@ async fn deliver( ); stats.batches_failed.fetch_add(1, Ordering::Relaxed); stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + stats.record_error(reason); break; }, Err(TransportError::Retryable(reason)) if attempt < config.max_retries => { @@ -318,6 +404,8 @@ async fn deliver( %reason, "retrying observability export" ); + stats.retries.fetch_add(1, Ordering::Relaxed); + stats.record_error(reason); tokio::time::sleep(delay).await; attempt += 1; }, @@ -331,6 +419,7 @@ async fn deliver( ); stats.batches_failed.fetch_add(1, Ordering::Relaxed); stats.dropped_failed.fetch_add(count, Ordering::Relaxed); + stats.record_error(reason); break; }, } @@ -437,6 +526,8 @@ mod tests { assert_eq!(transport.batches(), vec![2]); assert_eq!(sink.stats().accepted, 2); + assert_eq!(sink.stats().delivered, 2); + assert!(sink.stats().last_success_at.is_some()); } #[tokio::test] @@ -491,6 +582,10 @@ mod tests { assert_eq!(transport.batches(), vec![1]); assert_eq!(sink.stats().batches_sent, 1); assert_eq!(sink.stats().dropped_failed, 0); + assert_eq!(sink.stats().delivered, 1); + assert_eq!(sink.stats().retries, 2); + assert!(sink.stats().last_error.is_some()); + assert!(sink.stats().last_success_at.is_some()); } #[tokio::test] @@ -511,6 +606,9 @@ mod tests { assert!(transport.batches().is_empty()); assert_eq!(sink.stats().dropped_failed, 1); assert_eq!(sink.stats().batches_failed, 1); + assert_eq!(sink.stats().retries, 2); + assert_eq!(sink.stats().delivered, 0); + assert!(sink.stats().last_error_at.is_some()); } #[tokio::test] @@ -528,6 +626,12 @@ mod tests { assert_eq!(sink.stats().batches_failed, 1); assert_eq!(sink.stats().dropped_failed, 1); + assert_eq!(sink.stats().retries, 0); + assert!( + sink.stats() + .last_error + .is_some_and(|error| error.contains("unauthorized")) + ); } #[tokio::test] @@ -557,6 +661,7 @@ mod tests { stats.accepted < 500, "queue must not have absorbed every event" ); + assert_eq!(stats.accepted + stats.dropped_queue_full, 500); } #[tokio::test] diff --git a/crates/observability/src/sink.rs b/crates/observability/src/sink.rs index c80307037..edaaa93de 100644 --- a/crates/observability/src/sink.rs +++ b/crates/observability/src/sink.rs @@ -12,7 +12,7 @@ use std::{ use async_trait::async_trait; -use crate::model::Event; +use crate::{model::Event, runtime::SinkStatsSnapshot}; /// Destination for observability events. #[async_trait] @@ -25,6 +25,11 @@ pub trait ObservationSink: Send + Sync { /// Flush pending events, giving up after `timeout`. async fn flush(&self, timeout: Duration) -> anyhow::Result<()>; + + /// Point-in-time delivery health for this sink and any sinks below it. + fn delivery_stats(&self) -> Vec { + Vec::new() + } } /// Fans one event out to several sinks, so Langfuse and an OTLP collector can @@ -99,6 +104,13 @@ impl ObservationSink for SinkFanout { } Err(anyhow::anyhow!("sink flush failed: {}", errors.join("; "))) } + + fn delivery_stats(&self) -> Vec { + self.sinks + .iter() + .flat_map(|sink| sink.delivery_stats()) + .collect() + } } // ── Process-wide registry ─────────────────────────────────────────────────── @@ -107,6 +119,9 @@ impl ObservationSink for SinkFanout { /// reconfigure instrumentation without a restart. static GLOBAL_SINK: RwLock>> = RwLock::new(None); +#[cfg(test)] +pub(crate) static GLOBAL_TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + /// Install `sink` as the process-wide destination, replacing any previous one. pub fn set_global_sink(sink: Arc) { match GLOBAL_SINK.write() { @@ -283,6 +298,9 @@ mod tests { // single test to stay independent of test execution order. #[test] fn global_registry_install_record_and_clear() { + let _guard = GLOBAL_TEST_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); let sink = CollectingSink::new("global"); assert!(!is_enabled()); diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs index ee5cb8049..eeb264cdf 100644 --- a/crates/observability/tests/end_to_end.rs +++ b/crates/observability/tests/end_to_end.rs @@ -88,6 +88,7 @@ async fn run_turn_against(server: &MockServer, profile: ExportProfile) -> Vec = spans + .iter() + .filter_map(|span| Some((span["traceId"].as_str()?, span["spanId"].as_str()?))) + .collect(); + + assert_eq!(spans.len(), 3, "root, generation, and tool expected"); + assert_eq!(ids.len(), spans.len(), "duplicate OTLP wire span ID"); + + for span in &spans { + assert_eq!( + span_attr(span, "langfuse.trace.name"), + Some(&serde_json::json!({ "stringValue": "agent-run" })) + ); + let metadata = span_attr(span, "langfuse.trace.metadata") + .and_then(|value| value["stringValue"].as_str()) + .map(|value| serde_json::from_str::(value).expect("valid metadata")) + .expect("filterable trace metadata"); + assert_eq!(metadata["tenant"], "acme"); + } +} + #[tokio::test] async fn cache_tokens_survive_the_round_trip_unsummed() { let server = collector().await; diff --git a/crates/providers/src/anthropic.rs b/crates/providers/src/anthropic.rs index b171fbb37..a2b58ea6a 100644 --- a/crates/providers/src/anthropic.rs +++ b/crates/providers/src/anthropic.rs @@ -10,8 +10,8 @@ use {async_trait::async_trait, futures::StreamExt, secrecy::ExposeSecret, tokio_ use tracing::{debug, trace, warn}; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, ContentPart, LlmProvider, StreamEvent, - ToolCall, ToolChoice, Usage, UserContent, + AgentToolControls, ChatMessage, CompletionResponse, ContentPart, InputTokenAccounting, + LlmProvider, StreamEvent, ToolCall, ToolChoice, Usage, UserContent, }; pub struct AnthropicProvider { @@ -823,16 +823,17 @@ impl LlmProvider for AnthropicProvider { let tool_calls = parse_tool_calls(&content); - let usage = Usage { - input_tokens: resp["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32, - output_tokens: resp["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32, - cache_read_tokens: resp["usage"]["cache_read_input_tokens"] + let usage = Usage::from_input_tokens( + InputTokenAccounting::Exclusive, + resp["usage"]["input_tokens"].as_u64().unwrap_or(0) as u32, + resp["usage"]["output_tokens"].as_u64().unwrap_or(0) as u32, + resp["usage"]["cache_read_input_tokens"] .as_u64() .unwrap_or(0) as u32, - cache_write_tokens: resp["usage"]["cache_creation_input_tokens"] + resp["usage"]["cache_creation_input_tokens"] .as_u64() .unwrap_or(0) as u32, - }; + ); if usage.cache_read_tokens > 0 || usage.cache_write_tokens > 0 { debug!( @@ -1053,12 +1054,13 @@ impl LlmProvider for AnthropicProvider { } } "message_stop" => { - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Exclusive, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, - }); + )); return; } "error" => { diff --git a/crates/providers/src/async_openai_provider.rs b/crates/providers/src/async_openai_provider.rs index 61f83af5c..33e801d84 100644 --- a/crates/providers/src/async_openai_provider.rs +++ b/crates/providers/src/async_openai_provider.rs @@ -172,10 +172,14 @@ impl LlmProvider for AsyncOpenAiProvider { let usage = response .usage .as_ref() - .map(|u| Usage { - input_tokens: u.prompt_tokens, - output_tokens: u.completion_tokens, - ..Default::default() + .map(|u| { + Usage::from_input_tokens( + moltis_agents::model::InputTokenAccounting::Inclusive, + u.prompt_tokens, + u.completion_tokens, + 0, + 0, + ) }) .unwrap_or_default(); @@ -231,11 +235,13 @@ impl LlmProvider for AsyncOpenAiProvider { } } if let Some(ref u) = response.usage { - yield StreamEvent::Done(Usage { - input_tokens: u.prompt_tokens, - output_tokens: u.completion_tokens, - ..Default::default() - }); + yield StreamEvent::Done(Usage::from_input_tokens( + moltis_agents::model::InputTokenAccounting::Inclusive, + u.prompt_tokens, + u.completion_tokens, + 0, + 0, + )); return; } } diff --git a/crates/providers/src/genai_provider.rs b/crates/providers/src/genai_provider.rs index 439d70e4c..54434e9b0 100644 --- a/crates/providers/src/genai_provider.rs +++ b/crates/providers/src/genai_provider.rs @@ -3,7 +3,8 @@ use std::pin::Pin; use {async_trait::async_trait, futures::StreamExt, tokio_stream::Stream}; use moltis_agents::model::{ - ChatMessage, CompletionResponse, LlmProvider, StreamEvent, Usage, UserContent, + ChatMessage, CompletionResponse, InputTokenAccounting, LlmProvider, StreamEvent, Usage, + UserContent, }; /// Provider backed by the `genai` crate (supports Anthropic, OpenAI, Gemini, @@ -11,13 +12,19 @@ use moltis_agents::model::{ pub struct GenaiProvider { model: String, provider_name: String, + input_token_accounting: InputTokenAccounting, client: genai::Client, } impl GenaiProvider { /// Create a new `GenaiProvider` with an explicit API key passed via /// `AuthResolver`, avoiding the need to set environment variables. - pub fn new(model: String, provider_name: String, api_key: secrecy::Secret) -> Self { + pub fn new( + model: String, + provider_name: String, + input_token_accounting: InputTokenAccounting, + api_key: secrecy::Secret, + ) -> Self { use secrecy::ExposeSecret; // Expose the secret once to hand it to genai's auth resolver. let key = api_key.expose_secret().clone(); @@ -29,12 +36,16 @@ impl GenaiProvider { Self { model, provider_name, + input_token_accounting, client, } } } -fn genai_usage_to_usage(u: &genai::chat::Usage) -> Usage { +fn genai_usage_to_usage( + input_token_accounting: InputTokenAccounting, + u: &genai::chat::Usage, +) -> Usage { let (cache_read, cache_write) = u .prompt_tokens_details .as_ref() @@ -45,12 +56,13 @@ fn genai_usage_to_usage(u: &genai::chat::Usage) -> Usage { ) }) .unwrap_or((0, 0)); - Usage { - input_tokens: u.prompt_tokens.unwrap_or(0) as u32, - output_tokens: u.completion_tokens.unwrap_or(0) as u32, - cache_read_tokens: cache_read, - cache_write_tokens: cache_write, - } + Usage::from_input_tokens( + input_token_accounting, + u.prompt_tokens.unwrap_or(0) as u32, + u.completion_tokens.unwrap_or(0) as u32, + cache_read, + cache_write, + ) } fn build_genai_messages(messages: &[ChatMessage]) -> Vec { @@ -103,7 +115,7 @@ impl LlmProvider for GenaiProvider { .map_err(|e| anyhow::anyhow!("{e}"))?; let text = chat_res.first_text().map(|s| s.to_string()); - let usage = genai_usage_to_usage(&chat_res.usage); + let usage = genai_usage_to_usage(self.input_token_accounting, &chat_res.usage); Ok(CompletionResponse { text, @@ -147,7 +159,7 @@ impl LlmProvider for GenaiProvider { Ok(ChatStreamEvent::End(end)) => { let usage = end.captured_usage .as_ref() - .map(genai_usage_to_usage) + .map(|usage| genai_usage_to_usage(self.input_token_accounting, usage)) .unwrap_or_default(); yield StreamEvent::Done(usage); return; @@ -162,3 +174,30 @@ impl LlmProvider for GenaiProvider { }) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalized_prompt_usage_removes_cache_buckets_from_fresh_input() { + let usage = genai::chat::Usage { + prompt_tokens: Some(100), + prompt_tokens_details: Some(genai::chat::PromptTokensDetails { + cached_tokens: Some(30), + cache_creation_tokens: Some(20), + audio_tokens: None, + }), + completion_tokens: Some(10), + total_tokens: Some(110), + ..Default::default() + }; + + let converted = genai_usage_to_usage(InputTokenAccounting::Inclusive, &usage); + + assert_eq!(converted.input_tokens, 50); + assert_eq!(converted.cache_read_tokens, 30); + assert_eq!(converted.cache_write_tokens, 20); + assert_eq!(converted.output_tokens, 10); + } +} diff --git a/crates/providers/src/openai/provider/completion.rs b/crates/providers/src/openai/provider/completion.rs index c39fc3dcb..eb2e3e86e 100644 --- a/crates/providers/src/openai/provider/completion.rs +++ b/crates/providers/src/openai/provider/completion.rs @@ -15,7 +15,8 @@ use crate::{ }; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, Usage, decode_tool_call_arguments_from_str, + AgentToolControls, ChatMessage, CompletionResponse, InputTokenAccounting, Usage, + decode_tool_call_arguments_from_str, }; use { @@ -550,12 +551,13 @@ impl OpenAiProvider { Ok(CompletionResponse { text, tool_calls, - usage: Usage { + usage: Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, - }, + ), }) } } diff --git a/crates/providers/src/openai_codex.rs b/crates/providers/src/openai_codex.rs index 3f8ecfcc0..7c45e0ef5 100644 --- a/crates/providers/src/openai_codex.rs +++ b/crates/providers/src/openai_codex.rs @@ -12,8 +12,9 @@ use { }; use moltis_agents::model::{ - AgentToolControls, ChatMessage, CompletionResponse, LlmProvider, ReasoningEffort, StreamEvent, - ToolCall, Usage, UserContent, decode_tool_call_arguments_from_str, + AgentToolControls, ChatMessage, CompletionResponse, InputTokenAccounting, LlmProvider, + ReasoningEffort, StreamEvent, ToolCall, Usage, UserContent, + decode_tool_call_arguments_from_str, }; use crate::openai_compat::to_responses_api_tools; @@ -605,12 +606,13 @@ impl LlmProvider for OpenAiCodexProvider { Ok(CompletionResponse { text, tool_calls, - usage: Usage { + usage: Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }, + 0, + ), }) } @@ -755,12 +757,13 @@ impl LlmProvider for OpenAiCodexProvider { for index in tool_calls.keys() { yield StreamEvent::ToolCallComplete { index: *index }; } - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }); + 0, + )); return; } @@ -842,12 +845,13 @@ impl LlmProvider for OpenAiCodexProvider { for index in tool_calls.keys() { yield StreamEvent::ToolCallComplete { index: *index }; } - yield StreamEvent::Done(Usage { + yield StreamEvent::Done(Usage::from_input_tokens( + InputTokenAccounting::Inclusive, input_tokens, output_tokens, cache_read_tokens, - ..Default::default() - }); + 0, + )); return; } "error" | "response.failed" => { diff --git a/crates/providers/src/openai_compat/provider.rs b/crates/providers/src/openai_compat/provider.rs index 662e8a102..0e045d423 100644 --- a/crates/providers/src/openai_compat/provider.rs +++ b/crates/providers/src/openai_compat/provider.rs @@ -7,8 +7,8 @@ use std::collections::{HashMap, HashSet}; use { moltis_agents::model::{ - ChatMessage, CompletionResponse, StreamEvent, ToolCall, Usage, UserContent, - decode_tool_call_arguments_with_diagnostic, extract_tool_call_metadata, + ChatMessage, CompletionResponse, InputTokenAccounting, StreamEvent, ToolCall, Usage, + UserContent, decode_tool_call_arguments_with_diagnostic, extract_tool_call_metadata, }, serde::Serialize, tracing::trace, @@ -510,36 +510,59 @@ fn usage_object_from_payload(payload: &serde_json::Value) -> Option<&serde_json: /// - Cache fields may be top-level or nested in `*_tokens_details`. #[must_use] pub fn parse_openai_compat_usage(usage: &serde_json::Value) -> Usage { - Usage { - input_tokens: usage_field_u32(usage, &[ - &["prompt_tokens"], - &["promptTokens"], - &["input_tokens"], - &["inputTokens"], - ]), - output_tokens: usage_field_u32(usage, &[ + let prompt_tokens = usage_field_u32(usage, &[&["prompt_tokens"], &["promptTokens"]]); + let input_tokens = usage_field_u32(usage, &[&["input_tokens"], &["inputTokens"]]); + let cache_read_tokens = usage_field_u32(usage, &[ + &["prompt_tokens_details", "cached_tokens"], + &["promptTokensDetails", "cachedTokens"], + &["input_tokens_details", "cached_tokens"], + &["inputTokensDetails", "cachedTokens"], + &["cache_read_input_tokens"], + &["cacheReadInputTokens"], + &["input_tokens_details", "cache_read_input_tokens"], + &["inputTokensDetails", "cacheReadInputTokens"], + ]); + let cache_write_tokens = usage_field_u32(usage, &[ + &["cache_creation_input_tokens"], + &["cacheCreationInputTokens"], + &["input_tokens_details", "cache_creation_input_tokens"], + &["inputTokensDetails", "cacheCreationInputTokens"], + ]); + + // `prompt_tokens` and OpenAI's nested `cached_tokens` use inclusive + // accounting. Anthropic-compatible cache counters are siblings of an + // already-exclusive `input_tokens` value. + let has_openai_cache_details = usage + .pointer("/prompt_tokens_details/cached_tokens") + .or_else(|| usage.pointer("/promptTokensDetails/cachedTokens")) + .or_else(|| usage.pointer("/input_tokens_details/cached_tokens")) + .or_else(|| usage.pointer("/inputTokensDetails/cachedTokens")) + .is_some(); + let (accounting, reported_input_tokens) = if prompt_tokens > 0 || has_openai_cache_details { + ( + InputTokenAccounting::Inclusive, + if prompt_tokens > 0 { + prompt_tokens + } else { + input_tokens + }, + ) + } else { + (InputTokenAccounting::Exclusive, input_tokens) + }; + + Usage::from_input_tokens( + accounting, + reported_input_tokens, + usage_field_u32(usage, &[ &["completion_tokens"], &["completionTokens"], &["output_tokens"], &["outputTokens"], ]), - cache_read_tokens: usage_field_u32(usage, &[ - &["prompt_tokens_details", "cached_tokens"], - &["promptTokensDetails", "cachedTokens"], - &["input_tokens_details", "cached_tokens"], - &["inputTokensDetails", "cachedTokens"], - &["cache_read_input_tokens"], - &["cacheReadInputTokens"], - &["input_tokens_details", "cache_read_input_tokens"], - &["inputTokensDetails", "cacheReadInputTokens"], - ]), - cache_write_tokens: usage_field_u32(usage, &[ - &["cache_creation_input_tokens"], - &["cacheCreationInputTokens"], - &["input_tokens_details", "cache_creation_input_tokens"], - &["inputTokensDetails", "cacheCreationInputTokens"], - ]), - } + cache_read_tokens, + cache_write_tokens, + ) } /// Parse usage from an OpenAI-compatible payload, checking common nesting variants. diff --git a/crates/providers/src/openai_compat/tests/mod.rs b/crates/providers/src/openai_compat/tests/mod.rs index 2916581fc..0c703a278 100644 --- a/crates/providers/src/openai_compat/tests/mod.rs +++ b/crates/providers/src/openai_compat/tests/mod.rs @@ -1,11 +1,50 @@ mod schema_normalization; use super::{ - normalize_tool_call_arguments_from_schemas, parse_responses_completion, parse_tool_calls, - sanitize_schema_for_openai_compat, strict_mode::patch_schema_for_strict_mode, to_openai_tools, - to_responses_api_tools, + normalize_tool_call_arguments_from_schemas, parse_openai_compat_usage, + parse_responses_completion, parse_tool_calls, sanitize_schema_for_openai_compat, + strict_mode::patch_schema_for_strict_mode, to_openai_tools, to_responses_api_tools, }; +#[test] +fn openai_usage_removes_cached_tokens_from_flat_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "prompt_tokens": 1_000, + "completion_tokens": 50, + "prompt_tokens_details": {"cached_tokens": 800} + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.output_tokens, 50); +} + +#[test] +fn responses_usage_removes_cached_tokens_from_flat_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "input_tokens": 1_000, + "output_tokens": 50, + "input_tokens_details": {"cached_tokens": 800} + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); +} + +#[test] +fn anthropic_compatible_usage_keeps_exclusive_input_bucket() { + let usage = parse_openai_compat_usage(&serde_json::json!({ + "input_tokens": 200, + "output_tokens": 50, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 20 + })); + + assert_eq!(usage.input_tokens, 200); + assert_eq!(usage.cache_read_tokens, 800); + assert_eq!(usage.cache_write_tokens, 20); +} + /// Recursively assert that every `required` entry has a corresponding key in /// `properties`. Panics with `path` context on the first orphaned entry. fn assert_no_orphaned_required(schema: &serde_json::Value, path: &str) { diff --git a/crates/providers/src/registry/registration.rs b/crates/providers/src/registry/registration.rs index 3d661ecbb..3f7756c8c 100644 --- a/crates/providers/src/registry/registration.rs +++ b/crates/providers/src/registry/registration.rs @@ -457,6 +457,9 @@ impl ProviderRegistry { let provider = Arc::new(genai_provider::GenaiProvider::new( model_id.clone(), genai_provider_name.clone(), + // genai normalizes every adapter to inclusive prompt tokens, + // including Anthropic's otherwise-exclusive API counters. + moltis_agents::model::InputTokenAccounting::Inclusive, resolved_key, )); self.register( diff --git a/crates/swift-bridge/src/state.rs b/crates/swift-bridge/src/state.rs index e8719365e..779c81f68 100644 --- a/crates/swift-bridge/src/state.rs +++ b/crates/swift-bridge/src/state.rs @@ -197,14 +197,24 @@ pub(crate) static HTTPD: Mutex> = Mutex::new(None); pub(crate) fn stop_httpd_handle(handle: HttpdHandle, log_target: &str, stop_message: &str) { emit_log("INFO", log_target, stop_message); - let _ = handle.shutdown_tx.send(()); + let HttpdHandle { + shutdown_tx, + server_task, + state, + .. + } = handle; + let _ = shutdown_tx.send(()); BRIDGE.runtime.block_on(async { - if let Err(error) = handle.server_task.await { + if let Err(error) = server_task.await { emit_log( "WARN", log_target, &format!("httpd task join failed during shutdown: {error}"), ); } + state + .instrumentation + .shutdown(std::time::Duration::from_secs(5)) + .await; }); } diff --git a/crates/web/ui/e2e/specs/instrumentation.spec.js b/crates/web/ui/e2e/specs/instrumentation.spec.js index aac12cc35..ef05b4eef 100644 --- a/crates/web/ui/e2e/specs/instrumentation.spec.js +++ b/crates/web/ui/e2e/specs/instrumentation.spec.js @@ -17,9 +17,7 @@ test.describe("Instrumentation settings", () => { // an operator can see what is available without reading the docs first. await expect(page.getByText("Langfuse", { exact: true })).toBeVisible(); await expect(page.getByText("Datadog", { exact: true })).toBeVisible(); - await expect( - page.getByText("OpenTelemetry (Grafana, Honeycomb, Collector)", { exact: true }), - ).toBeVisible(); + await expect(page.getByText("OpenTelemetry (Grafana, Honeycomb, Collector)", { exact: true })).toBeVisible(); }); test("explains that backends receive different data", async ({ page }) => { @@ -52,6 +50,13 @@ test.describe("Instrumentation settings", () => { await expectRpcOk(page, "instrumentation.status", {}); }); + test("shows honest delivery status before exporters are enabled", async ({ page }) => { + await navigateAndWait(page, "/settings/instrumentation"); + + await expect(page.getByText("Delivery", { exact: true })).toBeVisible(); + await expect(page.getByText("no exporters running", { exact: true })).toBeVisible(); + }); + test("status RPC never returns the secret key", async ({ page }) => { await navigateAndWait(page, "/settings/instrumentation"); const res = await expectRpcOk(page, "instrumentation.status", {}); @@ -59,7 +64,7 @@ test.describe("Instrumentation settings", () => { // The UI only ever needs to know whether a key is configured. Returning // the value would put it in every browser devtools network log. const serialized = JSON.stringify(res); - expect(serialized).not.toContain("secret_key\""); + expect(serialized).not.toContain('secret_key"'); expect(serialized).toContain("secret_key_set"); }); diff --git a/crates/web/ui/src/message-actions.ts b/crates/web/ui/src/message-actions.ts index c88fad04d..f0e248b76 100644 --- a/crates/web/ui/src/message-actions.ts +++ b/crates/web/ui/src/message-actions.ts @@ -143,7 +143,7 @@ export function appendMessageActions(ctx: MessageActionContext): void { // Appended asynchronously: availability depends on whether any backend is // collecting scores, and the action bar must not wait on that check. void feedbackStatus().then((status) => { - if (!status?.enabled || !status.instrumentation_active) return; + if (!(status?.enabled && status.instrumentation_active)) return; if (!bar.isConnected) return; const buttons = buildFeedbackButtons({ sessionKey, runId: ctx.runId }, actionButton); if (!buttons) return; diff --git a/crates/web/ui/src/message-feedback.ts b/crates/web/ui/src/message-feedback.ts index 8c0f2d76a..893e58206 100644 --- a/crates/web/ui/src/message-feedback.ts +++ b/crates/web/ui/src/message-feedback.ts @@ -66,7 +66,7 @@ export function buildFeedbackButtons( }); const payload = result?.payload as { ok?: boolean; outcome?: string } | undefined; - if (!result?.ok || !payload?.ok) { + if (!(result?.ok && payload?.ok)) { // Say why rather than failing silently: "unknown_message" means the // turn aged out of the retention window, which is not the user's // fault and not something a retry fixes. diff --git a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx index a3790b3f6..0c67fd728 100644 --- a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx +++ b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx @@ -11,6 +11,7 @@ import type { VNode } from "preact"; import { useEffect, useState } from "preact/hooks"; import { Badge, Loading, SectionHeading, StatusMessage, SubHeading } from "../../components/forms"; import { sendRpc } from "../../helpers"; +import { connected } from "../../signals"; import { rerender } from "./_shared"; type ContentMode = "full" | "metadata_only" | "none"; @@ -59,9 +60,22 @@ interface InstrumentationStatus { active: boolean; backends: string[]; skipped: SkippedBackend[]; + delivery: DeliveryStats[]; config: InstrumentationConfig; } +interface DeliveryStats { + name: string; + accepted: number; + dropped_queue_full: number; + dropped_failed: number; + delivered: number; + retries: number; + last_success_at: string | null; + last_error: string | null; + last_error_at: string | null; +} + interface TestResult { ok: boolean; error?: string; @@ -93,6 +107,49 @@ function Row({ label, value, hint }: RowProps): VNode { ); } +function deliveryLabel(name: string): string { + if (name === "langfuse") return "Langfuse traces"; + if (name === "langfuse-scores") return "Langfuse scores"; + if (name === "otlp") return "OpenTelemetry"; + if (name === "datadog") return "Datadog"; + return name; +} + +function formatDeliveryTime(value?: string | null): string { + if (!value) return "never"; + const parsed = new Date(value); + return Number.isNaN(parsed.getTime()) ? value : parsed.toLocaleString(); +} + +interface DeliveryRowsProps { + delivery: DeliveryStats[]; +} + +function DeliveryRows({ delivery }: DeliveryRowsProps): VNode { + if (delivery.length === 0) { + return ; + } + + return ( + <> + {delivery.map((stats) => { + const failures = `${stats.dropped_queue_full} queue drops, ${stats.dropped_failed} failed, ${stats.retries} retries`; + const latest = stats.last_error + ? `Last success: ${formatDeliveryTime(stats.last_success_at)}. Last error: ${stats.last_error} (${formatDeliveryTime(stats.last_error_at)}).` + : `Last success: ${formatDeliveryTime(stats.last_success_at)}.`; + return ( + + ); + })} + + ); +} + interface BackendCardProps { title: string; purpose: string; @@ -108,7 +165,7 @@ function BackendCard({ title, purpose, enabled, running, skippedReason, children
{title} {running ? ( - + ) : enabled ? ( ) : ( @@ -151,7 +208,7 @@ function Backends({ config, status, testing, onTestLangfuse }: BackendsProps): V <>

- Langfuse gets the full conversation \u2014 its cost, session and evaluation features are built on it. OTLP and - Datadog get operational shape only: latency, errors, model and token counts, with payload sizes instead of - payloads. Prompt bodies in an APM mean unbounded span size, cardinality pressure, per-byte ingest billing, and - conversation content in a system nobody scoped for it. + Langfuse gets completed observations with the full conversation, token usage and session context. It infers cost + from its current model pricing. OTLP and Datadog get operational shape only: latency, errors, model and token + counts, with payload sizes instead of payloads. Prompt bodies in an APM mean unbounded span size, cardinality + pressure, per-byte ingest billing, and conversation content in a system nobody scoped for it.

); @@ -229,6 +286,9 @@ export function InstrumentationSection(): VNode { const [testing, setTesting] = useState(false); useEffect(() => { + if (!connected.value) return; + setLoading(true); + setError(null); sendRpc("instrumentation.status", {}) .then((res) => { if (res.payload) setStatus(res.payload); @@ -239,7 +299,7 @@ export function InstrumentationSection(): VNode { setLoading(false); rerender(); }); - }, []); + }, [connected.value]); function onTestLangfuse(): void { setTesting(true); @@ -297,6 +357,7 @@ export function InstrumentationSection(): VNode { + diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md index 2dabb9a36..e40c49130 100644 --- a/docs/src/instrumentation.md +++ b/docs/src/instrumentation.md @@ -1,8 +1,8 @@ # Instrumentation -Moltis can export what an agent run actually did — every LLM call, tool -invocation and retrieval, with timings, token usage and errors — to an external -backend. +Moltis can export what a completed agent run did — its LLM calls, tool +invocations and retrievals, with timings, token usage and errors — to an +external backend. Three backends are supported, and they are configured together under one `[instrumentation]` section because they are fed from a single instrumentation @@ -10,7 +10,7 @@ pass in the agent runtime: | Backend | What it is for | | --- | --- | -| **Langfuse** | LLM observability: prompts, completions, cost, sessions, prompt versions, evaluation | +| **Langfuse** | LLM observability: prompts, completions, sessions, token usage, inferred cost, and reaction feedback | | **OTLP** | Any OpenTelemetry backend — Grafana Tempo/Alloy, Honeycomb, an OpenTelemetry Collector | | **Datadog** | Datadog APM, through the Datadog Agent's OTLP intake | @@ -23,9 +23,9 @@ pass in the agent runtime: This is the most important thing to understand about the design. -Langfuse is an LLM-native product. Its cost attribution, session replay, -prompt-version comparison and evaluation features are all built on having the -actual conversation. Sending it prompts and completions is the entire point. +Langfuse is an LLM-native product. Moltis sends completed traces, conversation +content, token usage, session and user attribution, and reaction feedback so +Langfuse can provide LLM observability and infer costs. Datadog and Grafana are infrastructure tools. Sending them prompt bodies is actively harmful: @@ -43,11 +43,10 @@ So Moltis applies a different **export profile** per backend: | Payload sizes (`moltis.input.bytes`) | — | ✅ sent | | Observation type (`AGENT`, `TOOL`, `RETRIEVER`, …) | ✅ `langfuse.observation.type` | ✅ as `gen_ai.operation.name` | | Model, latency, errors | ✅ | ✅ | -| Token counts | ✅ with cache split and cost | ✅ counts only | +| Token counts | ✅ usage details with cache split; Langfuse infers cost | ✅ counts only | | End-user id | ✅ | ❌ off by default (cardinality) | | Session id | ✅ | ✅ | | Tags | ✅ | ✅ OTLP · ❌ Datadog (billing) | -| Managed-prompt version | ✅ | — | You can raise an APM's content level deliberately with `content = "full"`, or lower it to `none`. You cannot accidentally end up @@ -60,8 +59,18 @@ without this feature. Moltis exposes a Prometheus endpoint at `/metrics` with that for dashboards and alerts; use the OTLP exporter here when you want per-run traces to correlate a latency spike with a specific agent run. +Moltis does not currently integrate Langfuse Prompt Management, datasets or +dataset runs, evaluators, or media uploads. User reaction scores are the only +evaluation signal sent directly to Langfuse. + ## Quick start: Langfuse +Prefer the process environment for the secret key: + +```sh +export MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY='sk-lf-...' +``` + ```toml [instrumentation] enabled = true @@ -71,7 +80,6 @@ environment = "production" enabled = true host = "https://cloud.langfuse.com" # or your self-hosted URL public_key = "pk-lf-..." -secret_key = "sk-lf-..." ``` Keys come from your Langfuse project settings. For a self-hosted deployment, @@ -123,19 +131,23 @@ with a reason; the others keep running. ## What gets sent -With Langfuse enabled and default settings, each agent run produces: +With Langfuse enabled and default settings, each completed agent run produces: -- a **trace** named `agent-run`, carrying the user's message and the final - answer; -- a **`GENERATION`** observation per LLM call, with the full message array sent - to the provider, the completion text, model, token usage split into fresh / - cache-read / cache-write buckets, and time-to-first-token; -- a **`TOOL`** (or **`RETRIEVER`**) observation per tool call, with arguments - and results; +- a trace named `agent-run` whose root observation has type **`AGENT`** and + carries the user's message and final answer; +- a **`GENERATION`** observation for each completed LLM call, with the full + message array sent to the provider, completion text, model, token usage split + into fresh / cache-read / cache-write buckets, and time-to-first-token; +- a completed **`TOOL`** or **`RETRIEVER`** observation for each tool call, + with arguments and results; - the session key as the session id, so a Langfuse session matches a Moltis conversation; - the channel sender as the user id, namespaced per channel (`telegram:42`). +Root and child observations are immutable and exported exactly once, after +completion. Moltis does not send in-progress observation updates. Cancelled or +timed-out work is closed as an error rather than exported as successful work. + You can narrow this without turning the backend off: ```toml @@ -221,7 +233,7 @@ and shown in the settings UI — if you see them, raise `queue_capacity` or lowe | `enabled` | `false` | | | `host` | `https://cloud.langfuse.com` | Set to your self-hosted URL to keep data on-premises. | | `public_key` | `""` | | -| `secret_key` | unset | Held as a secret; never logged or shown in config dumps. | +| `secret_key` | unset | Prefer `MOLTIS_INSTRUMENTATION__LANGFUSE__SECRET_KEY`; a config value is also accepted and is never logged. | | `capture_input` | `true` | | | `capture_output` | `true` | | | `capture_tool_io` | `true` | | @@ -260,9 +272,9 @@ and shown in the settings UI — if you see them, raise `queue_capacity` or lowe ## User feedback -A thumbs up or down on a reply becomes a `user-feedback` score on the trace -that produced it — `1.0` for positive, `0.0` for negative — visible in -Langfuse alongside the conversation. +A thumbs up or down on a reply becomes a `user-feedback` score with Langfuse +data type `BOOLEAN`, attached to the trace that produced the reply. The Scores +API represents the value as `1.0` for positive and `0.0` for negative. Feedback works in three places: @@ -285,9 +297,11 @@ positive = ["👍", "🎉", "ship-it"] Setting one list leaves the other on its defaults. **Changing your mind works.** Score ids are derived from the trace and the -reacting user, and Langfuse upserts on that id, so switching from 👍 to 👎 -replaces your vote instead of recording both. Removing the reaction retracts -the score. Two people reacting to the same reply still count separately. +reacting user. Switching from 👍 to 👎 submits the same id, so Langfuse replaces +the vote instead of recording both. Removing the reaction deletes the score +through the dedicated Scores API. Creates, replacements and deletions share one +ordered delivery queue, preventing an older queued create from recreating a +score after deletion. Two reacting users still count separately. ### Why a reaction sometimes does nothing @@ -302,39 +316,30 @@ the score. Two people reacting to the same reply still count separately. ## Cost -Cost is computed in-process from a built-in price table, so it is available -whether or not any backend is configured, and every backend sees the same -number instead of each deriving its own. - -Cache reads and writes are priced on their own tiers rather than folded into -input — a cache read can be an order of magnitude cheaper, and summing them -would overstate spend on exactly the workloads caching exists to help. - -Model ids are matched by longest prefix after stripping any provider prefix, -so `anthropic/claude-opus-4` and a dated snapshot like -`claude-opus-4-5-20260101` both price as their family. +Moltis exports the model id and token usage details, including separate +cache-read and cache-write buckets, but it does not export costs from its static +local price table. Langfuse infers cost using its own versioned model definitions +and pricing tiers, avoiding stale Moltis prices overriding Langfuse's +calculations. -**A model with no entry reports no cost at all** rather than falling back to -an average. Prices go stale whenever a provider changes them, and a confident -wrong number that looks authoritative is worse than a visible gap. +Unknown, private or custom models may show no cost until their pricing is +configured in Langfuse. ## How it works -All three backends are fed over **OTLP/HTTP with a JSON payload**. +Trace observations are sent to Langfuse using OTLP/HTTP JSON at +`/api/public/otel/v1/traces`. Every OTLP request includes +`x-langfuse-ingestion-version: 4`, selecting Langfuse's v4 ingestion contract. +The exporter sends only completed, immutable root and child observations. -For Langfuse this is deliberate rather than incidental. OTLP is Langfuse's own -modern ingest path — the one its v3+ Python and v4+ JavaScript SDKs use — and -the only one that carries the current observation taxonomy. Langfuse's native -`/api/public/ingestion` API still exists, but its `observation-create` event is -marked deprecated upstream and the supported `span-create` / `generation-create` -pair cannot express `AGENT`, `TOOL`, `RETRIEVER` and the other newer types at -all. The ingestion API is still used for **scores**, which OTLP has no way to -represent. +OTLP cannot represent feedback scores. Moltis therefore creates, replaces and +deletes `BOOLEAN` feedback through Langfuse's dedicated Scores API at +`/api/public/scores`, using a separate ordered sink. Spans carry two attribute vocabularies depending on the profile: -- `langfuse.*` — Langfuse's own mapping, for the observation taxonomy, usage - and cost detail, and prompt linkage. +- `langfuse.*` — Langfuse's mapping for trace context, observation taxonomy, + content, and detailed token usage. - `gen_ai.*` — the OpenTelemetry GenAI semantic conventions, understood by Grafana, Datadog, Honeycomb and other OTel consumers. From 9365caaad2ec05d124b75c7fb5d19871695b234f Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 12:54:28 -0700 Subject: [PATCH 25/33] test(web): stabilize instrumentation validation Update the feedback selector and settings navigation expectation for the current UI, and wait for the initial session fetch before mutating the date-bucket test store. --- crates/web/ui/e2e/specs/feedback.spec.js | 2 +- crates/web/ui/e2e/specs/sessions.spec.js | 1 + crates/web/ui/e2e/specs/settings-nav.spec.js | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/web/ui/e2e/specs/feedback.spec.js b/crates/web/ui/e2e/specs/feedback.spec.js index 04c37616b..a8af51913 100644 --- a/crates/web/ui/e2e/specs/feedback.spec.js +++ b/crates/web/ui/e2e/specs/feedback.spec.js @@ -66,7 +66,7 @@ test.describe("Reaction feedback", () => { // The action bar appends thumbs asynchronously; a failed or disabled // status check must not surface as a page error. - await expect(page.locator("#chat-input")).toBeVisible(); + await expect(page.locator("#chatInput")).toBeVisible(); expect(pageErrors).toEqual([]); }); }); diff --git a/crates/web/ui/e2e/specs/sessions.spec.js b/crates/web/ui/e2e/specs/sessions.spec.js index 0da4ce7b6..95739edfe 100644 --- a/crates/web/ui/e2e/specs/sessions.spec.js +++ b/crates/web/ui/e2e/specs/sessions.spec.js @@ -82,6 +82,7 @@ test.describe("Session management", () => { await page.clock.install({ time: new Date(2026, 6, 23, 23, 58) }); const pageErrors = await navigateAndWait(page, "/"); await waitForWsConnected(page); + await expect(page.locator('#sessionList .session-item[data-session-key="main"]')).toBeVisible(); await page.clock.pauseAt(new Date(2026, 6, 23, 23, 59, 59, 500)); const expected = await page.evaluate(() => { diff --git a/crates/web/ui/e2e/specs/settings-nav.spec.js b/crates/web/ui/e2e/specs/settings-nav.spec.js index 5dcd534f0..71dc646c9 100644 --- a/crates/web/ui/e2e/specs/settings-nav.spec.js +++ b/crates/web/ui/e2e/specs/settings-nav.spec.js @@ -670,6 +670,7 @@ test.describe("Settings navigation", () => { "Terminal", "Monitoring", "Logs", + "Instrumentation", ...presentOptionalItems(["GraphQL"]), "Configuration", ]; From 5aa5b05941f9fe00bf476d38e3e3092aefa915fe Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 14:21:03 -0700 Subject: [PATCH 26/33] fix(scripts): classify targeted Rust test paths The targeted validator treated any Rust file below a tests directory as a Cargo integration-test target. In-source test modules such as config/src/validate/tests/instrumentation.rs therefore produced nonexistent --test targets. Match only direct crates//tests/.rs paths so nested source modules use filename filters instead. --- scripts/local-validate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index 103d924b4..f4e3a2927 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -274,7 +274,7 @@ build_targeted_rust_test_cmd() { local base_cmd base_cmd="$(nextest_base_cmd_for_package "$package")" - if [[ "$file" == */tests/*.rs ]]; then + if [[ "$file" =~ ^crates/[^/]+/tests/[^/]+\.rs$ ]]; then local test_name test_name="$(basename "$file" .rs)" commands+=("$base_cmd --test $test_name") From ab251236abc3a136c5c196fd7a35076528846c84 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 16:53:07 -0700 Subject: [PATCH 27/33] fix(observability): harden telemetry edge cases Prevent operational exporters from leaking channel session identifiers and omit inline image bytes from generation payloads. Restore logical tool failure propagation, make provider failover deterministic, sanitize exporter configuration and errors, and reserve shutdown time for queued telemetry. Keep feedback retention and reaction authorization consistent across runtime changes, add focused regression coverage, and correct nested Rust test selection in local validation. --- crates/agents/src/provider_chain.rs | 112 +++++++++- crates/agents/src/runner/instrumentation.rs | 65 +++++- crates/agents/src/runner/non_streaming.rs | 10 +- crates/agents/src/runner/streaming.rs | 10 +- crates/agents/src/runner/tests/basic.rs | 57 +++++ crates/agents/src/runner/tool_result.rs | 43 +++- crates/channels/src/feedback.rs | 202 +++++++++++++++--- crates/config/src/schema/instrumentation.rs | 30 ++- crates/config/src/validate/semantic.rs | 4 + .../src/validate/tests/instrumentation.rs | 4 + crates/discord/src/handler/reactions.rs | 58 ++++- .../gateway/src/methods/services/feedback.rs | 15 +- crates/gateway/src/server/instrumentation.rs | 88 +++++++- .../src/server/prepare_core/feedback.rs | 12 +- crates/observability/src/builder.rs | 93 +++++++- .../src/exporters/langfuse/scores.rs | 17 +- .../src/exporters/otlp/mapping.rs | 7 +- .../observability/src/exporters/otlp/mod.rs | 59 +++-- crates/observability/src/profile.rs | 10 +- crates/observability/src/recorder.rs | 30 +-- crates/observability/src/sink.rs | 13 ++ crates/observability/tests/end_to_end.rs | 15 +- crates/slack/src/socket.rs | 64 +++--- crates/telegram/src/handlers/reactions.rs | 72 +++++-- crates/web/src/assets/css/components.css | 4 + .../assets/icons/masks/instrumentation.svg | 1 + crates/web/ui/e2e/specs/settings-nav.spec.js | 6 +- crates/web/ui/input.css | 4 + .../ui/src/components/forms/SectionLayout.tsx | 7 +- crates/web/ui/src/pages/SettingsPage.tsx | 6 +- scripts/local-validate.sh | 4 + 31 files changed, 947 insertions(+), 175 deletions(-) create mode 100644 crates/web/src/assets/icons/masks/instrumentation.svg diff --git a/crates/agents/src/provider_chain.rs b/crates/agents/src/provider_chain.rs index f90916210..c35b59d73 100644 --- a/crates/agents/src/provider_chain.rs +++ b/crates/agents/src/provider_chain.rs @@ -228,9 +228,10 @@ impl ProviderChain { ) -> Pin + Send + '_>> { Box::pin(async_stream::stream! { let mut errors = Vec::new(); + let force_primary = self.chain.iter().all(|entry| entry.state.is_tripped()); - for entry in &self.chain { - if entry.state.is_tripped() { + for (index, entry) in self.chain.iter().enumerate() { + if entry.state.is_tripped() && !(force_primary && index == 0) { continue; } @@ -292,8 +293,17 @@ impl ProviderChain { continue; } - entry.state.record_success(); - return; + let error = "provider stream ended without a terminal event".to_string(); + entry.state.record_failure(); + yield TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { + identity: identity.clone(), + error: error.clone(), + }); + if emitted_output { + yield TrackedStreamEvent::Event(StreamEvent::Error(error)); + return; + } + errors.push(format!("{}: {error}", entry.provider.id())); } yield TrackedStreamEvent::Event(StreamEvent::Error(format!( @@ -350,11 +360,12 @@ impl LlmProvider for ProviderChain { on_attempt: &mut (dyn FnMut(ProviderAttemptEvent) + Send), ) -> anyhow::Result { let mut errors = Vec::new(); + let force_primary = self.chain.iter().all(|entry| entry.state.is_tripped()); #[cfg(feature = "metrics")] let start = Instant::now(); - for entry in &self.chain { - if entry.state.is_tripped() { + for (index, entry) in self.chain.iter().enumerate() { + if entry.state.is_tripped() && !(force_primary && index == 0) { continue; } @@ -590,6 +601,34 @@ mod tests { } } + struct EmptyStreamProvider; + + #[async_trait] + impl LlmProvider for EmptyStreamProvider { + fn name(&self) -> &str { + "empty" + } + + fn id(&self) -> &str { + "empty" + } + + async fn complete( + &self, + _messages: &[ChatMessage], + _tools: &[serde_json::Value], + ) -> anyhow::Result { + anyhow::bail!("not used") + } + + fn stream( + &self, + _messages: Vec, + ) -> Pin + Send + '_>> { + Box::pin(tokio_stream::empty()) + } + } + #[tokio::test] async fn primary_succeeds_no_failover() { let chain = ProviderChain::new(vec![ @@ -828,6 +867,67 @@ mod tests { ); } + #[tokio::test] + async fn empty_stream_fails_over_instead_of_reporting_success() { + let chain = ProviderChain::new(vec![ + Arc::new(EmptyStreamProvider), + Arc::new(SuccessProvider { id: "fallback" }), + ]); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(events.iter().any(|event| matches!( + event, + TrackedStreamEvent::Attempt(ProviderAttemptEvent::Failed { identity, error }) + if identity == &ProviderIdentity::new("empty", "empty") + && error.contains("without a terminal event") + ))); + assert!(matches!( + events.last(), + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn all_tripped_streaming_chain_still_probes_primary() { + let chain = ProviderChain::single(Arc::new(SuccessProvider { id: "primary" })); + for _ in 0..3 { + chain.chain[0].state.record_failure(); + } + assert!(chain.chain[0].state.is_tripped()); + + let events = chain + .stream_with_tools_and_options_tracked(vec![], vec![], AgentToolControls::default()) + .collect::>() + .await; + + assert!(matches!( + events.first(), + Some(TrackedStreamEvent::Attempt(ProviderAttemptEvent::Started(identity))) + if identity == &ProviderIdentity::new("success", "primary") + )); + assert!(matches!( + events.last(), + Some(TrackedStreamEvent::Event(StreamEvent::Done(_))) + )); + } + + #[tokio::test] + async fn all_tripped_non_streaming_chain_still_probes_primary() { + let chain = ProviderChain::single(Arc::new(SuccessProvider { id: "primary" })); + for _ in 0..3 { + chain.chain[0].state.record_failure(); + } + assert!(chain.chain[0].state.is_tripped()); + + let response = chain.complete(&[], &[]).await.unwrap(); + + assert_eq!(response.text.as_deref(), Some("ok")); + } + #[test] fn classify_rate_limit() { let err = anyhow::anyhow!("429 Too Many Requests: rate limit exceeded"); diff --git a/crates/agents/src/runner/instrumentation.rs b/crates/agents/src/runner/instrumentation.rs index 2eb6d130d..aaf1ab008 100644 --- a/crates/agents/src/runner/instrumentation.rs +++ b/crates/agents/src/runner/instrumentation.rs @@ -12,8 +12,11 @@ use { }; use crate::{ - model::{AgentToolControls, ChatMessage, LlmProvider, ProviderIdentity, Usage, UserContent}, - runner::{AgentRunError, AgentRunResult, tool_result::tool_result_failure}, + model::{ + AgentToolControls, ChatMessage, ContentPart, LlmProvider, ProviderIdentity, Usage, + UserContent, + }, + runner::{AgentRunError, AgentRunResult, tool_result::persisted_tool_result_failure}, }; /// Derive the trace scope from the session and channel context. @@ -199,12 +202,43 @@ pub fn begin_generation_step( step.set_metadata("iteration", serde_json::json!(iteration)); step.set_metadata("tool_count", serde_json::json!(tool_count)); step.set_input(serde_json::Value::Array( - messages.iter().map(ChatMessage::to_openai_value).collect(), + messages.iter().map(message_to_telemetry_value).collect(), )); step }) } +/// Serialize a provider message without copying inline image payloads into telemetry. +fn message_to_telemetry_value(message: &ChatMessage) -> serde_json::Value { + let ChatMessage::User { + content: UserContent::Multimodal(parts), + name, + } = message + else { + return message.to_openai_value(); + }; + + let content = parts + .iter() + .map(|part| match part { + ContentPart::Text(text) => serde_json::json!({ "type": "text", "text": text }), + ContentPart::Image { media_type, .. } => serde_json::json!({ + "type": "image", + "media_type": media_type, + "data": "[omitted]", + }), + }) + .collect::>(); + let mut value = serde_json::json!({ "role": "user", "content": content }); + if let Some(sanitized) = name + .as_ref() + .and_then(|name| ChatMessage::sanitize_message_name(name)) + { + value["name"] = serde_json::Value::String(sanitized); + } + value +} + /// Extract a provider's terminal reason from a raw streaming event. #[must_use] pub fn finish_reason(raw: &serde_json::Value) -> Option { @@ -262,7 +296,7 @@ pub fn finish_tool_step( step.set_output(serde_json::Value::String(persisted_output.to_string())); let failure = error .map(str::to_string) - .or_else(|| tool_result_failure(persisted_result)); + .or_else(|| persisted_tool_result_failure(persisted_result)); if !success || failure.is_some() { step.fail(failure.unwrap_or_else(|| "tool execution failed".to_string())); } @@ -451,6 +485,29 @@ mod tests { assert_eq!(json["parts"], 0); } + #[test] + fn generation_messages_omit_inline_image_data() { + let sentinel = "sensitive-base64-payload"; + let message = ChatMessage::user_multimodal_named( + vec![ + ContentPart::Text("describe this".into()), + ContentPart::Image { + media_type: "image/png".into(), + data: sentinel.into(), + }, + ], + "Channel User", + ); + + let value = message_to_telemetry_value(&message); + let encoded = value.to_string(); + assert!(!encoded.contains(sentinel)); + assert!(!encoded.contains("data:image/png;base64")); + assert_eq!(value["content"][1]["media_type"], "image/png"); + assert_eq!(value["content"][1]["data"], "[omitted]"); + assert_eq!(value["name"], "Channel_User"); + } + #[test] fn text_input_is_carried_verbatim() { let json = user_content_to_json(&UserContent::Text("hello".into())); diff --git a/crates/agents/src/runner/non_streaming.rs b/crates/agents/src/runner/non_streaming.rs index f21b1bcef..66972138e 100644 --- a/crates/agents/src/runner/non_streaming.rs +++ b/crates/agents/src/runner/non_streaming.rs @@ -37,7 +37,7 @@ use super::{ resolve_agent_max_iterations, }, sanitize_tool_name, - tool_result::sanitize_tool_result, + tool_result::{sanitize_tool_result, tool_result_failure}, }; use crate::tool_loop_detector::ToolLoopDetector; @@ -784,12 +784,14 @@ pub async fn run_agent_loop_with_context_and_limits( if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { + let error = tool_result_failure(&val); + let success = error.is_none(); // Dispatch AfterToolCall hook. if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: true, + success, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -799,9 +801,9 @@ pub async fn run_agent_loop_with_context_and_limits( } ( - true, + success, serde_json::json!({ "result": val }), - None, + error, false, tool_step, ) diff --git a/crates/agents/src/runner/streaming.rs b/crates/agents/src/runner/streaming.rs index 7aae1afa6..2a539dea6 100644 --- a/crates/agents/src/runner/streaming.rs +++ b/crates/agents/src/runner/streaming.rs @@ -43,7 +43,7 @@ use super::{ resolve_agent_max_iterations, }, sanitize_tool_name, streaming_tool_call_message_content, - tool_result::sanitize_tool_result, + tool_result::{sanitize_tool_result, tool_result_failure}, }; use crate::tool_loop_detector::ToolLoopDetector; @@ -958,11 +958,13 @@ pub async fn run_agent_loop_streaming_with_limits( if let Some(tool) = tool { match tool.execute(args).await { Ok(val) => { + let error = tool_result_failure(&val); + let success = error.is_none(); if let Some(ref hooks) = hook_registry { let payload = HookPayload::AfterToolCall { session_key: session_key.clone(), tool_name: tc_name.clone(), - success: true, + success, result: Some(val.clone()), channel: channel_for_hooks.clone(), }; @@ -972,9 +974,9 @@ pub async fn run_agent_loop_streaming_with_limits( } ( - true, + success, serde_json::json!({ "result": val }), - None, + error, false, tool_step, ) diff --git a/crates/agents/src/runner/tests/basic.rs b/crates/agents/src/runner/tests/basic.rs index 32f13a792..0e59d2390 100644 --- a/crates/agents/src/runner/tests/basic.rs +++ b/crates/agents/src/runner/tests/basic.rs @@ -135,6 +135,63 @@ async fn test_simple_text_response() { assert_eq!(result.tool_calls_made, 0); } +struct NestedErrorDataTool; + +#[async_trait] +impl crate::tool_registry::AgentTool for NestedErrorDataTool { + fn name(&self) -> &str { + "echo_tool" + } + + fn description(&self) -> &str { + "Returns successful nested data" + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({"type": "object"}) + } + + async fn execute(&self, _params: serde_json::Value) -> Result { + Ok(serde_json::json!({ + "success": true, + "result": {"error": "page value"} + })) + } +} + +#[tokio::test] +async fn test_nested_error_data_does_not_fail_a_successful_tool_call() { + let provider = Arc::new(ToolCallingProvider { + call_count: std::sync::atomic::AtomicUsize::new(0), + }); + let mut tools = ToolRegistry::new(); + tools.register(Box::new(NestedErrorDataTool)); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let captured = Arc::clone(&events); + let on_event: OnEvent = Box::new(move |event| { + captured.lock().unwrap().push(event); + }); + + run_agent_loop( + provider, + &tools, + "You are a test bot.", + &UserContent::text("Hi"), + Some(&on_event), + None, + ) + .await + .unwrap(); + + assert!( + events + .lock() + .unwrap() + .iter() + .any(|event| { matches!(event, RunnerEvent::ToolCallEnd { success: true, .. }) }) + ); +} + #[tokio::test] async fn test_non_streaming_runner_uses_max_iteration_override() { let provider = Arc::new(ToolCallingProvider { diff --git a/crates/agents/src/runner/tool_result.rs b/crates/agents/src/runner/tool_result.rs index d92df1aaa..6500034cb 100644 --- a/crates/agents/src/runner/tool_result.rs +++ b/crates/agents/src/runner/tool_result.rs @@ -106,8 +106,7 @@ pub fn sanitize_tool_result(input: &str, max_bytes: usize) -> String { /// Return a stable failure message for tool results that encode logical errors. #[must_use] pub fn tool_result_failure(result: &serde_json::Value) -> Option { - let payload = result.get("result").unwrap_or(result); - let error = payload.get("error").filter(|value| !value.is_null()); + let error = result.get("error").filter(|value| !value.is_null()); if let Some(error) = error { return Some( error @@ -115,13 +114,19 @@ pub fn tool_result_failure(result: &serde_json::Value) -> Option { .map_or_else(|| error.to_string(), str::to_string), ); } - (payload.get("success") == Some(&serde_json::Value::Bool(false))) + (result.get("success") == Some(&serde_json::Value::Bool(false))) .then(|| "tool returned success: false".to_string()) } +/// Inspect the runner's `{ "result": }` persistence wrapper. +#[must_use] +pub fn persisted_tool_result_failure(result: &serde_json::Value) -> Option { + tool_result_failure(result).or_else(|| result.get("result").and_then(tool_result_failure)) +} + #[cfg(test)] mod failure_tests { - use super::tool_result_failure; + use super::{persisted_tool_result_failure, tool_result_failure}; #[test] fn success_false_without_error_has_a_failure_message() { @@ -132,9 +137,35 @@ mod failure_tests { } #[test] - fn wrapped_tool_errors_are_detected() { + fn nested_result_data_is_not_treated_as_a_raw_tool_failure() { + assert_eq!( + tool_result_failure(&serde_json::json!({ + "success": true, + "result": {"error": "page value"} + })), + None + ); + } + + #[test] + fn top_level_failure_wins_even_when_result_is_present() { + assert_eq!( + tool_result_failure(&serde_json::json!({ + "error": "denied", + "result": {"value": 42} + })) + .as_deref(), + Some("denied") + ); + } + + #[test] + fn persisted_tool_errors_are_detected_inside_the_runner_wrapper() { assert_eq!( - tool_result_failure(&serde_json::json!({"result": {"error": "denied"}})).as_deref(), + persisted_tool_result_failure(&serde_json::json!({ + "result": {"error": "denied"} + })) + .as_deref(), Some("denied") ); } diff --git a/crates/channels/src/feedback.rs b/crates/channels/src/feedback.rs index b374c185f..e67f41ad8 100644 --- a/crates/channels/src/feedback.rs +++ b/crates/channels/src/feedback.rs @@ -39,6 +39,7 @@ struct Active { vocabulary: FeedbackVocabulary, enabled: bool, environment: Option, + retention_days: u32, } /// Records reply/trace links and converts reactions into scores. @@ -68,6 +69,7 @@ impl FeedbackService { vocabulary: FeedbackVocabulary::from_config(&settings.positive, &settings.negative), enabled: settings.enabled, environment, + retention_days: settings.link_retention_days, }); match self.active.write() { Ok(mut guard) => *guard = next, @@ -146,25 +148,55 @@ impl FeedbackService { ) -> FeedbackOutcome { // Classify before the database lookup: most reactions in a busy chat // are not feedback, and they should not cost a query each. - let Some((links, signal, environment)) = self.read(|active| { - ( - Arc::clone(&active.links), - active - .enabled - .then(|| active.vocabulary.classify(emoji)) - .flatten(), - active.environment.clone(), - ) - }) else { + let Some((enabled, signal)) = + self.read(|active| (active.enabled, active.vocabulary.classify(emoji))) + else { return FeedbackOutcome::Disabled; }; - if !self.enabled() { + if !enabled { return FeedbackOutcome::Disabled; } let Some(signal) = signal else { return FeedbackOutcome::NotFeedback; }; - if langfuse.is_none() { + + self.submit_signal( + channel, + account_id, + chat_id, + message_id, + added.then_some(signal), + user_id, + Some(format!("{channel} reaction {emoji}")), + langfuse, + ) + .await + } + + /// Submit already-classified feedback from a typed UI boundary. + #[allow(clippy::too_many_arguments)] + pub async fn submit_signal( + &self, + channel: &str, + account_id: &str, + chat_id: &str, + message_id: &str, + signal: Option, + user_id: &str, + comment: Option, + langfuse: Option<&Arc>, + ) -> FeedbackOutcome { + let Some((links, enabled, environment, retention_days)) = self.read(|active| { + ( + Arc::clone(&active.links), + active.enabled, + active.environment.clone(), + active.retention_days, + ) + }) else { + return FeedbackOutcome::Disabled; + }; + if !enabled || langfuse.is_none() { return FeedbackOutcome::Disabled; } @@ -182,19 +214,16 @@ impl FeedbackService { return FeedbackOutcome::UnknownMessage; }, }; + if link.created_at < retention_cutoff(retention_days) { + return FeedbackOutcome::UnknownMessage; + } let trace_id = TraceId(link.trace_id); // Namespaced so the same numeric user id on two channels is two people. let scoped_user = format!("{channel}:{user_id}"); - if added { - let score = feedback_score( - &trace_id, - signal, - Some(&scoped_user), - Some(format!("{channel} reaction {emoji}")), - environment, - ); + if let Some(signal) = signal { + let score = feedback_score(&trace_id, signal, Some(&scoped_user), comment, environment); moltis_observability::record(moltis_observability::Event::Score(Box::new(score))); debug!(channel, ?signal, "recorded reaction feedback"); return FeedbackOutcome::Recorded(signal); @@ -207,12 +236,14 @@ impl FeedbackService { FeedbackOutcome::Retracted } - /// Drop links older than `retention_days`. - pub async fn prune(&self, retention_days: u32) -> u64 { - let cutoff = now_unix() - i64::from(retention_days) * 86_400; - let Some(links) = self.read(|active| Arc::clone(&active.links)) else { + /// Drop links older than the active retention window. + pub async fn prune(&self) -> u64 { + let Some((links, retention_days)) = + self.read(|active| (Arc::clone(&active.links), active.retention_days)) + else { return 0; }; + let cutoff = retention_cutoff(retention_days); match links.prune(cutoff).await { Ok(removed) => removed, Err(error) => { @@ -227,6 +258,13 @@ fn now_unix() -> i64 { time::OffsetDateTime::now_utc().unix_timestamp() } +fn retention_cutoff(retention_days: u32) -> i64 { + time::OffsetDateTime::now_utc() + .checked_sub(time::Duration::days(i64::from(retention_days))) + .unwrap_or(time::OffsetDateTime::UNIX_EPOCH) + .unix_timestamp() +} + #[allow(clippy::unwrap_used, clippy::expect_used)] #[cfg(test)] mod tests { @@ -494,6 +532,82 @@ mod tests { moltis_observability::clear_global_sink(); } + #[tokio::test] + async fn typed_feedback_does_not_depend_on_the_emoji_vocabulary() { + let _guard = sink_guard().await; + let sink = Arc::new(CollectingSink::default()); + moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); + let links = Arc::new(MemoryLinks::default()); + let service = FeedbackService::default(); + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + positive: vec!["party".into()], + negative: vec!["boo".into()], + ..FeedbackSettings::default() + }, + Some("production".into()), + ); + link_reply(&links, "42", "trace-1").await; + let langfuse = langfuse_client(); + + let outcome = service + .submit_signal( + "telegram", + "bot-1", + "chat-1", + "42", + Some(FeedbackSignal::Positive), + "99", + Some("typed feedback".into()), + Some(&langfuse), + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::Recorded(FeedbackSignal::Positive)); + assert!(matches!( + sink.events + .lock() + .unwrap_or_else(|e| e.into_inner()) + .as_slice(), + [moltis_observability::Event::Score(_)] + )); + moltis_observability::clear_global_sink(); + } + + #[tokio::test] + async fn expired_links_are_rejected_even_before_pruning() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "old".into(), + trace_id: "trace-old".into(), + session_key: None, + created_at: retention_cutoff(30) - 1, + }) + .await + .unwrap(); + let langfuse = langfuse_client(); + + let outcome = service + .on_reaction( + "telegram", + "bot-1", + "chat-1", + "old", + "\u{1f44d}", + "99", + true, + Some(&langfuse), + ) + .await; + + assert_eq!(outcome, FeedbackOutcome::UnknownMessage); + } + #[tokio::test] async fn score_feedback_is_disabled_without_langfuse() { let (service, links) = service(true); @@ -591,7 +705,7 @@ mod tests { None, ) .await; - assert_eq!(service.prune(30).await, 0); + assert_eq!(service.prune().await, 0); } #[tokio::test] @@ -636,13 +750,13 @@ mod tests { message_id: "old".into(), trace_id: "trace-old".into(), session_key: None, - created_at: now_unix() - 60 * 86_400, + created_at: now_unix() - time::Duration::days(60).whole_seconds(), }) .await .unwrap(); link_reply(&links, "new", "trace-new").await; - let removed = service.prune(30).await; + let removed = service.prune().await; assert_eq!(removed, 1); assert!( @@ -653,4 +767,38 @@ mod tests { .is_some() ); } + + #[tokio::test] + async fn pruning_uses_reconfigured_retention_window() { + let (service, links) = service(true); + links + .link(TraceLink { + channel_type: "telegram".into(), + account_id: "bot-1".into(), + chat_id: "chat-1".into(), + message_id: "middle-aged".into(), + trace_id: "trace-middle-aged".into(), + session_key: None, + created_at: now_unix() - time::Duration::days(60).whole_seconds(), + }) + .await + .unwrap(); + service.apply( + Arc::clone(&links) as Arc, + &FeedbackSettings { + link_retention_days: 90, + ..FeedbackSettings::default() + }, + None, + ); + + assert_eq!(service.prune().await, 0); + assert!( + links + .lookup("telegram", "bot-1", "chat-1", "middle-aged") + .await + .unwrap() + .is_some() + ); + } } diff --git a/crates/config/src/schema/instrumentation.rs b/crates/config/src/schema/instrumentation.rs index 5aace9a23..ed39b7869 100644 --- a/crates/config/src/schema/instrumentation.rs +++ b/crates/config/src/schema/instrumentation.rs @@ -247,7 +247,7 @@ impl Default for LangfuseSettings { /// `metadata_only`: these are operational tools, and prompt bodies there mean /// unbounded span size, cardinality pressure, per-byte ingest billing, and /// conversation content sitting in a system nobody scoped for it. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Clone, Serialize, Deserialize)] #[serde(default)] pub struct OtlpSettings { /// Whether to export over OTLP. @@ -271,6 +271,19 @@ pub struct OtlpSettings { pub timeout_secs: u64, } +impl std::fmt::Debug for OtlpSettings { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OtlpSettings") + .field("enabled", &self.enabled) + .field("endpoint", &self.endpoint) + .field("header_names", &self.headers.keys().collect::>()) + .field("content", &self.content) + .field("emit_user_id", &self.emit_user_id) + .field("timeout_secs", &self.timeout_secs) + .finish() + } +} + impl Default for OtlpSettings { fn default() -> Self { Self { @@ -428,6 +441,21 @@ mod tests { ); } + #[test] + fn debug_output_does_not_leak_otlp_header_values() { + let settings = OtlpSettings { + headers: std::collections::BTreeMap::from([( + "Authorization".into(), + "Bearer super-secret".into(), + )]), + ..Default::default() + }; + let rendered = format!("{settings:?}"); + + assert!(rendered.contains("Authorization")); + assert!(!rendered.contains("super-secret")); + } + #[test] fn datadog_defaults_to_the_local_agent_endpoint() { // The overwhelmingly common deployment: Agent on localhost, no API key diff --git a/crates/config/src/validate/semantic.rs b/crates/config/src/validate/semantic.rs index 5097be71c..4551f9e7e 100644 --- a/crates/config/src/validate/semantic.rs +++ b/crates/config/src/validate/semantic.rs @@ -1034,6 +1034,10 @@ fn validate_instrumentation( config.datadog.timeout_secs == 0, "instrumentation.datadog.timeout_secs", ), + ( + config.feedback.link_retention_days == 0, + "instrumentation.feedback.link_retention_days", + ), ]; diagnostics.extend( zero_values diff --git a/crates/config/src/validate/tests/instrumentation.rs b/crates/config/src/validate/tests/instrumentation.rs index 9f2ca9537..07c1eb33b 100644 --- a/crates/config/src/validate/tests/instrumentation.rs +++ b/crates/config/src/validate/tests/instrumentation.rs @@ -58,6 +58,9 @@ timeout_secs = 0 [instrumentation.datadog] timeout_secs = 0 + +[instrumentation.feedback] +link_retention_days = 0 "#, Severity::Error, ); @@ -67,6 +70,7 @@ timeout_secs = 0 BTreeSet::from([ "instrumentation.datadog.timeout_secs".to_string(), "instrumentation.flush_interval_ms".to_string(), + "instrumentation.feedback.link_retention_days".to_string(), "instrumentation.langfuse.timeout_secs".to_string(), "instrumentation.max_batch_bytes".to_string(), "instrumentation.otlp.timeout_secs".to_string(), diff --git a/crates/discord/src/handler/reactions.rs b/crates/discord/src/handler/reactions.rs index 5743d041f..6302add04 100644 --- a/crates/discord/src/handler/reactions.rs +++ b/crates/discord/src/handler/reactions.rs @@ -4,9 +4,12 @@ use serenity::all::{Context, Reaction, ReactionType}; -use moltis_channels::{ChannelEvent, ChannelType}; +use { + moltis_channels::{ChannelEvent, ChannelType}, + moltis_common::types::ChatType, +}; -use super::implementation::Handler; +use {super::implementation::Handler, crate::access}; impl Handler { /// Surface a reaction change so the gateway can score it as feedback. @@ -27,10 +30,10 @@ impl Handler { return; } - let sink = { + let (config, sink) = { let accts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); match accts.get(&self.account_id) { - Some(state) => state.event_sink.clone(), + Some(state) => (state.config.clone(), state.event_sink.clone()), None => return, } }; @@ -38,6 +41,53 @@ impl Handler { return; }; + let chat_type = if reaction.guild_id.is_some() { + ChatType::Group + } else { + ChatType::Dm + }; + let guild_id = reaction.guild_id.map(|id| id.to_string()); + let username = if added { + reaction.user(&ctx.http).await.ok().map(|user| user.name) + } else { + None + }; + let basic_access = !added + || access::check_access( + &config, + &chat_type, + &user_id.to_string(), + username.as_deref(), + guild_id.as_deref(), + // A reaction is intentional interaction and does not need a fresh + // mention, while the account and guild allowlists still apply. + true, + ) + .is_ok(); + let filter_active = + !config.channel_name_patterns.is_empty() || !config.category_allowlist.is_empty(); + let channel_allowed = if !added || !filter_active { + true + } else { + reaction + .guild_id + .and_then(|guild_id| { + ctx.cache.guild(guild_id).and_then(|guild| { + guild.channels.get(&reaction.channel_id).map(|channel| { + access::channel_matches_filter( + &config, + Some(&channel.name), + channel.parent_id.map(|id| id.to_string()).as_deref(), + ) + }) + }) + }) + .unwrap_or(false) + }; + if !basic_access || !channel_allowed { + return; + } + // Custom guild emoji have no unicode form; their name is what a // feedback vocabulary can match against. let emoji = match &reaction.emoji { diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs index cf56bbda6..1a382031e 100644 --- a/crates/gateway/src/methods/services/feedback.rs +++ b/crates/gateway/src/methods/services/feedback.rs @@ -10,6 +10,7 @@ use { moltis_channels::FeedbackOutcome, + moltis_observability::FeedbackSignal, moltis_protocol::{ErrorShape, error_codes}, }; @@ -66,11 +67,11 @@ pub(super) fn register(reg: &mut MethodRegistry) { // `signal` rather than a raw score so the wire format cannot // smuggle an arbitrary value into the metric. let signal = ctx.params.get("signal").and_then(|v| v.as_str()); - let (emoji, added) = match signal { - Some("positive") => ("\u{1f44d}", true), - Some("negative") => ("\u{1f44e}", true), + let signal = match signal { + Some("positive") => Some(FeedbackSignal::Positive), + Some("negative") => Some(FeedbackSignal::Negative), // Clicking an active thumb clears it. - Some("clear") => ("\u{1f44d}", false), + Some("clear") => None, other => { return Err(ErrorShape::new( error_codes::INVALID_REQUEST, @@ -103,14 +104,14 @@ pub(super) fn register(reg: &mut MethodRegistry) { let outcome = ctx .state .feedback - .on_reaction( + .submit_signal( moltis_channels::trace_link::WEB_CHANNEL, moltis_channels::trace_link::WEB_CHANNEL, session_key, message_id, - emoji, + signal, user_id, - added, + Some("web feedback".to_string()), ctx.state.instrumentation.langfuse().as_ref(), ) .await; diff --git a/crates/gateway/src/server/instrumentation.rs b/crates/gateway/src/server/instrumentation.rs index ead3303cb..fcc6f4ca9 100644 --- a/crates/gateway/src/server/instrumentation.rs +++ b/crates/gateway/src/server/instrumentation.rs @@ -140,9 +140,10 @@ impl InstrumentationState { pub async fn shutdown(&self, timeout: Duration) { moltis_observability::clear_global_sink(); let started = tokio::time::Instant::now(); - if !moltis_observability::wait_for_active_turns(timeout).await { + let flush_reserve = timeout.min(Duration::from_millis(100)); + let wait_budget = timeout.saturating_sub(flush_reserve); + if !moltis_observability::wait_for_active_turns(wait_budget).await { warn!("instrumentation shutdown timed out waiting for active turns"); - return; } self.flush(timeout.saturating_sub(started.elapsed())).await; } @@ -173,8 +174,14 @@ impl InstrumentationSnapshot { #[allow(clippy::expect_used)] mod tests { use { - async_trait::async_trait, moltis_config::LangfuseSettings, secrecy::Secret, - std::sync::Mutex, tokio::sync::oneshot, + async_trait::async_trait, + moltis_config::LangfuseSettings, + secrecy::Secret, + std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, + }, + tokio::sync::oneshot, }; use super::*; @@ -197,6 +204,11 @@ mod tests { flushed: Mutex>>, } + struct TimeoutAwareFlushSink { + delivered: Mutex>>, + queued: AtomicUsize, + } + #[async_trait] impl ObservationSink for FlushTrackingSink { fn name(&self) -> &str { @@ -218,6 +230,34 @@ mod tests { } } + #[async_trait] + impl ObservationSink for TimeoutAwareFlushSink { + fn name(&self) -> &str { + "timeout-aware" + } + + fn record(&self, _event: moltis_observability::Event) { + self.queued.fetch_add(1, Ordering::Relaxed); + } + + async fn flush(&self, timeout: Duration) -> anyhow::Result<()> { + anyhow::ensure!( + timeout >= Duration::from_millis(25), + "insufficient flush budget" + ); + let queued = self.queued.swap(0, Ordering::Relaxed); + if let Some(delivered) = self + .delivered + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = delivered.send(queued); + } + Ok(()) + } + } + #[tokio::test] #[serial_test::serial(instrumentation_global_sink)] async fn disabled_config_reports_inactive_and_installs_no_sink() { @@ -391,4 +431,44 @@ mod tests { .await .expect("sink should flush after the turn closes"); } + + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn shutdown_flushes_queued_data_when_active_turns_time_out() { + let (delivered_tx, delivered_rx) = oneshot::channel(); + let sink = Arc::new(TimeoutAwareFlushSink { + delivered: Mutex::new(Some(delivered_tx)), + queued: AtomicUsize::new(0), + }); + let installed_sink: Arc = sink; + let state = InstrumentationState::default(); + moltis_observability::set_global_sink(Arc::clone(&installed_sink)); + state.replace(InstrumentationSnapshot { + sink: Some(installed_sink), + backends: vec!["tracking".into()], + ..Default::default() + }); + let completed = moltis_observability::TurnRecorder::begin( + "completed", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + completed.finish(); + let recorder = moltis_observability::TurnRecorder::begin( + "active", + moltis_observability::TraceScope::default(), + moltis_observability::RecorderSettings::default(), + ) + .expect("global sink installed"); + + state.shutdown(Duration::from_millis(100)).await; + + let delivered = tokio::time::timeout(Duration::from_secs(1), delivered_rx) + .await + .expect("shutdown should retain enough time to flush") + .expect("queued data should flush despite the active turn"); + assert!(delivered > 0, "the completed turn should reach the sink"); + drop(recorder); + } } diff --git a/crates/gateway/src/server/prepare_core/feedback.rs b/crates/gateway/src/server/prepare_core/feedback.rs index 2c15e3790..aa4803e85 100644 --- a/crates/gateway/src/server/prepare_core/feedback.rs +++ b/crates/gateway/src/server/prepare_core/feedback.rs @@ -26,11 +26,15 @@ pub(super) fn install_feedback(state: &Arc, db_pool: &SqlitePool) // Links accumulate one row per delivered reply; drop the ones too old // to attribute a reaction to. let feedback = Arc::clone(&state.feedback); - let retention_days = state.config.instrumentation.feedback.link_retention_days; tokio::spawn(async move { - let removed = feedback.prune(retention_days).await; - if removed > 0 { - tracing::debug!(removed, "pruned expired trace links"); + let prune_interval = std::time::Duration::try_from(time::Duration::days(1)) + .unwrap_or(std::time::Duration::MAX); + loop { + let removed = feedback.prune().await; + if removed > 0 { + tracing::debug!(removed, "pruned expired trace links"); + } + tokio::time::sleep(prune_interval).await; } }); } diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs index e773cd175..4a332bc30 100644 --- a/crates/observability/src/builder.rs +++ b/crates/observability/src/builder.rs @@ -74,6 +74,18 @@ fn batch_config(config: &InstrumentationConfig) -> BatchConfig { } } +fn validate_batch_config(config: &InstrumentationConfig) -> Result<(), String> { + let invalid = [ + (config.queue_capacity == 0, "queue_capacity"), + (config.flush_interval_ms == 0, "flush_interval_ms"), + (config.max_batch_bytes == 0, "max_batch_bytes"), + ]; + if let Some((_, field)) = invalid.into_iter().find(|(invalid, _)| *invalid) { + return Err(format!("{field} must be at least 1")); + } + Ok(()) +} + /// Validate a backend endpoint. /// /// Plaintext HTTP is refused for non-loopback hosts: these payloads carry @@ -82,8 +94,11 @@ fn batch_config(config: &InstrumentationConfig) -> BatchConfig { /// typing a URL. Loopback is allowed for a local Agent or collector. fn validate_endpoint(raw: &str) -> Result<(), String> { let Ok(url) = reqwest::Url::parse(raw) else { - return Err(format!("`{raw}` is not a valid URL")); + return Err("endpoint is not a valid URL".to_string()); }; + if !url.username().is_empty() || url.password().is_some() { + return Err("endpoint must not contain credentials; use headers instead".to_string()); + } match url.scheme() { "https" => Ok(()), "http" => { @@ -125,6 +140,9 @@ fn build_langfuse( if secret.expose_secret().trim().is_empty() { return Err("secret_key is empty".to_string()); } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } validate_endpoint(&settings.host)?; let langfuse_config = LangfuseConfig { @@ -152,6 +170,9 @@ fn build_otlp( if settings.endpoint.trim().is_empty() { return Err("endpoint is not set".to_string()); } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } validate_endpoint(&settings.endpoint)?; let profile = ExportProfile { @@ -182,6 +203,12 @@ fn build_datadog( if settings.endpoint.trim().is_empty() { return Err("endpoint is not set".to_string()); } + if settings.timeout_secs == 0 { + return Err("timeout_secs must be at least 1".to_string()); + } + if settings.service.trim().is_empty() { + return Err("service is not set".to_string()); + } validate_endpoint(&settings.endpoint)?; let mut headers = std::collections::BTreeMap::new(); @@ -225,6 +252,25 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { }; } + if let Err(reason) = validate_batch_config(config) { + let skipped = [ + ("langfuse", config.langfuse.enabled), + ("otlp", config.otlp.enabled), + ("datadog", config.datadog.enabled), + ] + .into_iter() + .filter(|(_, enabled)| *enabled) + .map(|(name, _)| SkippedBackend { + name: name.to_string(), + reason: reason.clone(), + }) + .collect(); + return BuildOutcome { + built: None, + skipped, + }; + } + let mut sinks: Vec> = Vec::new(); let mut backends = Vec::new(); let mut langfuse_client = None; @@ -516,6 +562,51 @@ mod tests { assert!(outcome.skipped[0].reason.contains("not a valid URL")); } + #[tokio::test] + async fn endpoint_credentials_are_rejected_without_echoing_them() { + let config = InstrumentationConfig { + otlp: OtlpSettings { + enabled: true, + endpoint: "https://operator:super-secret@collector.example/v1/traces".into(), + ..Default::default() + }, + ..enabled_config() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert!( + outcome.skipped[0] + .reason + .contains("must not contain credentials") + ); + assert!(!outcome.skipped[0].reason.contains("super-secret")); + } + + #[tokio::test] + async fn zero_batch_settings_skip_backends_without_panicking() { + for (queue_capacity, flush_interval_ms, max_batch_bytes, field) in [ + (0, 1, 1, "queue_capacity"), + (1, 0, 1, "flush_interval_ms"), + (1, 1, 0, "max_batch_bytes"), + ] { + let config = InstrumentationConfig { + queue_capacity, + flush_interval_ms, + max_batch_bytes, + langfuse: valid_langfuse(), + ..enabled_config() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert!(outcome.skipped[0].reason.contains(field)); + } + } + #[tokio::test] async fn sample_rate_is_clamped_into_range() { let config = InstrumentationConfig { diff --git a/crates/observability/src/exporters/langfuse/scores.rs b/crates/observability/src/exporters/langfuse/scores.rs index d2e126e55..35d0748d1 100644 --- a/crates/observability/src/exporters/langfuse/scores.rs +++ b/crates/observability/src/exporters/langfuse/scores.rs @@ -116,12 +116,7 @@ impl LangfuseClient { if status.is_success() { continue; } - let response_body = response.text().await.unwrap_or_default(); - let message = format!( - "Langfuse rejected score {}: HTTP {status}: {}", - score.id, - response_body.chars().take(512).collect::() - ); + let message = format!("Langfuse rejected score {} with HTTP {status}", score.id); if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status == reqwest::StatusCode::REQUEST_TIMEOUT || status.is_server_error() @@ -150,10 +145,8 @@ impl LangfuseClient { return Ok(()); } - let body = response.text().await.unwrap_or_default(); Err(anyhow::anyhow!( - "Langfuse rejected the score deletion: HTTP {status}: {}", - body.chars().take(512).collect::() + "Langfuse rejected the score deletion with HTTP {status}" )) } @@ -170,11 +163,9 @@ impl LangfuseClient { if status.is_success() || status == reqwest::StatusCode::NOT_FOUND { return Ok(()); } - let body = response.text().await.unwrap_or_default(); let message = format!( - "Langfuse rejected score deletion {}: HTTP {status}: {}", - deletion.score_id, - body.chars().take(512).collect::() + "Langfuse rejected score deletion {} with HTTP {status}", + deletion.score_id ); if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status == reqwest::StatusCode::REQUEST_TIMEOUT diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs index e9eb5345c..5141e5946 100644 --- a/crates/observability/src/exporters/otlp/mapping.rs +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -712,13 +712,14 @@ mod tests { } #[test] - fn generic_otel_omits_user_id_but_keeps_session() { + fn generic_otel_omits_user_and_session_ids() { let span = observation_to_span(&generation(), &otel_ctx()); // Per-user cardinality is a billing and compliance problem in an APM. assert!(attr_value(&span, attr::USER_ID).is_none()); - // Session grouping stays: it is bounded and operationally useful. - assert!(attr_value(&span, attr::GEN_AI_CONVERSATION_ID).is_some()); + // Channel session keys contain account and peer identifiers and are + // therefore no safer than the explicit user id. + assert!(attr_value(&span, attr::GEN_AI_CONVERSATION_ID).is_none()); } #[test] diff --git a/crates/observability/src/exporters/otlp/mod.rs b/crates/observability/src/exporters/otlp/mod.rs index 0b5184270..30d58c566 100644 --- a/crates/observability/src/exporters/otlp/mod.rs +++ b/crates/observability/src/exporters/otlp/mod.rs @@ -27,7 +27,7 @@ use crate::{ }; /// Configuration for an OTLP exporter. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct OtlpConfig { /// Display name for logs and status reporting. pub name: String, @@ -48,6 +48,21 @@ pub struct OtlpConfig { pub profile: ExportProfile, } +impl std::fmt::Debug for OtlpConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OtlpConfig") + .field("name", &self.name) + .field("endpoint", &self.endpoint) + .field("header_names", &self.headers.keys().collect::>()) + .field("timeout", &self.timeout) + .field("service_name", &self.service_name) + .field("service_version", &self.service_version) + .field("environment", &self.environment) + .field("profile", &self.profile) + .finish() + } +} + impl Default for OtlpConfig { fn default() -> Self { Self { @@ -123,14 +138,15 @@ impl OtlpTransport { /// /// 4xx other than 408/429 are fatal: retrying a rejected payload or bad /// credentials cannot succeed and only burns rate-limit budget. - fn classify(status: StatusCode, body: String) -> TransportError { + fn classify(status: StatusCode) -> TransportError { + let message = format!("collector rejected OTLP batch with HTTP {status}"); if status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::REQUEST_TIMEOUT || status.is_server_error() { - TransportError::Retryable(format!("HTTP {status}: {body}")) + TransportError::Retryable(message) } else { - TransportError::Fatal(format!("HTTP {status}: {body}")) + TransportError::Fatal(message) } } } @@ -168,10 +184,7 @@ impl Transport for OtlpTransport { .map_err(|e| { // Connection-level failures are always worth retrying: the // collector may simply be restarting. - TransportError::Retryable(format!( - "request to {} failed: {e}", - self.config.endpoint - )) + TransportError::Retryable(format!("collector request failed: {e}")) })?; let status = response.status(); @@ -184,10 +197,7 @@ impl Transport for OtlpTransport { return Ok(()); } - let body = response.text().await.unwrap_or_default(); - // Truncate: some collectors echo the whole rejected payload back. - let body = body.chars().take(512).collect::(); - Err(Self::classify(status, body)) + Err(Self::classify(status)) } fn accepts(&self, event: &Event) -> bool { @@ -327,7 +337,10 @@ mod tests { async fn bad_request_is_fatal() { let server = MockServer::start().await; Mock::given(method("POST")) - .respond_with(ResponseTemplate::new(400)) + .respond_with( + ResponseTemplate::new(400) + .set_body_string("echoed prompt and Authorization: Basic super-secret"), + ) .mount(&server) .await; @@ -337,7 +350,12 @@ mod tests { .await .expect_err("400 should fail"); - assert!(matches!(error, TransportError::Fatal(_)), "{error:?}"); + let TransportError::Fatal(message) = error else { + panic!("expected fatal error"); + }; + assert!(message.contains("HTTP 400")); + assert!(!message.contains("echoed prompt")); + assert!(!message.contains("super-secret")); } #[tokio::test] @@ -388,4 +406,17 @@ mod tests { assert!(value.is_sensitive()); assert!(!format!("{:?}", transport.headers).contains("Basic secret")); } + + #[test] + fn config_debug_omits_header_values() { + let mut config = config("https://collector.example/v1/traces".into()); + config + .headers + .insert("Authorization".into(), "Bearer super-secret".into()); + + let rendered = format!("{config:?}"); + + assert!(rendered.contains("Authorization")); + assert!(!rendered.contains("super-secret")); + } } diff --git a/crates/observability/src/profile.rs b/crates/observability/src/profile.rs index ad3959cce..47a47e4cb 100644 --- a/crates/observability/src/profile.rs +++ b/crates/observability/src/profile.rs @@ -101,7 +101,7 @@ impl ExportProfile { content: ContentCapture::MetadataOnly, vocabulary: Vocabulary::GenAi, emit_user_id: false, - emit_session_id: true, + emit_session_id: false, emit_tags: true, emit_usage: true, max_attribute_bytes: 4_096, @@ -117,7 +117,7 @@ impl ExportProfile { content: ContentCapture::MetadataOnly, vocabulary: Vocabulary::GenAi, emit_user_id: false, - emit_session_id: true, + emit_session_id: false, emit_tags: false, emit_usage: true, max_attribute_bytes: 4_096, @@ -174,6 +174,10 @@ mod tests { !profile.emit_user_id, "user id is high-cardinality in an APM" ); + assert!( + !profile.emit_session_id, + "channel session keys contain account and peer identifiers" + ); // Structural metadata is still useful for latency and error analysis. assert!(profile.emits_content_metadata()); } @@ -183,7 +187,7 @@ mod tests { let profile = ExportProfile::datadog(); assert!(!profile.emits_bodies()); assert!(!profile.emit_tags, "Datadog bills on tag cardinality"); - assert!(profile.emit_session_id); + assert!(!profile.emit_session_id); } #[test] diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index 936872cdf..4ca590cb5 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -105,6 +105,8 @@ pub async fn wait_for_active_turns(timeout: std::time::Duration) -> bool { let deadline = tokio::time::Instant::now() + timeout; loop { let changed = ACTIVE_TURNS_CHANGED.notified(); + tokio::pin!(changed); + changed.as_mut().enable(); if ACTIVE_TURNS.load(Ordering::Acquire) == 0 { return true; } @@ -123,13 +125,18 @@ impl TurnRecorder { scope: TraceScope, settings: RecorderSettings, ) -> Option { - Self::begin_inner( - sink::global_sink()?, + if !settings.sampled() { + return None; + } + let (sink, active_turn) = + sink::with_global_sink(|sink| (Arc::clone(sink), ActiveTurnGuard::new()))?; + Some(Self::begin_inner( + sink, name, scope, settings, - Some(ActiveTurnGuard::new()), - ) + Some(active_turn), + )) } /// Begin recording a turn against an explicit sink. @@ -145,7 +152,9 @@ impl TurnRecorder { scope: TraceScope, settings: RecorderSettings, ) -> Option { - Self::begin_inner(sink, name, scope, settings, None) + settings + .sampled() + .then(|| Self::begin_inner(sink, name, scope, settings, None)) } fn begin_inner( @@ -154,11 +163,7 @@ impl TurnRecorder { scope: TraceScope, settings: RecorderSettings, active_turn: Option, - ) -> Option { - if !settings.sampled() { - return None; - } - + ) -> Self { let mut trace = TraceRecord::new(name); trace.scope = scope.clone(); let trace_id = trace.id.clone(); @@ -167,7 +172,7 @@ impl TurnRecorder { // orphan steps onto it without extra bookkeeping. let root_id = ObservationId(trace_id.0.clone()); - let recorder = Self { + Self { sink, settings, trace_id: trace_id.clone(), @@ -175,8 +180,7 @@ impl TurnRecorder { root_id, trace: Arc::new(Mutex::new(trace)), _active_turn: active_turn, - }; - Some(recorder) + } } /// The trace being recorded, for correlating scores later. diff --git a/crates/observability/src/sink.rs b/crates/observability/src/sink.rs index edaaa93de..0876b2abb 100644 --- a/crates/observability/src/sink.rs +++ b/crates/observability/src/sink.rs @@ -147,6 +147,19 @@ pub fn global_sink() -> Option> { } } +/// Run a short operation while holding the global sink registry read lock. +/// +/// Used when cloning the sink must be atomic with related bookkeeping, such as +/// registering an active turn before shutdown can clear the registry. +pub(crate) fn with_global_sink( + operation: impl FnOnce(&Arc) -> T, +) -> Option { + match GLOBAL_SINK.read() { + Ok(guard) => guard.as_ref().map(operation), + Err(poisoned) => poisoned.into_inner().as_ref().map(operation), + } +} + /// Whether instrumentation is active. /// /// Call this before doing any work to build an event: when no sink is diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs index eeb264cdf..3db45a345 100644 --- a/crates/observability/tests/end_to_end.rs +++ b/crates/observability/tests/end_to_end.rs @@ -27,7 +27,7 @@ const SECRET_ANSWER: &str = "migration applied to prod-db-01"; fn scope() -> TraceScope { TraceScope { - session_id: Some("agent:main:main".into()), + session_id: Some("agent:main:channel:telegram:account:private-bot:peer:user:99".into()), user_id: Some("telegram:99".into()), tags: vec!["channel:telegram".into()], environment: Some("production".into()), @@ -204,6 +204,14 @@ async fn generic_otlp_profile_never_puts_conversation_on_the_wire() { ); // Per-user cardinality is a billing and compliance problem in an APM. assert!(!raw.contains("telegram:99"), "user id reached an APM"); + assert!( + !raw.contains("private-bot"), + "channel account id reached an APM" + ); + assert!( + !raw.contains("gen_ai.conversation.id"), + "channel session key reached an APM" + ); } #[tokio::test] @@ -220,10 +228,7 @@ async fn generic_otlp_profile_still_delivers_operational_signal() { "token counts missing" ); assert!(raw.contains("moltis.input.bytes"), "payload sizes missing"); - assert!( - raw.contains("gen_ai.conversation.id"), - "session grouping missing" - ); + assert!(!raw.contains("gen_ai.conversation.id")); } #[tokio::test] diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index ba8b53567..5e174b8a5 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -652,9 +652,8 @@ pub(crate) async fn handle_reaction_event( /// Core reaction handling shared by Socket Mode and the Events API. /// -/// Always emits a [`ChannelEvent::ReactionChange`] for observers; additionally -/// routes the reaction into the agent as a synthetic message when -/// `reaction_triggers` is enabled and the reaction is eligible. +/// Emits authorized additions and user-scoped removals to observers; +/// additionally routes eligible additions into the agent as synthetic messages. pub(crate) async fn dispatch_reaction( account_id: &str, user_id: &str, @@ -676,25 +675,6 @@ pub(crate) async fn dispatch_reaction( } }; - let Some(sink) = event_sink else { - return; - }; - - // Always surface the raw reaction change to observers (web UI, hooks). - sink.emit(ChannelEvent::ReactionChange { - channel_type: ChannelType::Slack, - account_id: account_id.to_string(), - chat_id: channel_id.clone(), - message_id: message_ts.clone(), - user_id: user_id.to_string(), - emoji: emoji.to_string(), - added, - }) - .await; - - // Optionally route the reaction into the agent as a message. The bot's own - // acknowledgment reactions (👀/✅/❌) are always ignored to avoid loops. - // // Fail closed: if the bot user id is unknown we cannot distinguish the bot's // own reactions, so treat the reactor as "self" and skip. Triggering here // would let the bot's own ACK reactions fire agent turns and loop. @@ -708,16 +688,6 @@ pub(crate) async fn dispatch_reaction( true }, }; - if !reaction_should_trigger( - config.reaction_triggers, - added, - is_self, - emoji, - &config.reaction_trigger_emojis, - ) { - return; - } - let is_dm = channel_id.starts_with('D'); let access_granted = check_access( is_dm, @@ -728,14 +698,40 @@ pub(crate) async fn dispatch_reaction( &config.allowlist, &config.channel_allowlist, ); - if !access_granted { + if is_self || (added && !access_granted) { debug!( account_id, - user_id, "slack reaction trigger denied by access control" + user_id, "slack reaction denied by access control" ); return; } + let Some(sink) = event_sink else { + return; + }; + + sink.emit(ChannelEvent::ReactionChange { + channel_type: ChannelType::Slack, + account_id: account_id.to_string(), + chat_id: channel_id.clone(), + message_id: message_ts.clone(), + user_id: user_id.to_string(), + emoji: emoji.to_string(), + added, + }) + .await; + + // Optionally route the reaction into the agent as a message. + if !reaction_should_trigger( + config.reaction_triggers, + added, + is_self, + emoji, + &config.reaction_trigger_emojis, + ) { + return; + } + // Thread the synthetic message under the reacted message so the agent sees // the original content as thread context (dispatch fetches thread history). let reply_to = ChannelReplyTarget { diff --git a/crates/telegram/src/handlers/reactions.rs b/crates/telegram/src/handlers/reactions.rs index 3e9acf1c4..617e7b519 100644 --- a/crates/telegram/src/handlers/reactions.rs +++ b/crates/telegram/src/handlers/reactions.rs @@ -8,11 +8,12 @@ use std::sync::Arc; use { moltis_channels::{ChannelEvent, ChannelType}, + moltis_common::types::ChatType, teloxide::types::{MessageReactionUpdated, ReactionType}, tracing::debug, }; -use crate::state::AccountStateMap; +use crate::{access, state::AccountStateMap}; /// Emoji for a reaction, or `None` for kinds a vocabulary cannot match. fn reaction_token(reaction: &ReactionType) -> Option { @@ -42,6 +43,15 @@ fn diff(before: &[ReactionType], after: &[ReactionType]) -> (Vec, Vec Vec<(String, bool)> { + let (added, removed) = diff(before, after); + removed + .into_iter() + .map(|emoji| (emoji, false)) + .chain(added.into_iter().map(|emoji| (emoji, true))) + .collect() +} + /// Handle a `message_reaction` update. pub async fn handle_message_reaction( update: MessageReactionUpdated, @@ -54,10 +64,10 @@ pub async fn handle_message_reaction( return; }; - let event_sink = { + let (config, event_sink) = { let accts = accounts.read().unwrap_or_else(|e| e.into_inner()); match accts.get(account_id) { - Some(state) => state.event_sink.clone(), + Some(state) => (state.config.clone(), state.event_sink.clone()), None => return, } }; @@ -65,15 +75,37 @@ pub async fn handle_message_reaction( return; }; - let (added, removed) = diff(&update.old_reaction, &update.new_reaction); - if added.is_empty() && removed.is_empty() { + let (chat_type, group_id) = match update.chat.kind { + teloxide::types::ChatKind::Private(_) => (ChatType::Dm, None), + teloxide::types::ChatKind::Public(ref public) => { + let group_id = Some(update.chat.id.0.to_string()); + match public.kind { + teloxide::types::PublicChatKind::Channel(_) => (ChatType::Channel, group_id), + _ => (ChatType::Group, group_id), + } + }, + }; + let peer_id = user.id.0.to_string(); + let access_granted = access::check_access( + &config, + &chat_type, + &peer_id, + user.username.as_deref(), + group_id.as_deref(), + // Reactions do not need a fresh mention, but all other DM/group policy + // and allowlist checks still apply. + true, + ) + .is_ok(); + + let changes = ordered_changes(&update.old_reaction, &update.new_reaction); + if changes.is_empty() { return; } debug!( account_id, - added = added.len(), - removed = removed.len(), + changes = changes.len(), "telegram reaction update" ); @@ -82,7 +114,7 @@ pub async fn handle_message_reaction( let account_id = account_id.to_string(); let chat_id = update.chat.id.0.to_string(); let message_id = update.message_id.0.to_string(); - let user_id = user.id.0.to_string(); + let user_id = peer_id.clone(); async move { sink.emit(ChannelEvent::ReactionChange { channel_type: ChannelType::Telegram, @@ -97,11 +129,14 @@ pub async fn handle_message_reaction( } }; - for emoji in added { - emit(emoji, true).await; - } - for emoji in removed { - emit(emoji, false).await; + for (emoji, added) in changes { + // A removal can only retract this user's own score, so preserve source + // truth even if an allowlist changed after the score was recorded. + if added && !access_granted { + debug!(account_id, user_id = %peer_id, "telegram reaction denied by access control"); + continue; + } + emit(emoji, added).await; } } @@ -141,6 +176,17 @@ mod tests { assert_eq!(removed, vec!["\u{1f44d}".to_string()]); } + #[test] + fn switching_reactions_retracts_before_setting_the_replacement() { + assert_eq!( + ordered_changes(&[emoji("\u{1f44d}")], &[emoji("\u{1f44e}")]), + vec![ + ("\u{1f44d}".to_string(), false), + ("\u{1f44e}".to_string(), true) + ] + ); + } + #[test] fn an_unchanged_reaction_set_produces_nothing() { let (added, removed) = diff(&[emoji("\u{1f44d}")], &[emoji("\u{1f44d}")]); diff --git a/crates/web/src/assets/css/components.css b/crates/web/src/assets/css/components.css index 0d95e1443..4dc63ca76 100644 --- a/crates/web/src/assets/css/components.css +++ b/crates/web/src/assets/css/components.css @@ -1243,6 +1243,10 @@ -webkit-mask-image: url("../icons/masks/mask-b9bf0315417c03e4.svg"); mask-image: url("../icons/masks/mask-b9bf0315417c03e4.svg"); } +.settings-nav-item[data-section="instrumentation"]::before { + -webkit-mask-image: url("../icons/masks/instrumentation.svg"); + mask-image: url("../icons/masks/instrumentation.svg"); +} .settings-nav-item[data-section="logs"]::before { -webkit-mask-image: url("../icons/masks/mask-5883df2e24ebc841.svg"); mask-image: url("../icons/masks/mask-5883df2e24ebc841.svg"); diff --git a/crates/web/src/assets/icons/masks/instrumentation.svg b/crates/web/src/assets/icons/masks/instrumentation.svg new file mode 100644 index 000000000..7c9aa8eb7 --- /dev/null +++ b/crates/web/src/assets/icons/masks/instrumentation.svg @@ -0,0 +1 @@ + diff --git a/crates/web/ui/e2e/specs/settings-nav.spec.js b/crates/web/ui/e2e/specs/settings-nav.spec.js index 71dc646c9..f9d163b33 100644 --- a/crates/web/ui/e2e/specs/settings-nav.spec.js +++ b/crates/web/ui/e2e/specs/settings-nav.spec.js @@ -24,9 +24,7 @@ test.describe("Settings navigation", () => { await expect(page.getByRole("heading", { name: "User Profile", exact: true })).toBeVisible(); }); - test("settings nav keeps distinct icons for nodes, remote access, network audit, and openclaw import", async ({ - page, - }) => { + test("settings nav keeps distinct icons for systems and integrations", async ({ page }) => { const pageErrors = watchPageErrors(page); await navigateAndWait(page, "/settings/profile"); await expect(page.locator(".settings-sidebar-nav")).toBeVisible(); @@ -58,6 +56,7 @@ test.describe("Settings navigation", () => { networkAudit: readRuleMask('.settings-nav-item[data-section="network-audit"]::before'), mcp: readRuleMask('.settings-nav-item[data-section="mcp"]::before'), openclawImport: readRuleMask('.settings-nav-item[data-section="import"]::before'), + instrumentation: readRuleMask('.settings-nav-item[data-section="instrumentation"]::before'), }; }); @@ -73,6 +72,7 @@ test.describe("Settings navigation", () => { expect(hasMask(masks.tools)).toBeTruthy(); expect(hasMask(masks.remoteAccess)).toBeTruthy(); expect(hasMask(masks.networkAudit)).toBeTruthy(); + expect(hasMask(masks.instrumentation)).toBeTruthy(); expect(hasMask(masks.mcp)).toBeTruthy(); expect(masks.remoteAccess).not.toBe(masks.networkAudit); diff --git a/crates/web/ui/input.css b/crates/web/ui/input.css index 84bafd8a8..bea646d8e 100644 --- a/crates/web/ui/input.css +++ b/crates/web/ui/input.css @@ -1953,6 +1953,10 @@ -webkit-mask-image: url("./icons/masks/mask-b9bf0315417c03e4.svg"); mask-image: url("./icons/masks/mask-b9bf0315417c03e4.svg"); } + .icon-instrumentation { + -webkit-mask-image: url("./icons/masks/instrumentation.svg"); + mask-image: url("./icons/masks/instrumentation.svg"); + } .icon-server { -webkit-mask-image: url("./icons/masks/mask-7a1361dacb5b8427.svg"); mask-image: url("./icons/masks/mask-7a1361dacb5b8427.svg"); diff --git a/crates/web/ui/src/components/forms/SectionLayout.tsx b/crates/web/ui/src/components/forms/SectionLayout.tsx index 1df282dc2..b247f3044 100644 --- a/crates/web/ui/src/components/forms/SectionLayout.tsx +++ b/crates/web/ui/src/components/forms/SectionLayout.tsx @@ -126,7 +126,12 @@ export function StatusMessage({ error, success, className }: StatusMessageProps) const color = error ? "var(--error)" : "var(--accent)"; const text = error ?? success; return ( -
+
{text}
); diff --git a/crates/web/ui/src/pages/SettingsPage.tsx b/crates/web/ui/src/pages/SettingsPage.tsx index 0bcf7267c..0faa31c07 100644 --- a/crates/web/ui/src/pages/SettingsPage.tsx +++ b/crates/web/ui/src/pages/SettingsPage.tsx @@ -201,7 +201,11 @@ const sections: SectionItem[] = [ { id: "terminal", label: "Terminal", page: true }, { id: "monitoring", label: "Monitoring", page: true }, { id: "logs", label: "Logs", page: true }, - { id: "instrumentation", label: "Instrumentation" }, + { + id: "instrumentation", + label: "Instrumentation", + icon: , + }, { id: "graphql", label: "GraphQL" }, { id: "config", label: "Configuration" }, ]; diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index f4e3a2927..80c718163 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -278,6 +278,10 @@ build_targeted_rust_test_cmd() { local test_name test_name="$(basename "$file" .rs)" commands+=("$base_cmd --test $test_name") + elif [[ "$file" == */tests/*.rs ]]; then + # Nested source test modules do not map to Cargo --test targets, and a + # basename such as `mod` is not a reliable nextest filter. + commands+=("$base_cmd") elif grep -Eq '#\[(tokio::)?test\]' "$file" 2>/dev/null; then local filter_name filter_name="$(basename "$file" .rs)" From d7b645d80ce4bf9854a84a9ce85492905666ff51 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:03:37 -0700 Subject: [PATCH 28/33] refactor(observability): drop the unused local price table `pricing.rs` computed a USD cost breakdown for every generation and stored it on the observation record, but nothing ever read it: the OTLP mapping deliberately omits the attribute for both the Langfuse and APM profiles, and no Moltis surface displays cost. The module's own doc comment pointed at a "Metrics page" consumer that was never built. A static price table only stays correct until a provider changes prices, so keeping one that nothing reads is pure decay. Langfuse already prices spend from the model and the usage details we do export, which is why the mapping was written not to send ours in the first place. Removes the module, the `cost_details` record field and the dead `langfuse.observation.cost_details` constant. `set_model` and `set_usage` become plain setters, so their ordering no longer matters; the test that covered re-pricing on late model discovery now covers exactly that. --- .../src/exporters/otlp/attributes.rs | 2 - .../src/exporters/otlp/mapping.rs | 17 +- crates/observability/src/lib.rs | 2 - crates/observability/src/model.rs | 3 - crates/observability/src/pricing.rs | 257 ------------------ crates/observability/src/recorder.rs | 90 ++---- docs/src/instrumentation.md | 8 +- 7 files changed, 35 insertions(+), 344 deletions(-) delete mode 100644 crates/observability/src/pricing.rs diff --git a/crates/observability/src/exporters/otlp/attributes.rs b/crates/observability/src/exporters/otlp/attributes.rs index e0b1ab172..b92982464 100644 --- a/crates/observability/src/exporters/otlp/attributes.rs +++ b/crates/observability/src/exporters/otlp/attributes.rs @@ -57,8 +57,6 @@ pub const OBSERVATION_MODEL: &str = "langfuse.observation.model.name"; pub const OBSERVATION_MODEL_PARAMETERS: &str = "langfuse.observation.model.parameters"; /// Token usage breakdown. pub const OBSERVATION_USAGE_DETAILS: &str = "langfuse.observation.usage_details"; -/// Cost breakdown in USD. -pub const OBSERVATION_COST_DETAILS: &str = "langfuse.observation.cost_details"; /// Managed-prompt name this generation was rendered from. pub const OBSERVATION_PROMPT_NAME: &str = "langfuse.observation.prompt.name"; /// Managed-prompt version this generation was rendered from. diff --git a/crates/observability/src/exporters/otlp/mapping.rs b/crates/observability/src/exporters/otlp/mapping.rs index 5141e5946..9a38e7ed3 100644 --- a/crates/observability/src/exporters/otlp/mapping.rs +++ b/crates/observability/src/exporters/otlp/mapping.rs @@ -515,9 +515,9 @@ fn push_usage_attrs( )); } - // Keep locally computed costs inside Moltis. Langfuse maintains versioned - // model definitions and pricing tiers; sending our static table would take - // precedence over that inference and make stale prices authoritative. + // No cost attribute: Langfuse maintains versioned model definitions and + // prices spend from the model plus these counts, and no other backend has + // a price table to send one to. } /// Managed-prompt linkage. Langfuse-only: no other backend models it. @@ -755,18 +755,17 @@ mod tests { } #[test] - fn token_counts_reach_every_backend_but_cost_detail_does_not() { - let mut obs = generation(); - obs.cost_details.insert("total".into(), 0.42); + fn token_counts_reach_every_backend_but_usage_details_do_not() { + let obs = generation(); let otel = observation_to_span(&obs, &otel_ctx()); assert!(attr_value(&otel, attr::GEN_AI_USAGE_INPUT_TOKENS).is_some()); - // No APM has a model price table, so the breakdown is dead weight. - assert!(attr_value(&otel, attr::OBSERVATION_COST_DETAILS).is_none()); + // The cache-aware breakdown is a Langfuse concept; an APM gets counts. assert!(attr_value(&otel, attr::OBSERVATION_USAGE_DETAILS).is_none()); let lf = observation_to_span(&obs, &langfuse_ctx()); - assert!(attr_value(&lf, attr::OBSERVATION_COST_DETAILS).is_none()); + assert!(attr_value(&lf, attr::GEN_AI_USAGE_INPUT_TOKENS).is_some()); + assert!(attr_value(&lf, attr::OBSERVATION_USAGE_DETAILS).is_some()); } #[test] diff --git a/crates/observability/src/lib.rs b/crates/observability/src/lib.rs index 4922801a9..1e2494654 100644 --- a/crates/observability/src/lib.rs +++ b/crates/observability/src/lib.rs @@ -19,7 +19,6 @@ pub mod builder; pub mod exporters; pub mod feedback; pub mod model; -pub mod pricing; pub mod profile; pub mod recent; pub mod recorder; @@ -36,7 +35,6 @@ pub use { Event, Level, ObservationId, ObservationKind, ObservationRecord, ScoreDeleteRecord, ScoreRecord, ScoreValue, TokenUsage, TraceId, TraceRecord, TraceScope, }, - pricing::{ModelPrice, cost_details, price_for, total_cost}, profile::{ContentCapture, ExportProfile, Vocabulary}, recent::{recent_trace, remember_trace}, recorder::{RecorderSettings, StepGuard, TurnRecorder, wait_for_active_turns}, diff --git a/crates/observability/src/model.rs b/crates/observability/src/model.rs index 6095a6143..1ed53db62 100644 --- a/crates/observability/src/model.rs +++ b/crates/observability/src/model.rs @@ -385,8 +385,6 @@ pub struct ObservationRecord { pub model_parameters: Metadata, /// Token usage, for generation-like observations. pub usage: Option, - /// Cost breakdown in USD, when locally known. - pub cost_details: BTreeMap, /// Managed-prompt name this generation was rendered from. pub prompt_name: Option, /// Managed-prompt version this generation was rendered from. @@ -417,7 +415,6 @@ impl ObservationRecord { model: None, model_parameters: Metadata::new(), usage: None, - cost_details: BTreeMap::new(), prompt_name: None, prompt_version: None, } diff --git a/crates/observability/src/pricing.rs b/crates/observability/src/pricing.rs deleted file mode 100644 index e75a7034d..000000000 --- a/crates/observability/src/pricing.rs +++ /dev/null @@ -1,257 +0,0 @@ -//! Local model pricing. -//! -//! Cost is computed here rather than left to the backend so it exists whether -//! or not anything is configured, and so the numbers on the Metrics page do not -//! depend on a third party being reachable. -//! -//! Prices are USD per million tokens and go stale whenever a provider changes -//! them, which is the accepted cost of not needing a backend to see spend. A -//! model with no entry reports no cost rather than guessing: a wrong number -//! that looks authoritative is worse than a visible gap. - -use std::collections::BTreeMap; - -use crate::model::TokenUsage; - -/// USD per million tokens for one model. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ModelPrice { - /// Fresh input tokens. - pub input: f64, - /// Generated tokens. - pub output: f64, - /// Tokens read from the prompt cache, usually far cheaper than fresh input. - pub cache_read: f64, - /// Tokens written to the prompt cache, usually dearer than fresh input. - pub cache_write: f64, -} - -impl ModelPrice { - /// A price with no cache tiers, for providers that do not bill them - /// separately. - const fn flat(input: f64, output: f64) -> Self { - Self { - input, - output, - cache_read: input, - cache_write: input, - } - } - - /// A price with explicit cache tiers. - const fn cached(input: f64, output: f64, cache_read: f64, cache_write: f64) -> Self { - Self { - input, - output, - cache_read, - cache_write, - } - } - - /// Cost in USD for `usage`, broken down by token bucket. - /// - /// Cache buckets are priced separately rather than folded into input: - /// a cache read can be an order of magnitude cheaper, and summing them - /// would overstate spend on exactly the workloads caching is meant to help. - #[must_use] - pub fn cost_details(&self, usage: &TokenUsage) -> BTreeMap { - const PER: f64 = 1_000_000.0; - let mut details = BTreeMap::new(); - let mut put = |key: &str, tokens: u32, rate: f64| { - if tokens > 0 && rate > 0.0 { - details.insert(key.to_string(), (tokens as f64) * rate / PER); - } - }; - put("input", usage.input, self.input); - put("output", usage.output, self.output); - put("cache_read", usage.cache_read, self.cache_read); - put("cache_write", usage.cache_write, self.cache_write); - - let total: f64 = details.values().sum(); - if total > 0.0 { - details.insert("total".to_string(), total); - } - details - } -} - -/// Built-in prices, keyed by a normalized model id prefix. -/// -/// Matched by longest prefix so a dated release like -/// `claude-opus-4-5-20260101` picks up its family's price without an entry per -/// snapshot. -const PRICES: &[(&str, ModelPrice)] = &[ - // Anthropic - ("claude-opus-4", ModelPrice::cached(15.0, 75.0, 1.5, 18.75)), - ("claude-sonnet-4", ModelPrice::cached(3.0, 15.0, 0.3, 3.75)), - ("claude-haiku-4", ModelPrice::cached(1.0, 5.0, 0.1, 1.25)), - ("claude-3-5-haiku", ModelPrice::cached(0.8, 4.0, 0.08, 1.0)), - ( - "claude-3-5-sonnet", - ModelPrice::cached(3.0, 15.0, 0.3, 3.75), - ), - ("claude-3-opus", ModelPrice::cached(15.0, 75.0, 1.5, 18.75)), - // OpenAI - ("gpt-4o-mini", ModelPrice::cached(0.15, 0.6, 0.075, 0.15)), - ("gpt-4o", ModelPrice::cached(2.5, 10.0, 1.25, 2.5)), - ("gpt-4.1-mini", ModelPrice::cached(0.4, 1.6, 0.1, 0.4)), - ("gpt-4.1", ModelPrice::cached(2.0, 8.0, 0.5, 2.0)), - ("o3-mini", ModelPrice::flat(1.1, 4.4)), - ("o3", ModelPrice::flat(2.0, 8.0)), - // Google - ("gemini-2.5-pro", ModelPrice::flat(1.25, 10.0)), - ("gemini-2.5-flash", ModelPrice::flat(0.3, 2.5)), - ("gemini-2.0-flash", ModelPrice::flat(0.1, 0.4)), - // Meta / open weights, typical hosted rates - ("llama-3.3-70b", ModelPrice::flat(0.6, 0.6)), - ("llama-3.1-8b", ModelPrice::flat(0.05, 0.08)), - ("mistral-large", ModelPrice::flat(2.0, 6.0)), - ("deepseek-chat", ModelPrice::flat(0.27, 1.1)), - ("deepseek-reasoner", ModelPrice::flat(0.55, 2.19)), -]; - -/// Look up a price for `model`. -/// -/// Provider prefixes (`anthropic/claude-opus-4`) are stripped first so a model -/// routed through OpenRouter prices the same as one called directly. -#[must_use] -pub fn price_for(model: &str) -> Option { - let normalized = model - .rsplit('/') - .next() - .unwrap_or(model) - .trim() - .to_ascii_lowercase(); - if normalized.is_empty() { - return None; - } - - PRICES - .iter() - .filter(|(prefix, _)| normalized.starts_with(prefix)) - // Longest prefix wins, so `gpt-4o-mini` does not match `gpt-4o`. - .max_by_key(|(prefix, _)| prefix.len()) - .map(|(_, price)| *price) -} - -/// Cost breakdown for `usage` on `model`, empty when the model is unpriced. -#[must_use] -pub fn cost_details(model: &str, usage: &TokenUsage) -> BTreeMap { - price_for(model).map_or_else(BTreeMap::new, |price| price.cost_details(usage)) -} - -/// Total USD for `usage` on `model`, or `None` when the model is unpriced. -#[must_use] -pub fn total_cost(model: &str, usage: &TokenUsage) -> Option { - cost_details(model, usage).get("total").copied() -} - -#[cfg(test)] -#[allow(clippy::expect_used, clippy::unwrap_used)] -mod tests { - use super::*; - - fn usage(input: u32, output: u32, cache_read: u32, cache_write: u32) -> TokenUsage { - TokenUsage::from_provider_totals(input, output, cache_read, cache_write) - } - - #[test] - fn a_known_model_prices_input_and_output() { - let details = cost_details("claude-opus-4", &usage(1_000_000, 1_000_000, 0, 0)); - - assert!((details["input"] - 15.0).abs() < 1e-9); - assert!((details["output"] - 75.0).abs() < 1e-9); - assert!((details["total"] - 90.0).abs() < 1e-9); - } - - #[test] - fn cache_reads_are_not_priced_as_fresh_input() { - // Folding cache into input would overstate spend by 10x on exactly the - // workloads caching exists to make cheaper. - let details = cost_details("claude-opus-4", &usage(0, 0, 1_000_000, 0)); - - assert!((details["cache_read"] - 1.5).abs() < 1e-9); - assert!(!details.contains_key("input")); - } - - #[test] - fn cache_writes_are_priced_above_fresh_input() { - let details = cost_details("claude-opus-4", &usage(0, 0, 0, 1_000_000)); - assert!(details["cache_write"] > 15.0); - } - - #[test] - fn the_longest_matching_prefix_wins() { - // `gpt-4o-mini` must not be priced as `gpt-4o`. - let mini = price_for("gpt-4o-mini").expect("priced"); - let full = price_for("gpt-4o").expect("priced"); - - assert!((mini.input - 0.15).abs() < 1e-9); - assert!((full.input - 2.5).abs() < 1e-9); - } - - #[test] - fn dated_snapshots_inherit_the_family_price() { - let dated = price_for("claude-opus-4-5-20260101").expect("priced"); - let family = price_for("claude-opus-4").expect("priced"); - - assert_eq!(dated, family); - } - - #[test] - fn a_provider_prefix_is_stripped() { - // The same model routed through OpenRouter must price identically. - assert_eq!( - price_for("anthropic/claude-opus-4"), - price_for("claude-opus-4") - ); - assert_eq!(price_for("openai/gpt-4o"), price_for("gpt-4o")); - } - - #[test] - fn model_ids_are_matched_case_insensitively() { - assert_eq!(price_for("Claude-Opus-4"), price_for("claude-opus-4")); - } - - #[test] - fn an_unknown_model_reports_no_cost_rather_than_guessing() { - // A confident wrong number is worse than a visible gap. - assert!(price_for("some-private-finetune").is_none()); - assert!(cost_details("some-private-finetune", &usage(1000, 1000, 0, 0)).is_empty()); - assert_eq!( - total_cost("some-private-finetune", &usage(1000, 1000, 0, 0)), - None - ); - } - - #[test] - fn an_empty_model_id_is_unpriced() { - assert!(price_for("").is_none()); - assert!(price_for(" ").is_none()); - } - - #[test] - fn zero_usage_produces_no_entries() { - assert!(cost_details("claude-opus-4", &usage(0, 0, 0, 0)).is_empty()); - } - - #[test] - fn the_total_is_the_sum_of_the_buckets() { - let details = cost_details("claude-opus-4", &usage(1_000, 2_000, 3_000, 4_000)); - let summed: f64 = details - .iter() - .filter(|(k, _)| k.as_str() != "total") - .map(|(_, v)| v) - .sum(); - - assert!((details["total"] - summed).abs() < 1e-12); - } - - #[test] - fn flat_priced_models_charge_cache_reads_as_input() { - // Providers that do not bill a cache tier must not silently price - // cached tokens at zero. - let details = cost_details("gemini-2.5-pro", &usage(0, 0, 1_000_000, 0)); - assert!((details["cache_read"] - 1.25).abs() < 1e-9); - } -} diff --git a/crates/observability/src/recorder.rs b/crates/observability/src/recorder.rs index 4ca590cb5..c12cfe25d 100644 --- a/crates/observability/src/recorder.rs +++ b/crates/observability/src/recorder.rs @@ -332,22 +332,10 @@ impl StepGuard { } } - /// Set the model this step used. /// Name the model this step used. - /// - /// Re-prices any usage already recorded: callers are free to set usage - /// before the model is known, and without this the step would keep a cost - /// of zero forever. pub fn set_model(&mut self, model: impl Into) { if let Some(record) = self.record.as_mut() { - let model = model.into(); - if let Some(usage) = record.usage { - let details = crate::pricing::cost_details(&model, &usage); - if !details.is_empty() { - record.cost_details = details; - } - } - record.model = Some(model); + record.model = Some(model.into()); } } @@ -358,20 +346,13 @@ impl StepGuard { } } - /// Set token usage. - /// Attach token usage, pricing it locally when the model is known. + /// Attach token usage. /// - /// Cost is computed here rather than left to the backend so it exists - /// without one configured, and so every backend sees the same number - /// instead of each deriving its own from a private price table. + /// Cost is deliberately not derived from it: Langfuse maintains versioned + /// model prices and infers spend from the model plus these counts, and no + /// other backend has a price table at all. pub fn set_usage(&mut self, usage: TokenUsage) { if let Some(record) = self.record.as_mut() { - if let Some(model) = record.model.as_deref() { - let details = crate::pricing::cost_details(model, &usage); - if !details.is_empty() { - record.cost_details = details; - } - } record.usage = Some(usage); } } @@ -934,56 +915,31 @@ mod tests { } #[test] - fn usage_is_priced_when_the_model_is_known() { + fn model_and_usage_are_recorded_in_either_order() { + // Callers set whichever they learn first: a streaming provider names + // the model up front and reports usage at the end, a non-streaming one + // returns both together. with_sink(|collected| { let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) .expect("recorder starts"); - let mut step = recorder.step(ObservationKind::Generation, "gen"); - step.set_model("claude-opus-4"); - step.set_usage(TokenUsage::from_provider_totals(1_000_000, 0, 0, 0)); - step.finish(); - let observations = collected.observations(); - let record = observations.last().expect("one observation"); - assert!((record.cost_details["total"] - 15.0).abs() < 1e-9); - }); - } + let mut model_first = recorder.step(ObservationKind::Generation, "model-first"); + model_first.set_model("claude-opus-4"); + model_first.set_usage(TokenUsage::from_provider_totals(1_000, 500, 0, 0)); + model_first.finish(); - #[test] - fn usage_recorded_before_the_model_is_still_priced() { - // Callers are free to set usage first; without re-pricing on - // `set_model` the step would keep a cost of zero forever. - with_sink(|collected| { - let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) - .expect("recorder starts"); - let mut step = recorder.step(ObservationKind::Generation, "gen"); - step.set_usage(TokenUsage::from_provider_totals(1_000_000, 0, 0, 0)); - step.set_model("claude-opus-4"); - step.finish(); + let mut usage_first = recorder.step(ObservationKind::Generation, "usage-first"); + usage_first.set_usage(TokenUsage::from_provider_totals(1_000, 500, 0, 0)); + usage_first.set_model("claude-opus-4"); + usage_first.finish(); let observations = collected.observations(); - let record = observations.last().expect("one observation"); - assert!((record.cost_details["total"] - 15.0).abs() < 1e-9); - }); - } - - #[test] - fn an_unpriced_model_records_usage_without_a_cost() { - with_sink(|collected| { - let recorder = TurnRecorder::begin("run", scope(), RecorderSettings::default()) - .expect("recorder starts"); - let mut step = recorder.step(ObservationKind::Generation, "gen"); - step.set_model("some-private-finetune"); - step.set_usage(TokenUsage::from_provider_totals(1_000, 1_000, 0, 0)); - step.finish(); - - let observations = collected.observations(); - let record = observations.last().expect("one observation"); - assert!(record.usage.is_some()); - assert!( - record.cost_details.is_empty(), - "a guessed cost is worse than none" - ); + assert_eq!(observations.len(), 2); + for record in observations { + assert_eq!(record.model.as_deref(), Some("claude-opus-4")); + assert_eq!(record.usage.map(|u| u.input), Some(1_000)); + assert_eq!(record.usage.map(|u| u.output), Some(500)); + } }); } diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md index e40c49130..002ac72fc 100644 --- a/docs/src/instrumentation.md +++ b/docs/src/instrumentation.md @@ -317,10 +317,10 @@ score after deletion. Two reacting users still count separately. ## Cost Moltis exports the model id and token usage details, including separate -cache-read and cache-write buckets, but it does not export costs from its static -local price table. Langfuse infers cost using its own versioned model definitions -and pricing tiers, avoiding stale Moltis prices overriding Langfuse's -calculations. +cache-read and cache-write buckets, but it does not compute or export a cost of +its own. Langfuse infers cost from its own versioned model definitions and +pricing tiers, which stay current without Moltis shipping a price table that +would go stale and then override them. Unknown, private or custom models may show no cost until their pricing is configured in Langfuse. From 6461777efbfb019e90881a0c51debd67487559ad Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:03:57 -0700 Subject: [PATCH 29/33] fix(observability): make the exporter features real build toggles `moltis-observability` declared `langfuse`, `otlp` and `metrics` features that nothing could reach. No dependent forwarded them, `crates/cli` never mentioned the crate, and the two exporter features did not even build when switched off: `builder.rs` imported both exporter modules unconditionally, so `cargo check -p moltis-observability --no-default-features` failed. The one metrics counter in the crate was compiled out of every build we ship. Gates the exporters properly and forwards the features up through `moltis-gateway` to `moltis-cli`, default-on per workspace policy. All four combinations of `langfuse`/`otlp` now build, lint and test clean. `langfuse` implies `otlp` because Langfuse ingests traces over the OTLP wire format, which also carries Datadog. A backend enabled in config that the binary lacks is reported as skipped with a reason rather than silently doing nothing, so it cannot be mistaken for a credential problem. The feedback paths took a `&Arc` they never called a method on: scores travel through the global sink, and the argument only served as an "is a score backend up?" flag. Replaced with `scores_available: bool`, sourced from a new `BuiltInstrumentation::scores`. That drops the Langfuse type out of `moltis-channels` entirely and keeps every `#[cfg]` for this feature inside the observability crate. --- Cargo.toml | 2 +- crates/channels/src/feedback.rs | 58 +++----- crates/cli/Cargo.toml | 10 +- crates/gateway/Cargo.toml | 7 +- crates/gateway/src/channel_events/sink.rs | 2 +- .../gateway/src/methods/services/feedback.rs | 4 +- .../src/methods/services/instrumentation.rs | 8 ++ crates/gateway/src/server/instrumentation.rs | 75 ++++++++-- crates/observability/Cargo.toml | 2 +- crates/observability/src/builder.rs | 136 ++++++++++++++++-- crates/observability/tests/end_to_end.rs | 4 + docs/src/instrumentation.md | 19 +++ 12 files changed, 253 insertions(+), 74 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8689d22bd..690abc835 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -403,7 +403,7 @@ moltis-network-filter = { default-features = false, path = "crates/network-fil moltis-node-host = { path = "crates/node-host" } moltis-nostr = { path = "crates/nostr" } moltis-oauth = { path = "crates/oauth" } -moltis-observability = { path = "crates/observability" } +moltis-observability = { default-features = false, path = "crates/observability" } moltis-onboarding = { path = "crates/onboarding" } moltis-openclaw-import = { path = "crates/openclaw-import" } moltis-plugins = { path = "crates/plugins" } diff --git a/crates/channels/src/feedback.rs b/crates/channels/src/feedback.rs index e67f41ad8..d259d83ed 100644 --- a/crates/channels/src/feedback.rs +++ b/crates/channels/src/feedback.rs @@ -10,8 +10,8 @@ use std::sync::{Arc, RwLock}; use { moltis_config::FeedbackSettings, moltis_observability::{ - FeedbackSignal, FeedbackVocabulary, ScoreDeleteRecord, TraceId, - exporters::langfuse::LangfuseClient, feedback_score, feedback_score_id, + FeedbackSignal, FeedbackVocabulary, ScoreDeleteRecord, TraceId, feedback_score, + feedback_score_id, }, tracing::{debug, warn}, }; @@ -132,9 +132,9 @@ impl FeedbackService { /// Handle a reaction change on a channel message. /// - /// `langfuse` is only needed to retract: adding a score goes through the - /// instrumentation sink like any other event, but deletion has no sink - /// representation and must call the API directly. + /// `scores_available` says whether a backend that can store a score is + /// running. Without one, both halves of the toggle would be recorded into + /// nothing, so the reaction is reported as disabled instead. pub async fn on_reaction( &self, channel: &str, @@ -144,7 +144,7 @@ impl FeedbackService { emoji: &str, user_id: &str, added: bool, - langfuse: Option<&Arc>, + scores_available: bool, ) -> FeedbackOutcome { // Classify before the database lookup: most reactions in a busy chat // are not feedback, and they should not cost a query each. @@ -168,7 +168,7 @@ impl FeedbackService { added.then_some(signal), user_id, Some(format!("{channel} reaction {emoji}")), - langfuse, + scores_available, ) .await } @@ -184,7 +184,7 @@ impl FeedbackService { signal: Option, user_id: &str, comment: Option, - langfuse: Option<&Arc>, + scores_available: bool, ) -> FeedbackOutcome { let Some((links, enabled, environment, retention_days)) = self.read(|active| { ( @@ -196,7 +196,7 @@ impl FeedbackService { }) else { return FeedbackOutcome::Disabled; }; - if !enabled || langfuse.is_none() { + if !enabled || !scores_available { return FeedbackOutcome::Disabled; } @@ -333,19 +333,6 @@ mod tests { (service, links) } - fn langfuse_client() -> Arc { - Arc::new(LangfuseClient::new( - moltis_observability::exporters::langfuse::LangfuseConfig { - host: "https://cloud.langfuse.com".into(), - public_key: "pk-test".into(), - secret_key: "sk-test".to_string().into(), - environment: Some("test".into()), - release: None, - timeout: std::time::Duration::from_secs(1), - }, - )) - } - /// Collects scores emitted through the global sink. /// /// The sink is process-wide, so the tests that assert on emitted scores @@ -431,7 +418,6 @@ mod tests { let (service, links) = service(true); link_reply(&links, "42", "trace-1").await; - let langfuse = langfuse_client(); let outcome = service .on_reaction( @@ -442,7 +428,7 @@ mod tests { "\u{1f44d}", "99", true, - Some(&langfuse), + true, ) .await; @@ -474,7 +460,7 @@ mod tests { "\u{1f389}", "99", true, - None, + false, ) .await; @@ -484,7 +470,6 @@ mod tests { #[tokio::test] async fn a_reaction_on_an_unlinked_message_is_ignored() { let (service, _links) = service(true); - let langfuse = langfuse_client(); let outcome = service .on_reaction( "telegram", @@ -494,7 +479,7 @@ mod tests { "\u{1f44d}", "99", true, - Some(&langfuse), + true, ) .await; @@ -508,7 +493,6 @@ mod tests { moltis_observability::set_global_sink(Arc::clone(&sink) as Arc<_>); let (service, links) = service(true); link_reply(&links, "42", "trace-1").await; - let langfuse = langfuse_client(); let outcome = service .on_reaction( @@ -519,7 +503,7 @@ mod tests { "\u{1f44d}", "99", false, - Some(&langfuse), + true, ) .await; @@ -549,7 +533,6 @@ mod tests { Some("production".into()), ); link_reply(&links, "42", "trace-1").await; - let langfuse = langfuse_client(); let outcome = service .submit_signal( @@ -560,7 +543,7 @@ mod tests { Some(FeedbackSignal::Positive), "99", Some("typed feedback".into()), - Some(&langfuse), + true, ) .await; @@ -590,7 +573,6 @@ mod tests { }) .await .unwrap(); - let langfuse = langfuse_client(); let outcome = service .on_reaction( @@ -601,7 +583,7 @@ mod tests { "\u{1f44d}", "99", true, - Some(&langfuse), + true, ) .await; @@ -609,7 +591,7 @@ mod tests { } #[tokio::test] - async fn score_feedback_is_disabled_without_langfuse() { + async fn score_feedback_is_disabled_without_a_score_backend() { let (service, links) = service(true); link_reply(&links, "42", "trace-1").await; @@ -622,7 +604,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + false, ) .await; @@ -643,7 +625,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + true, ) .await; @@ -690,7 +672,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + true, ) .await, FeedbackOutcome::Disabled @@ -732,7 +714,7 @@ mod tests { "\u{1f44d}", "99", true, - None, + true, ) .await, FeedbackOutcome::Disabled diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 6aa8a59e8..3de28ce58 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -158,6 +158,7 @@ full = [ "lang-rust", "lang-toml", "lang-typescript", + "langfuse", "llm-compaction", "local-embeddings", "local-llm", @@ -170,6 +171,7 @@ full = [ "ngrok", "nostr", "openclaw-import", + "otlp", "prometheus", "provider-github-copilot", "provider-kimi-code", @@ -221,8 +223,13 @@ lang-typescript = ["moltis-memory/lang-typescript"] # jemalloc is intentionally unavailable on linux/aarch64 to avoid runtime # crashes on 16 KiB page kernels (e.g. Raspberry Pi OS variants). claude-import = ["dep:moltis-claude-import", "moltis-gateway/claude-import"] -codex-import = ["dep:moltis-codex-import", "moltis-gateway/codex-import"] +codex-import = ["dep:moltis-codex-import", "moltis-gateway/codex-import"] hermes-import = ["dep:moltis-hermes-import", "moltis-gateway/hermes-import"] +# Instrumentation exporters. Default-on so the `[instrumentation]` config can be +# switched on without a rebuild; enabling the config, not the feature, is what +# starts sending data. `langfuse` implies `otlp`: Langfuse ingests traces over +# the OTLP wire format. +langfuse = ["moltis-gateway/langfuse"] lightweight = ["jemalloc", "tls", "web-ui"] local-embeddings = ["moltis-gateway/local-embeddings"] local-llm = ["moltis-gateway/local-llm"] @@ -237,6 +244,7 @@ netbird = ["moltis-httpd/netbird", "moltis-web?/netbird"] ngrok = ["moltis-httpd/ngrok", "moltis-web?/ngrok"] nostr = ["moltis-httpd/nostr"] openclaw-import = ["dep:moltis-openclaw-import", "moltis-gateway/openclaw-import"] +otlp = ["moltis-gateway/otlp"] prometheus = ["moltis-httpd/prometheus"] provider-github-copilot = [ "moltis-gateway/provider-github-copilot", diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 5c4816a0f..92469d71c 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -47,8 +47,8 @@ moltis-msteams = { optional = true, workspace = true } moltis-netbird = { optional = true, workspace = true } moltis-network-filter = { features = ["proxy", "service"], optional = true, workspace = true } moltis-nostr = { optional = true, workspace = true } -moltis-observability = { workspace = true } moltis-oauth = { workspace = true } +moltis-observability = { workspace = true } moltis-onboarding = { workspace = true } moltis-openclaw-import = { optional = true, workspace = true } moltis-plugins = { workspace = true } @@ -126,6 +126,7 @@ default = [ "graphql", "hermes-import", "home-assistant", + "langfuse", "local-llm", "local-llm-metal", "matrix", @@ -135,6 +136,7 @@ default = [ "netbird", "nostr", "openclaw-import", + "otlp", "prometheus", "provider-github-copilot", "provider-kimi-code", @@ -166,6 +168,7 @@ fs-tools = ["moltis-tools/fs-tools"] graphql = [] hermes-import = ["dep:moltis-hermes-import"] home-assistant = ["dep:moltis-home-assistant"] +langfuse = ["moltis-observability/langfuse"] llm-compaction = ["moltis-chat/llm-compaction"] local-embeddings = ["moltis-memory/local-embeddings"] local-llm = [ @@ -186,12 +189,14 @@ metrics = [ "moltis-metrics/sqlite", "moltis-msteams?/metrics", "moltis-nostr?/metrics", + "moltis-observability/metrics", "moltis-signal?/metrics", ] msteams = ["dep:moltis-msteams"] netbird = ["dep:moltis-netbird"] nostr = ["dep:moltis-nostr"] openclaw-import = ["dep:moltis-openclaw-import"] +otlp = ["moltis-observability/otlp"] prometheus = ["metrics", "moltis-metrics/prometheus"] provider-github-copilot = [ "moltis-provider-setup/provider-github-copilot", diff --git a/crates/gateway/src/channel_events/sink.rs b/crates/gateway/src/channel_events/sink.rs index 3330f16cc..85d267289 100644 --- a/crates/gateway/src/channel_events/sink.rs +++ b/crates/gateway/src/channel_events/sink.rs @@ -41,7 +41,7 @@ async fn handle_reaction_feedback(state: &Arc, event: &ChannelEven emoji, user_id, *added, - state.instrumentation.langfuse().as_ref(), + state.instrumentation.scores_available(), ) .await; debug!( diff --git a/crates/gateway/src/methods/services/feedback.rs b/crates/gateway/src/methods/services/feedback.rs index 1a382031e..05d774ddd 100644 --- a/crates/gateway/src/methods/services/feedback.rs +++ b/crates/gateway/src/methods/services/feedback.rs @@ -28,7 +28,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { // the result, so the UI hides it unless both are on. "enabled": config.enabled && config.feedback.enabled - && ctx.state.instrumentation.langfuse().is_some(), + && ctx.state.instrumentation.scores_available(), "instrumentation_active": ctx.state.instrumentation.status().active, "retention_days": config.feedback.link_retention_days, })) @@ -112,7 +112,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { signal, user_id, Some("web feedback".to_string()), - ctx.state.instrumentation.langfuse().as_ref(), + ctx.state.instrumentation.scores_available(), ) .await; diff --git a/crates/gateway/src/methods/services/instrumentation.rs b/crates/gateway/src/methods/services/instrumentation.rs index ea3ad2541..cf553a9de 100644 --- a/crates/gateway/src/methods/services/instrumentation.rs +++ b/crates/gateway/src/methods/services/instrumentation.rs @@ -70,6 +70,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { .unwrap_or("langfuse"); match backend { + #[cfg(feature = "langfuse")] "langfuse" => { let Some(client) = ctx.state.instrumentation.langfuse() else { return Ok(serde_json::json!({ @@ -86,6 +87,13 @@ pub(super) fn register(reg: &mut MethodRegistry) { })), } }, + // Say so rather than falling through to the generic "no test + // available" answer, which would read as a Langfuse quirk. + #[cfg(not(feature = "langfuse"))] + "langfuse" => Ok(serde_json::json!({ + "ok": false, + "error": "this build was compiled without the `langfuse` feature", + })), // OTLP collectors have no standard health endpoint, and // POSTing a probe span would pollute the operator's traces // with fake data. Live delivery counters are the honest diff --git a/crates/gateway/src/server/instrumentation.rs b/crates/gateway/src/server/instrumentation.rs index fcc6f4ca9..5adf9e171 100644 --- a/crates/gateway/src/server/instrumentation.rs +++ b/crates/gateway/src/server/instrumentation.rs @@ -11,13 +11,13 @@ use std::{ use { moltis_config::InstrumentationConfig, - moltis_observability::{ - BuiltInstrumentation, ObservationSink, SinkStatsSnapshot, SkippedBackend, - exporters::langfuse::LangfuseClient, - }, + moltis_observability::{ObservationSink, SinkStatsSnapshot, SkippedBackend}, tracing::{info, warn}, }; +#[cfg(feature = "langfuse")] +use moltis_observability::exporters::langfuse::LangfuseClient; + /// Live instrumentation state, surfaced by the `instrumentation.*` RPC methods. #[derive(Default)] pub struct InstrumentationState { @@ -30,6 +30,8 @@ struct InstrumentationSnapshot { sink: Option>, backends: Vec, skipped: Vec, + scores: bool, + #[cfg(feature = "langfuse")] langfuse: Option>, } @@ -55,19 +57,16 @@ impl InstrumentationState { pub fn apply(&self, config: &InstrumentationConfig, release: &str) -> InstrumentationStatus { let outcome = moltis_observability::build(config, release); let snapshot = match outcome.built { - Some(BuiltInstrumentation { - sink, - langfuse, - backends, - .. - }) => { - moltis_observability::set_global_sink(Arc::clone(&sink)); - info!(backends = ?backends, "agent instrumentation active"); + Some(built) => { + moltis_observability::set_global_sink(Arc::clone(&built.sink)); + info!(backends = ?built.backends, "agent instrumentation active"); InstrumentationSnapshot { - sink: Some(sink), - backends, + sink: Some(built.sink), + backends: built.backends, skipped: outcome.skipped, - langfuse, + scores: built.scores, + #[cfg(feature = "langfuse")] + langfuse: built.langfuse, } }, None => { @@ -109,7 +108,21 @@ impl InstrumentationState { guard.status() } + /// Whether a running backend can store scores. + /// + /// The feedback surfaces ask this instead of naming a backend: a thumb that + /// nothing records is worse than no thumb at all. + #[must_use] + pub fn scores_available(&self) -> bool { + let guard = self + .inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + guard.scores + } + /// The Langfuse client, when that backend is running. + #[cfg(feature = "langfuse")] #[must_use] pub fn langfuse(&self) -> Option> { let guard = self @@ -186,6 +199,7 @@ mod tests { use super::*; + #[cfg(feature = "langfuse")] fn valid_langfuse_config() -> InstrumentationConfig { InstrumentationConfig { enabled: true, @@ -271,6 +285,7 @@ mod tests { moltis_observability::clear_global_sink(); } + #[cfg(feature = "langfuse")] #[tokio::test] #[serial_test::serial(instrumentation_global_sink)] async fn applying_a_valid_config_installs_the_sink() { @@ -294,6 +309,7 @@ mod tests { moltis_observability::clear_global_sink(); } + #[cfg(feature = "langfuse")] #[tokio::test] #[serial_test::serial(instrumentation_global_sink)] async fn reapplying_a_disabled_config_tears_the_sink_down() { @@ -310,6 +326,7 @@ mod tests { assert!(state.langfuse().is_none()); } + #[cfg(feature = "langfuse")] #[tokio::test] #[serial_test::serial(instrumentation_global_sink)] async fn skipped_backends_are_reported_not_just_logged() { @@ -330,6 +347,34 @@ mod tests { moltis_observability::clear_global_sink(); } + #[cfg(not(feature = "langfuse"))] + #[tokio::test] + #[serial_test::serial(instrumentation_global_sink)] + async fn a_backend_this_build_lacks_is_reported_as_skipped() { + // Without the exporter compiled in, enabling it in config must explain + // itself rather than looking like a credential mistake. + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + }; + + let state = InstrumentationState::default(); + let status = state.apply(&config, "test"); + + assert!(!status.active); + assert!(!state.scores_available()); + assert_eq!(status.skipped.len(), 1); + assert_eq!(status.skipped[0].name, "langfuse"); + assert_eq!(status.skipped[0].reason, "not compiled into this build"); + } + #[tokio::test] async fn status_before_any_apply_is_inactive() { let state = InstrumentationState::default(); diff --git a/crates/observability/Cargo.toml b/crates/observability/Cargo.toml index 9981b5752..05374f281 100644 --- a/crates/observability/Cargo.toml +++ b/crates/observability/Cargo.toml @@ -5,7 +5,7 @@ version.workspace = true [features] default = ["langfuse", "otlp"] -langfuse = [] +langfuse = ["otlp"] metrics = ["dep:moltis-metrics"] otlp = [] diff --git a/crates/observability/src/builder.rs b/crates/observability/src/builder.rs index 4a332bc30..8ab37445b 100644 --- a/crates/observability/src/builder.rs +++ b/crates/observability/src/builder.rs @@ -1,27 +1,42 @@ //! Construction of live sinks from `[instrumentation]` configuration. -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use { - moltis_config::{ - ContentCaptureMode, DatadogSettings, InstrumentationConfig, LangfuseSettings, OtlpSettings, - }, - secrecy::ExposeSecret, + moltis_config::{ContentCaptureMode, InstrumentationConfig}, tracing::{info, warn}, }; use crate::{ - exporters::{ - langfuse::{LangfuseClient, LangfuseConfig, ScoreSink}, - otlp::{OtlpConfig, OtlpTransport}, - }, - profile::{ContentCapture, ExportProfile}, + profile::ContentCapture, recorder::RecorderSettings, redact::RedactionPolicy, - runtime::{BatchConfig, BatchSink}, sink::{ObservationSink, SinkFanout}, }; +// Everything below is needed only by an exporter that was compiled in. A build +// with no exporter feature still parses `[instrumentation]` and reports each +// enabled backend as skipped, so the config never silently does nothing. +#[cfg(feature = "langfuse")] +use { + crate::exporters::langfuse::{LangfuseClient, LangfuseConfig, ScoreSink}, + moltis_config::LangfuseSettings, +}; +#[cfg(any(feature = "langfuse", feature = "otlp"))] +use { + crate::runtime::{BatchConfig, BatchSink}, + secrecy::ExposeSecret, + std::time::Duration, +}; +#[cfg(feature = "otlp")] +use { + crate::{ + exporters::otlp::{OtlpConfig, OtlpTransport}, + profile::ExportProfile, + }, + moltis_config::{DatadogSettings, OtlpSettings}, +}; + /// Convert the config-level content mode to the profile-level one. impl From for ContentCapture { fn from(mode: ContentCaptureMode) -> Self { @@ -39,8 +54,14 @@ pub struct BuiltInstrumentation { pub sink: Arc, /// Settings the agent runtime applies when recording. pub recorder_settings: RecorderSettings, + /// Whether a backend that accepts scores is running. + /// + /// Scores are the one event type most backends have no representation for, + /// so the feedback surfaces ask this rather than naming a backend. + pub scores: bool, /// Langfuse REST client, present when the Langfuse backend is enabled. - /// Used for scores and the settings UI connection test. + /// Used for the settings UI connection test. + #[cfg(feature = "langfuse")] pub langfuse: Option>, /// Names of the backends that were built. pub backends: Vec, @@ -65,6 +86,7 @@ pub struct BuildOutcome { } /// Batching parameters shared by every backend. +#[cfg(any(feature = "langfuse", feature = "otlp"))] fn batch_config(config: &InstrumentationConfig) -> BatchConfig { BatchConfig { max_batch_bytes: config.max_batch_bytes, @@ -92,6 +114,7 @@ fn validate_batch_config(config: &InstrumentationConfig) -> Result<(), String> { /// conversation content and credentials in a header, and shipping them /// unencrypted across a network is not a trade-off an operator should make by /// typing a URL. Loopback is allowed for a local Agent or collector. +#[cfg(any(feature = "langfuse", feature = "otlp"))] fn validate_endpoint(raw: &str) -> Result<(), String> { let Ok(url) = reqwest::Url::parse(raw) else { return Err("endpoint is not a valid URL".to_string()); @@ -126,6 +149,7 @@ fn validate_endpoint(raw: &str) -> Result<(), String> { /// no representation for a score and drops those events, so scores need their /// own sink onto the ingestion API — otherwise every score recorded would be /// silently discarded on the way out. +#[cfg(feature = "langfuse")] fn build_langfuse( settings: &LangfuseSettings, config: &InstrumentationConfig, @@ -162,6 +186,7 @@ fn build_langfuse( } /// Build a generic OTLP backend. +#[cfg(feature = "otlp")] fn build_otlp( settings: &OtlpSettings, config: &InstrumentationConfig, @@ -194,7 +219,8 @@ fn build_otlp( Ok(Arc::new(BatchSink::spawn(transport, batch_config(config)))) } -/// Build the Datadog backend. +/// Build the Datadog backend, over the same OTLP wire format. +#[cfg(feature = "otlp")] fn build_datadog( settings: &DatadogSettings, config: &InstrumentationConfig, @@ -242,6 +268,13 @@ fn build_datadog( /// Must be called from within a Tokio runtime — each backend spawns a /// background export task. #[must_use] +// With no exporter compiled in there is nothing to build: every enabled backend +// is reported as skipped, so the accumulators are never written and the release +// string is never read. +#[cfg_attr( + not(any(feature = "langfuse", feature = "otlp")), + allow(unused_mut, unused_variables) +)] pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { let mut skipped = Vec::new(); @@ -273,9 +306,11 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { let mut sinks: Vec> = Vec::new(); let mut backends = Vec::new(); + #[cfg(feature = "langfuse")] let mut langfuse_client = None; if config.langfuse.enabled { + #[cfg(feature = "langfuse")] match build_langfuse(&config.langfuse, config, release) { Ok((langfuse_sinks, client)) => { sinks.extend(langfuse_sinks); @@ -290,9 +325,12 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { }); }, } + #[cfg(not(feature = "langfuse"))] + skipped.push(not_compiled_in("langfuse")); } if config.otlp.enabled { + #[cfg(feature = "otlp")] match build_otlp(&config.otlp, config, release) { Ok(sink) => { sinks.push(sink); @@ -306,9 +344,12 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { }); }, } + #[cfg(not(feature = "otlp"))] + skipped.push(not_compiled_in("otlp")); } if config.datadog.enabled { + #[cfg(feature = "otlp")] match build_datadog(&config.datadog, config, release) { Ok(sink) => { sinks.push(sink); @@ -322,6 +363,9 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { }); }, } + // Datadog is reached over the same OTLP wire format. + #[cfg(not(feature = "otlp"))] + skipped.push(not_compiled_in("datadog")); } if sinks.is_empty() { @@ -348,6 +392,10 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { built: Some(BuiltInstrumentation { sink: Arc::new(SinkFanout::new(sinks)), recorder_settings, + // Langfuse is the only backend with a score representation; the + // OTLP mapping drops score events on the floor. + scores: backends.iter().any(|name| name == "langfuse"), + #[cfg(feature = "langfuse")] langfuse: langfuse_client, backends, }), @@ -355,7 +403,27 @@ pub fn build(config: &InstrumentationConfig, release: &str) -> BuildOutcome { } } -#[cfg(test)] +/// Report a backend the operator enabled but this binary cannot speak to. +/// +/// Reported as a skipped backend rather than logged, so a build without the +/// exporter feature explains itself in the settings UI instead of looking like +/// a misconfiguration. +#[cfg(not(all(feature = "langfuse", feature = "otlp")))] +fn not_compiled_in(name: &str) -> SkippedBackend { + warn!( + backend = name, + "instrumentation backend enabled in config but not compiled into this build" + ); + SkippedBackend { + name: name.to_string(), + reason: "not compiled into this build".to_string(), + } +} + +// These build real backends, so they need the exporters compiled in. `langfuse` +// implies `otlp`, so gating on it covers the default feature set; the reduced +// builds are covered by `reduced_build_tests` below. +#[cfg(all(test, feature = "langfuse"))] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use { @@ -677,3 +745,43 @@ mod tests { ); } } + +/// Coverage for builds that left an exporter out. +#[cfg(all(test, not(feature = "langfuse")))] +mod reduced_build_tests { + use {moltis_config::LangfuseSettings, secrecy::Secret}; + + use super::*; + + #[tokio::test] + async fn a_backend_this_build_lacks_is_skipped_with_a_reason() { + // Enabling a backend the binary cannot speak to must explain itself, + // otherwise it is indistinguishable from bad credentials. + let config = InstrumentationConfig { + enabled: true, + langfuse: LangfuseSettings { + enabled: true, + host: "https://cloud.langfuse.com".into(), + public_key: "pk-lf-1".into(), + secret_key: Some(Secret::new("sk-lf-1".to_string())), + ..Default::default() + }, + ..Default::default() + }; + + let outcome = build(&config, "test"); + + assert!(outcome.built.is_none()); + assert_eq!(outcome.skipped.len(), 1); + assert_eq!(outcome.skipped[0].name, "langfuse"); + assert_eq!(outcome.skipped[0].reason, "not compiled into this build"); + } + + #[tokio::test] + async fn an_untouched_config_still_builds_nothing() { + let outcome = build(&InstrumentationConfig::default(), "test"); + + assert!(outcome.built.is_none()); + assert!(outcome.skipped.is_empty()); + } +} diff --git a/crates/observability/tests/end_to_end.rs b/crates/observability/tests/end_to_end.rs index 3db45a345..b7d2f8d0f 100644 --- a/crates/observability/tests/end_to_end.rs +++ b/crates/observability/tests/end_to_end.rs @@ -1,4 +1,8 @@ #![allow(clippy::unwrap_used, clippy::expect_used)] +// Every case here drives the OTLP transport, which a build without that feature +// does not have. The Langfuse cases use the Langfuse *profile* over the same +// transport, so `otlp` is the only feature they need. +#![cfg(feature = "otlp")] //! End-to-end tests over the real export path. //! //! The unit tests check the mapping in isolation; these drive a recorder diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md index 002ac72fc..8feaf02cb 100644 --- a/docs/src/instrumentation.md +++ b/docs/src/instrumentation.md @@ -122,6 +122,25 @@ service = "moltis" To post to Datadog's intake directly instead, set `endpoint` to the regional OTLP intake URL and supply `api_key`. +## Build features + +Both exporter families are compiled in by default, so nothing here needs a +rebuild — only the config above switches them on. To leave one out of a build: + +```bash +# Langfuse only (drops the generic OTLP and Datadog exporters). +cargo build --no-default-features --features "web-ui,tls,langfuse" + +# No exporter at all. +cargo build --no-default-features --features "web-ui,tls" +``` + +`langfuse` implies `otlp`, because Langfuse ingests traces over the OTLP wire +format, and `otlp` also carries Datadog. A backend enabled in config that the +running binary was not built with is reported in **Settings → Instrumentation** +as skipped with the reason `not compiled into this build`, rather than failing +quietly. + ## Running several backends at once Backends are independent. Enabling all three sends full traces to Langfuse and From c38e1e23a8646bff65f04f70804694f16567dc90 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:04:11 -0700 Subject: [PATCH 30/33] docs(instrumentation): say the settings page is read-only, register feedback RPCs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two loose ends around the instrumentation UI. The Instrumentation settings page shows status and a connection test; it has no editable fields, and configuration lives in `moltis.toml` (or Settings → Configuration, which edits the same file) and applies on restart. Neither the docs nor the page said so, which left "configure Langfuse in Settings → Instrumentation" as an instruction nobody could follow. `rpc-methods.ts` documents itself as mapping every RPC method, but `feedback.status` and `feedback.submit` were missing while the `instrumentation.*` pair was added. `sendRpc` takes a plain string, so nothing caught it. Adds both and moves the instrumentation block into alphabetical order with its neighbours. --- .../pages/sections/InstrumentationSection.tsx | 6 +++--- crates/web/ui/src/types/rpc-methods.ts | 13 +++++++++---- docs/src/instrumentation.md | 19 +++++++++++++++++-- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx index 0c67fd728..bbfe1dbf9 100644 --- a/crates/web/ui/src/pages/sections/InstrumentationSection.tsx +++ b/crates/web/ui/src/pages/sections/InstrumentationSection.tsx @@ -335,9 +335,9 @@ export function InstrumentationSection(): VNode {

- Exports what each agent run did \u2014 LLM calls, tool calls, retrievals \u2014 to an external backend. - Configured in moltis.toml under [instrumentation]; this page shows what is running and - lets you verify connectivity. + Exports what each agent run did (LLM calls, tool calls, retrievals) to an external backend. Configured in{" "} + moltis.toml under [instrumentation], or in Settings then Configuration, and applied on + restart. This page is read-only: it shows what is running and lets you verify connectivity.

{config.enabled ? null : ( diff --git a/crates/web/ui/src/types/rpc-methods.ts b/crates/web/ui/src/types/rpc-methods.ts index ce883c58e..b6ac2b97e 100644 --- a/crates/web/ui/src/types/rpc-methods.ts +++ b/crates/web/ui/src/types/rpc-methods.ts @@ -47,10 +47,6 @@ export interface RpcMethodMap { "chat.send": unknown; "chat.send_sync": unknown; - // ── Instrumentation ───────────────────────────────────────── - "instrumentation.status": unknown; - "instrumentation.test": unknown; - // ── Cron ──────────────────────────────────────────────────── "cron.list": unknown; "cron.remove": unknown; @@ -64,6 +60,11 @@ export interface RpcMethodMap { // ── Exec ──────────────────────────────────────────────────── "exec.approval.resolve": unknown; + + // ── Feedback ──────────────────────────────────────────────── + "feedback.status": unknown; + "feedback.submit": unknown; + "external_agents.bind": unknown; "external_agents.list": ExternalAgentInfo[]; "external_agents.status": unknown; @@ -84,6 +85,10 @@ export interface RpcMethodMap { "hooks.reload": unknown; "hooks.save": unknown; + // ── Instrumentation ───────────────────────────────────────── + "instrumentation.status": unknown; + "instrumentation.test": unknown; + // ── Location ──────────────────────────────────────────────── "location.result": unknown; diff --git a/docs/src/instrumentation.md b/docs/src/instrumentation.md index 8feaf02cb..f66d6633a 100644 --- a/docs/src/instrumentation.md +++ b/docs/src/instrumentation.md @@ -85,8 +85,9 @@ public_key = "pk-lf-..." Keys come from your Langfuse project settings. For a self-hosted deployment, point `host` at your own instance — no data leaves your network. -Use **Settings → Instrumentation → Test connection** in the web UI to verify -the host is reachable and the credentials are accepted before relying on it. +Once Moltis has restarted, use **Settings → Instrumentation → Test connection** +in the web UI to verify the host is reachable and the credentials are accepted +before relying on it. ## Quick start: Grafana / OpenTelemetry Collector @@ -122,6 +123,20 @@ service = "moltis" To post to Datadog's intake directly instead, set `endpoint` to the regional OTLP intake URL and supply `api_key`. +## Where the settings live + +Instrumentation is configured in `moltis.toml` under `[instrumentation]`, either +by editing the file directly or through **Settings → Configuration** in the web +UI, which edits the same TOML. Either way the change takes effect on restart. + +**Settings → Instrumentation is a read-only view.** It reports what is actually +running — active backends, any that failed to start and why, delivery counters, +the capture policy in force, and whether credentials are present — and it can +test the Langfuse connection. It deliberately has no fields to edit: turning +instrumentation on sends conversation data to a third party, so that decision +lives in the config file next to the credential it needs, not behind a toggle in +a browser. + ## Build features Both exporter families are compiled in by default, so nothing here needs a From 138bd902415f3a95783b49e2fbeee0f5f02125a6 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:14:43 -0700 Subject: [PATCH 31/33] fix(scripts): stop local validation from passing without running tests `gh pr diff --name-only` answers HTTP 406 once a PR exceeds 20,000 diff lines, and it exits 0 while printing nothing. `changed_files()` took that empty output at face value, so `build_targeted_rust_test_cmd` and `build_targeted_e2e_cmd` found nothing to run and both contexts reported `passed` in about a second. The larger the PR, the less of it was tested. Observed on PR #1174: `[local/test] passed in 1s` with no test executed. Falls back to the local branch range when the PR diff is unavailable, and announces an empty targeted selection in the main log rather than only in the per-context log, which is discarded on success. A skip is now visible as a skip instead of looking like coverage. --- scripts/local-validate.sh | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index 80c718163..75f088d69 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -210,8 +210,17 @@ strip_all_features_flag() { changed_files() { if [[ "$LOCAL_ONLY" -eq 0 ]]; then - gh pr diff "$PR_NUMBER" --repo "$BASE_REPO" --name-only - return + # `gh pr diff` answers HTTP 406 once a PR exceeds 20,000 diff lines, and it + # exits 0 while printing nothing usable. Left alone that turns the targeted + # contexts into no-ops that still report "passed", so a large PR would be + # the least tested one. Fall back to the local range instead. + local pr_files="" + if pr_files="$(gh pr diff "$PR_NUMBER" --repo "$BASE_REPO" --name-only 2>/dev/null)" \ + && [[ -n "$pr_files" ]]; then + printf '%s\n' "$pr_files" + return + fi + echo "[changed-files] gh pr diff unavailable for #${PR_NUMBER}; using the local branch range." >&2 fi local base_ref="${LOCAL_VALIDATE_BASE_REF:-$BASE_REF_NAME}" @@ -290,6 +299,9 @@ build_targeted_rust_test_cmd() { done < <(changed_files) if [[ "${#commands[@]}" -eq 0 ]]; then + # Announced here, in the main log: the context's own output is captured and + # discarded on success, so a skip would otherwise look like a green run. + echo "[local/test] no changed Rust tests detected; nothing to run." >&2 printf '%s' 'echo "No changed Rust tests detected; skipping local/test."' return fi @@ -315,6 +327,7 @@ build_targeted_e2e_cmd() { done < <(changed_files) if [[ "${#specs[@]}" -eq 0 ]]; then + echo "[local/e2e] no changed Playwright specs detected; nothing to run." >&2 printf '%s' 'echo "No changed Playwright specs detected; skipping local/e2e."' return fi From 4082f4f6a2f3904c3e7c4bac5d02d30ab23d0347 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:40:20 -0700 Subject: [PATCH 32/33] fix(e2e): make the RPC retry actually retry, and wait for the socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `feedback.spec.js` "status RPC responds" failed under full-suite contention with "Service temporarily unavailable. Please try again." — the client's own message for a WebSocket that is not open yet. Two bugs behind one flake. `sendRpcFromPage` only retried errors whose text contained "WebSocket not connected" or "WebSocket disconnected", but `helpers.ts` localizes that error before returning it, so the predicate never matched and the retry was dead code. It now matches on the error code, which survives localization. The retry is only a safety net, though: `navigateAndWait` waits for the DOM to mount, not for the socket, so every RPC fired straight after navigation raced the connection. The RPC-driven cases in both instrumentation specs now await `waitForWsConnected` first. Verified with the four targeted specs together (86 passed) and with `--repeat-each=3` over the two instrumentation specs (45 passed). --- crates/web/ui/e2e/helpers.js | 11 +++++++++-- crates/web/ui/e2e/specs/feedback.spec.js | 7 ++++++- crates/web/ui/e2e/specs/instrumentation.spec.js | 4 +++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/crates/web/ui/e2e/helpers.js b/crates/web/ui/e2e/helpers.js index aa42c4ca6..2aaf43e4f 100644 --- a/crates/web/ui/e2e/helpers.js +++ b/crates/web/ui/e2e/helpers.js @@ -250,7 +250,14 @@ async function waitForChatSessionReady(page) { ); } -function isRetryableRpcError(message) { +function isRetryableRpcError(error) { + // The client localizes its own errors before returning them, so + // "WebSocket not connected" reaches us as "Service temporarily + // unavailable. Please try again." Matching the raw English here made this + // retry dead code, and the flake it exists to absorb surfaced as a failed + // assertion instead. The code survives localization, so match on that. + if (error?.code === "UNAVAILABLE") return true; + const message = error?.message; if (typeof message !== "string") return false; return message.includes("WebSocket not connected") || message.includes("WebSocket disconnected"); } @@ -298,7 +305,7 @@ async function sendRpcFromPage(page, method, params) { .catch((error) => ({ ok: false, error: { message: error?.message || String(error) } })); if (lastResponse?.ok) return lastResponse; - if (!isRetryableRpcError(lastResponse?.error?.message)) return lastResponse; + if (!isRetryableRpcError(lastResponse?.error)) return lastResponse; } console.log(`[sendRpc] ${method} FAILED after 3 attempts, last: ${lastResponse?.error?.message?.slice(0, 100)}`); return lastResponse; diff --git a/crates/web/ui/e2e/specs/feedback.spec.js b/crates/web/ui/e2e/specs/feedback.spec.js index a8af51913..11eef4b6b 100644 --- a/crates/web/ui/e2e/specs/feedback.spec.js +++ b/crates/web/ui/e2e/specs/feedback.spec.js @@ -1,9 +1,10 @@ const { expect, test } = require("../base-test"); -const { navigateAndWait, watchPageErrors, expectRpcOk, sendRpcFromPage } = require("../helpers"); +const { navigateAndWait, watchPageErrors, expectRpcOk, sendRpcFromPage, waitForWsConnected } = require("../helpers"); test.describe("Reaction feedback", () => { test("status RPC responds", async ({ page }) => { await navigateAndWait(page, "/"); + await waitForWsConnected(page); const response = await expectRpcOk(page, "feedback.status", {}); expect(typeof response.payload.enabled).toBe("boolean"); @@ -12,6 +13,7 @@ test.describe("Reaction feedback", () => { test("reports feedback as unavailable while instrumentation is off", async ({ page }) => { await navigateAndWait(page, "/"); + await waitForWsConnected(page); // A thumb that goes nowhere is worse than no thumb, so the control is // gated on something actually collecting the score. @@ -21,6 +23,7 @@ test.describe("Reaction feedback", () => { test("submitting without a session key is rejected", async ({ page }) => { await navigateAndWait(page, "/"); + await waitForWsConnected(page); const response = await sendRpcFromPage(page, "feedback.submit", { messageId: "run-1", @@ -32,6 +35,7 @@ test.describe("Reaction feedback", () => { test("submitting an unknown signal is rejected", async ({ page }) => { await navigateAndWait(page, "/"); + await waitForWsConnected(page); // The wire format must not be able to smuggle an arbitrary value into // the score. @@ -46,6 +50,7 @@ test.describe("Reaction feedback", () => { test("a thumb on an unlinked message reports why rather than failing silently", async ({ page }) => { await navigateAndWait(page, "/"); + await waitForWsConnected(page); const response = await sendRpcFromPage(page, "feedback.submit", { sessionKey: "agent:main:main", diff --git a/crates/web/ui/e2e/specs/instrumentation.spec.js b/crates/web/ui/e2e/specs/instrumentation.spec.js index ef05b4eef..584b9f6e3 100644 --- a/crates/web/ui/e2e/specs/instrumentation.spec.js +++ b/crates/web/ui/e2e/specs/instrumentation.spec.js @@ -1,5 +1,5 @@ const { expect, test } = require("../base-test"); -const { navigateAndWait, watchPageErrors, expectRpcOk } = require("../helpers"); +const { navigateAndWait, watchPageErrors, expectRpcOk, waitForWsConnected } = require("../helpers"); test.describe("Instrumentation settings", () => { test("page loads with the section heading", async ({ page }) => { @@ -47,6 +47,7 @@ test.describe("Instrumentation settings", () => { test("status RPC responds", async ({ page }) => { await navigateAndWait(page, "/settings/instrumentation"); + await waitForWsConnected(page); await expectRpcOk(page, "instrumentation.status", {}); }); @@ -59,6 +60,7 @@ test.describe("Instrumentation settings", () => { test("status RPC never returns the secret key", async ({ page }) => { await navigateAndWait(page, "/settings/instrumentation"); + await waitForWsConnected(page); const res = await expectRpcOk(page, "instrumentation.status", {}); // The UI only ever needs to know whether a key is configured. Returning From 12e3ca8e368a8c7186be53e14c9eccd8ccbc6007 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 17:31:23 -0700 Subject: [PATCH 33/33] style(gateway): keep post-state within file limit --- crates/gateway/src/server/prepare_core/post_state.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/crates/gateway/src/server/prepare_core/post_state.rs b/crates/gateway/src/server/prepare_core/post_state.rs index 952199ca4..afe634b5d 100644 --- a/crates/gateway/src/server/prepare_core/post_state.rs +++ b/crates/gateway/src/server/prepare_core/post_state.rs @@ -1,13 +1,10 @@ use { super::feedback::install_feedback, + secrecy::Secret, std::{ path::PathBuf, sync::{Arc, atomic::Ordering}, }, -}; - -use { - secrecy::Secret, tracing::{debug, info, warn}, };