From 041637348f0a34733e7f29ade82fe6b6f0da913b Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 10:08:43 -0700 Subject: [PATCH 01/23] feat(slack): per-turn reaction controller with phases, and fix premature terminal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `chat.send` spawns the agent run and returns immediately (fire-and-forget), so the previous ack βœ…/❌ β€” applied right after `send` returned in `dispatch_to_chat` β€” fired before the reply was actually delivered. This reworks acknowledgment reactions into a per-turn controller driven by the run lifecycle. - New `ChannelReactionController` (gateway): a serialized worker owning a single reaction slot on the inbound message. Adds πŸ‘€ on receipt, swaps to a phase emoji (🌐/πŸ’»/✏️/πŸ› οΈ/…) classified from the running tool, and finalizes to βœ…/❌ β€” or nothing on cancel. Phase changes are debounced (700ms) to avoid flicker; a stall marker (⏳) appears after prolonged inactivity; terminal transitions win over late phases. - Seam: `ChatRuntime::note_channel_activity(session_key, ChannelActivity)` (default no-op). The agent loop emits `Tool(name)` on each tool start (via the existing `send_tool_status_to_channels` choke point); the run completion emits `Finished(Success|Failure)`; the abort path emits `Finished(Cancelled)`. - Gateway owns a session-keyed controller registry (bounded by register-on-receive / remove-on-finish). `dispatch_to_chat` registers the controller (πŸ‘€) and only finalizes here for early returns where the run never ran (send error, or an Ok payload with a terminal state). - Cancelled turns now strip πŸ‘€ and leave no βœ…/❌ (matches hermes) via the new `ChannelAckOutcome` enum. - Glyphβ†’shortcode normalization (`crates/slack/src/emoji.rs`) applied to all outbound reactions so config/phase/model-specified names are Slack-safe (Slack silently drops raw glyphs and colon-wrapped names). Unit tests cover toolβ†’emoji classification, outcomeβ†’emoji mapping, and emoji normalization. Existing chat/channels/slack/gateway suites pass. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/channels/src/lib.rs | 14 +- crates/channels/src/plugin.rs | 28 ++ crates/chat/src/channels.rs | 8 + crates/chat/src/runtime.rs | 13 +- crates/chat/src/service/chat_impl.rs | 11 + crates/chat/src/service/chat_impl/send.rs | 19 ++ crates/gateway/src/channel_events.rs | 112 +++---- crates/gateway/src/channel_events/dispatch.rs | 21 +- crates/gateway/src/channel_reactions.rs | 300 ++++++++++++++++++ crates/gateway/src/chat.rs | 29 +- crates/gateway/src/lib.rs | 1 + crates/gateway/src/state.rs | 6 + crates/slack/src/emoji.rs | 118 +++++++ crates/slack/src/lib.rs | 1 + crates/slack/src/outbound.rs | 4 +- 15 files changed, 603 insertions(+), 82 deletions(-) create mode 100644 crates/gateway/src/channel_reactions.rs create mode 100644 crates/slack/src/emoji.rs diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index 46f16ba8ef..6b098e8401 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -27,13 +27,13 @@ pub use { error::{Error, Result}, media_download::{InboundMediaDownloader, InboundMediaSource}, plugin::{ - ButtonRow, ButtonStyle, ChannelAttachment, ChannelCapabilities, ChannelDescriptor, - ChannelDocumentFile, ChannelEvent, ChannelEventSink, ChannelHealthSnapshot, - ChannelMessageKind, ChannelMessageMeta, ChannelOtpProvider, ChannelOutbound, ChannelPlugin, - ChannelReplyTarget, ChannelStatus, ChannelStreamOutbound, ChannelThreadContext, - ChannelType, InboundMode, InteractiveButton, InteractiveMessage, SavedChannelFile, - StreamEvent, StreamReceiver, StreamSender, ThreadMessage, resolve_session_channel_binding, - web_session_channel_binding, + ButtonRow, ButtonStyle, ChannelAckOutcome, ChannelActivity, ChannelAttachment, + ChannelCapabilities, ChannelDescriptor, ChannelDocumentFile, ChannelEvent, + ChannelEventSink, ChannelHealthSnapshot, ChannelMessageKind, ChannelMessageMeta, + ChannelOtpProvider, ChannelOutbound, ChannelPlugin, ChannelReplyTarget, ChannelStatus, + ChannelStreamOutbound, ChannelThreadContext, ChannelType, InboundMode, InteractiveButton, + InteractiveMessage, SavedChannelFile, StreamEvent, StreamReceiver, StreamSender, + ThreadMessage, resolve_session_channel_binding, web_session_channel_binding, }, registry::{ChannelRegistry, RegistryOutboundRouter}, slack_api_url::{normalize_slack_api_base_url, validate_slack_api_base_url}, diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index e7212befcf..a8a5ebd70f 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -671,6 +671,34 @@ impl ChannelReplyTarget { } } +/// Terminal outcome of a channel-dispatched agent turn, used to finalize +/// acknowledgment reactions (βœ… / ❌ / none). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChannelAckOutcome { + /// The reply was delivered successfully. + Success, + /// The turn errored (timeout, provider failure, etc.). + Failure, + /// The turn was cancelled/aborted β€” leave no terminal marker. + Cancelled, +} + +/// A mid-turn activity signal emitted by the agent run, used to drive channel +/// acknowledgment reactions (phase emojis) and, later, live status text. +/// +/// The chat layer emits these; the gateway forwards them to a per-session +/// reaction controller. Kept intentionally small β€” richer phases are derived +/// from the tool name at the controller. +#[derive(Debug, Clone)] +pub enum ChannelActivity { + /// The model is reasoning/planning before or between tool calls. + Thinking, + /// A tool call started; carries the tool name for phase classification. + Tool(String), + /// The turn finished with the given terminal outcome. + Finished(ChannelAckOutcome), +} + impl From<&ChannelReplyTarget> for ChannelBinding { fn from(target: &ChannelReplyTarget) -> Self { let channel_type = target.channel_type.as_str().to_string(); diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 8a96719ebc..4cd7b464bf 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -851,6 +851,14 @@ pub(crate) async fn send_tool_status_to_channels( return; } + // Drive the acknowledgment reaction into its "tool" phase (πŸ› οΈ/🌐/πŸ’»/…). + state + .note_channel_activity( + session_key, + moltis_channels::ChannelActivity::Tool(tool_name.to_string()), + ) + .await; + // Buffer the status message for the logbook let message = format_tool_status_message(tool_name, arguments); state.push_channel_status_log(session_key, message).await; diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index cf430dceb1..f265d580fd 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -7,7 +7,10 @@ use std::sync::Arc; use serde_json::Value; -use {moltis_channels::ChannelReplyTarget, moltis_tools::sandbox::SandboxRouter}; +use { + moltis_channels::{ChannelActivity, ChannelReplyTarget}, + moltis_tools::sandbox::SandboxRouter, +}; /// TTS runtime override configuration (provider/voice/model). /// @@ -130,6 +133,14 @@ pub trait ChatRuntime: Send + Sync { /// Take (and remove) the last error for a run_id. async fn last_run_error(&self, run_id: &str) -> Option; + /// Report a mid-turn activity for a channel-dispatched session so the + /// runtime can drive acknowledgment reactions (phase emojis) and status. + /// + /// Default no-op: runtimes without channel reactions (tests, headless) + /// ignore it. Emitted from the agent loop (Thinking/Tool) and from the run + /// completion (Finished). + async fn note_channel_activity(&self, _session_key: &str, _activity: ChannelActivity) {} + // ── Push notifications ─────────────────────────────────────────────── /// Send a push notification to all subscribed devices. diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index 72d7c0a5ac..0706d8bab6 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -390,6 +390,17 @@ impl ChatService for LiveChatService { } } broadcast(&self.state, "chat", payload, BroadcastOpts::default()).await; + + // Finalize channel acknowledgment reactions: cancelled leaves no + // βœ…/❌ terminal, only strips the in-progress marker. + self.state + .note_channel_activity( + key, + moltis_channels::ChannelActivity::Finished( + moltis_channels::ChannelAckOutcome::Cancelled, + ), + ) + .await; } Ok(serde_json::json!({ diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index b8d33a0e34..d0cf5d792a 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -1271,6 +1271,25 @@ impl LiveChatService { agent_fut.await }; + // Signal the channel acknowledgment reactions that the turn finished. + // By this point the reply (if any) has already been delivered inside + // `agent_fut`, so a present assistant response means success. Aborted + // runs never reach here (the task is cancelled); the abort path emits + // `Cancelled` instead. + { + let outcome = if assistant_text.is_some() { + moltis_channels::ChannelAckOutcome::Success + } else { + moltis_channels::ChannelAckOutcome::Failure + }; + state + .note_channel_activity( + &session_key_clone, + moltis_channels::ChannelActivity::Finished(outcome), + ) + .await; + } + // Persist assistant response (even empty ones β€” needed for LLM history coherence). if let Some(assistant_output) = assistant_text { let assistant_msg = build_persisted_assistant_message( diff --git a/crates/gateway/src/channel_events.rs b/crates/gateway/src/channel_events.rs index cbfcf9a0a8..bd609ddf1c 100644 --- a/crates/gateway/src/channel_events.rs +++ b/crates/gateway/src/channel_events.rs @@ -8,8 +8,8 @@ use { use { moltis_channels::{ - ChannelAttachment, ChannelEvent, ChannelEventSink, ChannelMessageMeta, ChannelReplyTarget, - Error as ChannelError, Result as ChannelResult, SavedChannelFile, + ChannelAckOutcome, ChannelAttachment, ChannelEvent, ChannelEventSink, ChannelMessageMeta, + ChannelReplyTarget, Error as ChannelError, Result as ChannelResult, SavedChannelFile, }, moltis_sessions::metadata::{SessionEntry, SqliteSessionMetadata}, moltis_tools::approval::PendingApprovalView, @@ -17,6 +17,7 @@ use { use crate::{ broadcast::{BroadcastOpts, broadcast}, + channel_reactions::ChannelReactionController, state::GatewayState, }; @@ -336,83 +337,66 @@ fn start_channel_typing_loop( /// Emoji shortcodes for acknowledgment reactions (Slack-style names). Only /// channels that populate [`ChannelReplyTarget::ack_message_id`] receive these; /// today that is Slack, whose Web API expects shortcodes (not raw glyphs). -const ACK_RECEIVED_EMOJI: &str = "eyes"; -const ACK_SUCCESS_EMOJI: &str = "white_check_mark"; -const ACK_ERROR_EMOJI: &str = "x"; - -/// Add the "received" reaction (πŸ‘€) to the exact inbound message when the -/// channel asked for acknowledgment reactions (`ack_message_id` set). This -/// signals "I got your message and I'm working on it" before any reply text -/// arrives β€” the primary ack path for Slack, whose bots cannot show typing. +/// Create and register a per-turn acknowledgment reaction controller, keyed by +/// session key. The controller immediately adds πŸ‘€ to the exact inbound message +/// and is later driven through phase emojis and finalized by the agent run. /// -/// Best-effort: a missing outbound or a reaction API error is logged at debug -/// (`already_reacted` / `missing_scope` are expected) and never blocks the turn. -async fn channel_ack_received(state: &Arc, reply_to: &ChannelReplyTarget) { - let Some(message_id) = reply_to.ack_message_id.as_deref() else { +/// No-op unless the channel populated `ack_message_id` (bot directly addressed +/// and reactions enabled) and an outbound implementation is available. +async fn register_channel_reaction_controller( + state: &Arc, + session_key: &str, + reply_to: &ChannelReplyTarget, +) { + let Some(message_id) = reply_to.ack_message_id.clone() else { return; }; let Some(outbound) = state.services.channel_outbound_arc() else { return; }; - if let Err(e) = outbound - .add_reaction( - &reply_to.account_id, - &reply_to.chat_id, - message_id, - ACK_RECEIVED_EMOJI, - ) + let controller = ChannelReactionController::start( + outbound, + reply_to.account_id.clone(), + reply_to.chat_id.clone(), + message_id, + ); + state + .channel_reaction_controllers + .lock() .await - { - debug!( - account_id = %reply_to.account_id, - chat_id = %reply_to.chat_id, - "ack reaction (received) failed: {e}" - ); - } + .insert(session_key.to_string(), controller); } -/// Finalize acknowledgment reactions after the turn: remove πŸ‘€ and add βœ… on -/// success or ❌ on failure. Best-effort; mirrors [`channel_ack_received`]. -async fn channel_ack_finish( +/// Finalize the acknowledgment reaction only when `chat.send` returned before +/// the run executed β€” an error, or an `Ok` payload carrying a terminal state +/// (`rejected`/`error`/`blocked`/`aborted`). Normal runs (which return +/// `{ok, runId}` with no terminal state) finalize from the run's completion, so +/// this leaves their controller in place. +async fn finalize_reaction_on_early_return( state: &Arc, - reply_to: &ChannelReplyTarget, - success: bool, + session_key: &str, + send_result: &Result, ) { - let Some(message_id) = reply_to.ack_message_id.as_deref() else { - return; + let outcome = match send_result { + Err(_) => Some(ChannelAckOutcome::Failure), + Ok(payload) => match payload.get("state").and_then(|v| v.as_str()) { + Some("rejected" | "error" | "blocked") => Some(ChannelAckOutcome::Failure), + Some("aborted") => Some(ChannelAckOutcome::Cancelled), + _ => None, + }, }; - let Some(outbound) = state.services.channel_outbound_arc() else { + let Some(outcome) = outcome else { return; }; - if let Err(e) = outbound - .remove_reaction( - &reply_to.account_id, - &reply_to.chat_id, - message_id, - ACK_RECEIVED_EMOJI, - ) - .await - { - debug!( - account_id = %reply_to.account_id, - chat_id = %reply_to.chat_id, - "ack reaction (remove received) failed: {e}" - ); - } - let emoji = if success { - ACK_SUCCESS_EMOJI - } else { - ACK_ERROR_EMOJI - }; - if let Err(e) = outbound - .add_reaction(&reply_to.account_id, &reply_to.chat_id, message_id, emoji) + let controller = state + .channel_reaction_controllers + .lock() .await - { - debug!( - account_id = %reply_to.account_id, - chat_id = %reply_to.chat_id, - "ack reaction (finish) failed: {e}" - ); + .remove(session_key); + if let Some(controller) = controller { + controller + .note(moltis_channels::ChannelActivity::Finished(outcome)) + .await; } } diff --git a/crates/gateway/src/channel_events/dispatch.rs b/crates/gateway/src/channel_events/dispatch.rs index a984ebc6a3..bab7355bae 100644 --- a/crates/gateway/src/channel_events/dispatch.rs +++ b/crates/gateway/src/channel_events/dispatch.rs @@ -11,17 +11,20 @@ pub(in crate::channel_events) async fn dispatch_to_chat( // does not delay channel feedback. let typing_done = start_channel_typing_loop(state, &reply_to); - // Acknowledge receipt with a reaction (πŸ‘€) on the exact inbound message - // before any reply text arrives. No-op unless the channel populated - // `ack_message_id` (i.e. the bot was directly addressed and reactions - // are enabled). Finalized to βœ…/❌ after the turn completes below. - channel_ack_received(state, &reply_to).await; - let session_key = if let Some(ref sm) = state.services.session_metadata { resolve_channel_session(&reply_to, sm).await } else { default_channel_session_key(&reply_to) }; + + // Acknowledge receipt with a reaction (πŸ‘€) on the exact inbound message + // before any reply text arrives, and register a per-turn controller that + // the agent run drives through phase emojis and finalizes to βœ…/❌ (or + // nothing on cancel). No-op unless the channel populated `ack_message_id` + // (bot directly addressed and reactions enabled). Because the run is + // fire-and-forget, the terminal is applied from the run's completion β€” + // NOT from `chat.send` returning below. + register_channel_reaction_controller(state, &session_key, &reply_to).await; let effective_text = if state.is_channel_command_mode_enabled(&session_key).await { rewrite_for_shell_mode(text).unwrap_or_else(|| text.to_string()) } else { @@ -229,8 +232,10 @@ pub(in crate::channel_events) async fn dispatch_to_chat( let _ = done_tx.send(()); } - // Swap the πŸ‘€ acknowledgment for βœ… (success) or ❌ (failure). - channel_ack_finish(state, &reply_to, send_result.is_ok()).await; + // Only finalize the reaction here for early returns where the run never + // executes (rejection/error before spawn). Normal runs finalize from the + // run's completion via `note_channel_activity`. + finalize_reaction_on_early_return(state, &session_key, &send_result).await; if let Err(e) = send_result { error!("channel dispatch_to_chat failed: {e}"); diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs new file mode 100644 index 0000000000..eaf5ee9f76 --- /dev/null +++ b/crates/gateway/src/channel_reactions.rs @@ -0,0 +1,300 @@ +//! Per-turn channel acknowledgment reaction controller. +//! +//! A channel-dispatched agent turn is fire-and-forget: `chat.send` spawns the +//! run and returns immediately, so the βœ…/❌ terminal cannot be applied from the +//! dispatch call site. Instead, a controller is created when the message is +//! received (adds πŸ‘€), driven through phase emojis (πŸ› οΈ/🌐/πŸ’»/…) by the agent +//! loop, and finalized (βœ…/❌ or nothing on cancel) when the run completes. +//! +//! The controller owns a single reaction "slot" on the inbound message: each +//! transition removes the current emoji and adds the next, so at most one +//! reaction from the bot is visible at a time. All Slack API calls run on one +//! serialized worker task (no concurrent add/remove races), phase changes are +//! debounced to avoid flicker, and terminal transitions win over late phases. +//! +//! Design ported from openclaw's `status-reactions.ts`. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use { + moltis_channels::{ChannelAckOutcome, ChannelActivity, ChannelOutbound}, + tokio::sync::{Mutex, mpsc}, + tracing::debug, +}; + +/// Emoji shortcodes for acknowledgment/phase reactions (Slack-style names). +const RECEIVED_EMOJI: &str = "eyes"; +const SUCCESS_EMOJI: &str = "white_check_mark"; +const ERROR_EMOJI: &str = "x"; +const STALL_EMOJI: &str = "hourglass_flowing_sand"; + +/// Coalesce rapid phase changes so the reaction does not flicker. +const PHASE_DEBOUNCE: Duration = Duration::from_millis(700); +/// After this idle time with no activity, show a "still working" marker. +const STALL_AFTER: Duration = Duration::from_secs(20); + +/// Registry of active controllers, keyed by session key (runs serialize per +/// session via the send permit, so at most one is active per session). +pub type ReactionControllerRegistry = Arc>>>; + +/// Classify a tool name into a phase emoji shortcode. +/// +/// Mirrors openclaw's token lists so the single reaction slot communicates what +/// kind of work is happening. Unknown tools fall back to the generic tool emoji. +#[must_use] +pub fn classify_tool_emoji(tool_name: &str) -> &'static str { + let name = tool_name.to_ascii_lowercase(); + let has = |tokens: &[&str]| tokens.iter().any(|t| name.contains(*t)); + + if has(&[ + "web_search", + "search", + "web_fetch", + "fetch", + "browse", + "navigate", + "http", + "url", + ]) { + "globe_with_meridians" + } else if has(&["bash", "exec", "shell", "process", "command", "run"]) { + "computer" + } else if has(&[ + "edit", + "write", + "patch", + "apply", + "str_replace", + "create_file", + "code", + ]) { + "pencil2" + } else if has(&["deploy", "fly", "release", "publish"]) { + "airplane_departure" + } else if has(&["build", "compile", "cargo", "npm", "make"]) { + "building_construction" + } else { + "hammer_and_wrench" + } +} + +/// Terminal emoji for an outcome, or `None` when no marker should be left +/// (cancelled turns strip the in-progress reaction and add nothing). +#[must_use] +fn terminal_emoji(outcome: ChannelAckOutcome) -> Option<&'static str> { + match outcome { + ChannelAckOutcome::Success => Some(SUCCESS_EMOJI), + ChannelAckOutcome::Failure => Some(ERROR_EMOJI), + ChannelAckOutcome::Cancelled => None, + } +} + +/// Command sent to the controller's serialized worker. +#[derive(Debug)] +enum Command { + Phase(String), + Finish(ChannelAckOutcome), +} + +/// A per-turn reaction controller. Cheap handle around an mpsc sender to the +/// worker task that owns the reaction state machine. +pub struct ChannelReactionController { + tx: mpsc::Sender, +} + +impl ChannelReactionController { + /// Start a controller: spawns the worker, which immediately adds πŸ‘€ to the + /// target message. `message_id` is the exact inbound message ts. + #[must_use] + pub fn start( + outbound: Arc, + account_id: String, + chat_id: String, + message_id: String, + ) -> Arc { + let (tx, rx) = mpsc::channel(32); + tokio::spawn(run_worker(rx, outbound, account_id, chat_id, message_id)); + Arc::new(Self { tx }) + } + + /// Forward an activity signal from the agent run. + pub async fn note(&self, activity: ChannelActivity) { + let cmd = match activity { + ChannelActivity::Tool(name) => Command::Phase(classify_tool_emoji(&name).to_string()), + // Thinking maps back to the base received/working marker. + ChannelActivity::Thinking => Command::Phase(RECEIVED_EMOJI.to_string()), + ChannelActivity::Finished(outcome) => Command::Finish(outcome), + }; + // A closed channel means the worker already finalized; drop silently. + let _ = self.tx.send(cmd).await; + } +} + +/// The serialized reaction state machine. +async fn run_worker( + mut rx: mpsc::Receiver, + outbound: Arc, + account_id: String, + chat_id: String, + message_id: String, +) { + // Initial acknowledgment: πŸ‘€. + add_reaction( + &outbound, + &account_id, + &chat_id, + &message_id, + RECEIVED_EMOJI, + ) + .await; + let mut current: Option = Some(RECEIVED_EMOJI.to_string()); + let mut pending: Option = None; + let mut stalled = false; + + loop { + // Wait for the next command. If a phase change is pending, apply it once + // the debounce window elapses; otherwise fall back to the stall timer. + let wait = if pending.is_some() { + PHASE_DEBOUNCE + } else { + STALL_AFTER + }; + + match tokio::time::timeout(wait, rx.recv()).await { + Ok(Some(Command::Phase(emoji))) => { + // Coalesce: remember the latest requested phase, apply after debounce. + if current.as_deref() != Some(emoji.as_str()) { + pending = Some(emoji); + } + }, + Ok(Some(Command::Finish(outcome))) => { + if let Some(cur) = current.take() { + remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur).await; + } + if let Some(term) = terminal_emoji(outcome) { + add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; + } + return; + }, + Ok(None) => { + // Sender dropped without a terminal β€” leave the current marker. + return; + }, + Err(_) => { + // Timeout elapsed. + if let Some(emoji) = pending.take() { + swap( + &outbound, + &account_id, + &chat_id, + &message_id, + &mut current, + &emoji, + ) + .await; + stalled = false; + } else if !stalled { + // Idle too long: show a "still working" marker. + swap( + &outbound, + &account_id, + &chat_id, + &message_id, + &mut current, + STALL_EMOJI, + ) + .await; + stalled = true; + } + }, + } + } +} + +/// Remove the current emoji (if any) and add the new one, updating `current`. +async fn swap( + outbound: &Arc, + account_id: &str, + chat_id: &str, + message_id: &str, + current: &mut Option, + emoji: &str, +) { + if current.as_deref() == Some(emoji) { + return; + } + if let Some(cur) = current.take() { + remove_reaction(outbound, account_id, chat_id, message_id, &cur).await; + } + add_reaction(outbound, account_id, chat_id, message_id, emoji).await; + *current = Some(emoji.to_string()); +} + +async fn add_reaction( + outbound: &Arc, + account_id: &str, + chat_id: &str, + message_id: &str, + emoji: &str, +) { + if let Err(e) = outbound + .add_reaction(account_id, chat_id, message_id, emoji) + .await + { + debug!( + account_id, + chat_id, emoji, "channel add_reaction failed: {e}" + ); + } +} + +async fn remove_reaction( + outbound: &Arc, + account_id: &str, + chat_id: &str, + message_id: &str, + emoji: &str, +) { + if let Err(e) = outbound + .remove_reaction(account_id, chat_id, message_id, emoji) + .await + { + debug!( + account_id, + chat_id, emoji, "channel remove_reaction failed: {e}" + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn classifies_web_tools() { + assert_eq!(classify_tool_emoji("web_search"), "globe_with_meridians"); + assert_eq!(classify_tool_emoji("web_fetch"), "globe_with_meridians"); + } + + #[test] + fn classifies_shell_and_edit_tools() { + assert_eq!(classify_tool_emoji("exec"), "computer"); + assert_eq!(classify_tool_emoji("bash"), "computer"); + assert_eq!(classify_tool_emoji("str_replace_editor"), "pencil2"); + } + + #[test] + fn unknown_tool_uses_generic_emoji() { + assert_eq!(classify_tool_emoji("some_mcp_tool"), "hammer_and_wrench"); + } + + #[test] + fn terminal_emoji_maps_outcomes() { + assert_eq!( + terminal_emoji(ChannelAckOutcome::Success), + Some("white_check_mark") + ); + assert_eq!(terminal_emoji(ChannelAckOutcome::Failure), Some("x")); + assert_eq!(terminal_emoji(ChannelAckOutcome::Cancelled), None); + } +} diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index f27fad7cde..ac27862e85 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -7,7 +7,10 @@ use std::sync::Arc; use {async_trait::async_trait, serde_json::Value}; -use {moltis_channels::ChannelReplyTarget, moltis_tools::sandbox::SandboxRouter}; +use { + moltis_channels::{ChannelActivity, ChannelReplyTarget}, + moltis_tools::sandbox::SandboxRouter, +}; use crate::state::GatewayState; @@ -53,6 +56,30 @@ impl ChatRuntime for GatewayChatRuntime { self.state.peek_channel_replies(session_key).await } + // ── Channel acknowledgment reactions ──────────────────────────────────── + + async fn note_channel_activity(&self, session_key: &str, activity: ChannelActivity) { + // A terminal activity finalizes and removes the controller; phase + // activities just forward. Look up without holding the lock across the + // (awaiting) forward to avoid contention. + let is_terminal = matches!(activity, ChannelActivity::Finished(_)); + let controller = { + let map = self.state.channel_reaction_controllers.lock().await; + map.get(session_key).cloned() + }; + let Some(controller) = controller else { + return; + }; + controller.note(activity).await; + if is_terminal { + self.state + .channel_reaction_controllers + .lock() + .await + .remove(session_key); + } + } + // ── Channel status log ────────────────────────────────────────────────── async fn push_channel_status_log(&self, session_key: &str, message: String) { diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index dbefa210ab..9eba3dcede 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -18,6 +18,7 @@ pub mod broadcast; pub mod channel; pub mod channel_agent_tools; pub mod channel_events; +pub mod channel_reactions; pub mod channel_store; pub mod channel_webhook_dedup; pub mod channel_webhook_middleware; diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index 857f9f56a3..3b3211ab99 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -498,6 +498,11 @@ pub struct GatewayState { // ── Mutable runtime state (single lock) ───────────────────────────────── /// All mutable runtime state, behind a single lock. pub inner: RwLock, + + /// Active per-turn channel acknowledgment reaction controllers, keyed by + /// session key. Created when a channel message is received (adds πŸ‘€), + /// driven by the agent run, and removed when the turn finalizes. + pub channel_reaction_controllers: crate::channel_reactions::ReactionControllerRegistry, } impl GatewayState { @@ -601,6 +606,7 @@ impl GatewayState { broadcaster: Arc::new(Broadcaster::new()), client_registry: RwLock::new(ClientRegistryInner::new()), inner: RwLock::new(GatewayInner::new(hook_registry)), + channel_reaction_controllers: Arc::new(tokio::sync::Mutex::new(HashMap::new())), }) } diff --git a/crates/slack/src/emoji.rs b/crates/slack/src/emoji.rs new file mode 100644 index 0000000000..1b8e60e6d6 --- /dev/null +++ b/crates/slack/src/emoji.rs @@ -0,0 +1,118 @@ +//! Emoji name normalization for Slack reactions. +//! +//! Slack's `reactions.add` / `reactions.remove` expect emoji **shortcodes** +//! (e.g. `white_check_mark`), not raw Unicode glyphs (`βœ…`) and not +//! colon-wrapped forms (`:eyes:`). Slack silently drops names it does not +//! recognize, so any name that reaches the Web API β€” from a config override, a +//! phase-reaction map, or a future model-specified value β€” must be normalized +//! first. This mirrors the map openclaw's `actions.ts` maintains for the same +//! reason. + +use std::borrow::Cow; + +/// Normalize an emoji name into a Slack reaction shortcode. +/// +/// - Strips surrounding colons (`:eyes:` β†’ `eyes`). +/// - Drops a trailing skin-tone modifier (`thumbsup::skin-tone-3` β†’ `thumbsup`). +/// - Maps common Unicode glyphs to their shortcode (`βœ…` β†’ `white_check_mark`). +/// - Otherwise returns the input unchanged (already a shortcode). +/// +/// Returns a [`Cow`] so the common already-normalized case does not allocate. +#[must_use] +pub fn normalize_reaction_name(name: &str) -> Cow<'_, str> { + let trimmed = name.trim(); + + // Map a raw glyph first (before colon/skin-tone handling, which only apply + // to shortcode forms). + if let Some(shortcode) = glyph_to_shortcode(trimmed) { + return Cow::Borrowed(shortcode); + } + + // Strip surrounding colons and any `::skin-tone-N` suffix. + let without_colons = trimmed.trim_matches(':'); + let base = without_colons + .split_once("::") + .map_or(without_colons, |(head, _tail)| head); + + if base == name { + Cow::Borrowed(name) + } else { + Cow::Owned(base.to_string()) + } +} + +/// Map a raw Unicode emoji glyph to its Slack shortcode, if known. +/// +/// Covers the glyphs Moltis emits for acknowledgment and phase reactions plus a +/// handful of frequently-used ones. Unknown glyphs fall through to the caller. +fn glyph_to_shortcode(glyph: &str) -> Option<&'static str> { + let shortcode = match glyph { + "βœ…" | "βœ”οΈ" | "βœ”" => "white_check_mark", + "❌" | "βœ–οΈ" => "x", + "πŸ‘€" => "eyes", + "⏳" => "hourglass_flowing_sand", + "βŒ›" => "hourglass", + "⚠️" | "⚠" => "warning", + "🧠" => "brain", + "πŸ› οΈ" | "πŸ› " => "hammer_and_wrench", + "🌐" => "globe_with_meridians", + "πŸ’»" => "computer", + "πŸ—οΈ" | "πŸ—" => "building_construction", + "πŸ›«" => "airplane_departure", + "πŸ—œοΈ" | "πŸ—œ" => "clamp", + "πŸ‘" => "thumbsup", + "πŸ‘Ž" => "thumbsdown", + "πŸŽ‰" => "tada", + "πŸ”₯" => "fire", + "πŸ‘‹" => "wave", + "πŸš€" => "rocket", + _ => return None, + }; + Some(shortcode) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn passes_through_plain_shortcode() { + assert_eq!( + normalize_reaction_name("white_check_mark"), + "white_check_mark" + ); + } + + #[test] + fn strips_surrounding_colons() { + assert_eq!(normalize_reaction_name(":eyes:"), "eyes"); + } + + #[test] + fn maps_common_glyphs() { + assert_eq!(normalize_reaction_name("βœ…"), "white_check_mark"); + assert_eq!(normalize_reaction_name("❌"), "x"); + assert_eq!(normalize_reaction_name("πŸ‘€"), "eyes"); + assert_eq!(normalize_reaction_name("🧠"), "brain"); + } + + #[test] + fn strips_skin_tone_modifier() { + assert_eq!(normalize_reaction_name("thumbsup::skin-tone-3"), "thumbsup"); + assert_eq!(normalize_reaction_name(":wave::skin-tone-2:"), "wave"); + } + + #[test] + fn trims_whitespace() { + assert_eq!(normalize_reaction_name(" eyes "), "eyes"); + } + + #[test] + fn unknown_glyph_passes_through() { + // Not in the map β€” returned unchanged for Slack to accept or reject. + assert_eq!( + normalize_reaction_name("some_custom_emoji"), + "some_custom_emoji" + ); + } +} diff --git a/crates/slack/src/lib.rs b/crates/slack/src/lib.rs index 4b26f4dfb1..0481da9035 100644 --- a/crates/slack/src/lib.rs +++ b/crates/slack/src/lib.rs @@ -8,6 +8,7 @@ pub mod channel_webhook_verifier; pub mod client; pub mod commands; pub mod config; +pub mod emoji; pub mod markdown; pub mod outbound; pub mod plugin; diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 8e4726fbcb..9185b76696 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -437,7 +437,9 @@ async fn modify_reaction( let session = client.open_session(token); let channel_id: SlackChannelId = channel.into(); let ts: SlackTs = timestamp.into(); - let reaction = SlackReactionName::new(emoji.to_string()); + // Slack expects shortcodes (no colons, no raw glyphs, no skin-tone suffix). + let reaction = + SlackReactionName::new(crate::emoji::normalize_reaction_name(emoji).into_owned()); if add { let req = SlackApiReactionsAddRequest::new(channel_id, reaction, ts); From 95eaea7a046e108bbed2761d818d270de9f628af Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 10:21:01 -0700 Subject: [PATCH 02/23] feat(slack): reconnect supervision, Block Kit rendering, and assistant status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reconnect resilience: `run_socket_listener` now supervises the Socket Mode connection. slack-morphism's `serve()` returns on disconnect; previously that silently killed inbound delivery. It now rebuilds the listener and retries with exponential backoff + jitter (1sβ†’30s cap), resetting after a healthy connection, with cancellation always winning. - Rich Block Kit rendering (opt-in `rich_blocks`): replies render block-level markdown (headings, dividers, fenced code, sections) as Block Kit, enforcing Slack's limits and always falling back to plain chunked text when content can't fit β€” a rich render never loses a message. - Live status (opt-in `assistant_status`): `send_typing` now calls `assistant.threads.setStatus` ("is thinking…") in assistant threads. Off by default and best-effort, since it only applies to AI/Assistant apps. Adds config fields (+ redaction), edit-modal UI toggles, docs, config template, and unit tests for the Block Kit renderer and reconnect backoff. E2E extended to cover the new toggles. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/config/src/template.rs | 4 +- crates/slack/src/blocks.rs | 238 ++++++++++++++++++ crates/slack/src/config.rs | 21 +- crates/slack/src/lib.rs | 1 + crates/slack/src/outbound.rs | 121 ++++++++- crates/slack/src/socket.rs | 150 +++++++++-- .../web/ui/e2e/specs/channels-slack.spec.js | 11 +- crates/web/ui/src/pages/ChannelsPage.tsx | 2 + .../channels/modals/EditChannelModal.tsx | 38 +++ docs/src/slack.md | 7 +- 10 files changed, 555 insertions(+), 38 deletions(-) create mode 100644 crates/slack/src/blocks.rs diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index 93e3f46ec7..8e6bf7f74c 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -816,8 +816,10 @@ port = {port} # Port number (auto-generated for this i # dm_policy = "allowlist" # allowlist = [] # thread_replies = true -# ack_reactions = true # πŸ‘€ on receipt, βœ…/❌ on completion (DMs + @mentions) +# ack_reactions = true # πŸ‘€ on receipt, phase emoji while working, βœ…/❌ on completion # reaction_triggers = false # route user reactions into the agent (react βœ… to approve) +# rich_blocks = false # render replies as Block Kit blocks (fallback to plain text) +# assistant_status = false # live "is thinking…" status (needs an AI/Assistant app) # otp_self_approval = true # otp_cooldown_secs = 300 diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs new file mode 100644 index 0000000000..3c42daf517 --- /dev/null +++ b/crates/slack/src/blocks.rs @@ -0,0 +1,238 @@ +//! Markdown β†’ Slack Block Kit rendering (opt-in via `rich_blocks`). +//! +//! Renders an assistant reply's block-level structure (headings, dividers, +//! fenced code, paragraphs) into Block Kit blocks so long replies read better +//! than one flat `mrkdwn` blob. Inline formatting (bold/italic/links/inline +//! code) is left to Slack's `mrkdwn`, produced by [`markdown_to_slack`]. +//! +//! Hard rule (from hermes): a rich render must never lose a message. If the +//! content cannot be represented within Slack's limits, [`markdown_to_blocks`] +//! returns `None` and the caller falls back to plain chunked text. + +use serde_json::{Value, json}; + +use crate::markdown::markdown_to_slack; + +/// Slack Block Kit limits. +const MAX_BLOCKS: usize = 50; +const MAX_SECTION_CHARS: usize = 3000; +const MAX_HEADER_CHARS: usize = 150; + +/// Convert markdown to a Block Kit block array, or `None` if it should fall back +/// to plain text (empty, or too large to represent within Slack's limits). +#[must_use] +pub fn markdown_to_blocks(markdown: &str) -> Option> { + let mut blocks: Vec = Vec::new(); + let mut paragraph = String::new(); + let mut code: Option = None; + + let flush_paragraph = |paragraph: &mut String, blocks: &mut Vec| { + let trimmed = paragraph.trim(); + if !trimmed.is_empty() { + for chunk in split_section(&markdown_to_slack(trimmed)) { + blocks.push(section_block(&chunk)); + } + } + paragraph.clear(); + }; + + for line in markdown.lines() { + // Fenced code block toggling. + if line.trim_start().starts_with("```") { + match code.take() { + Some(buf) => { + // Closing fence: emit the accumulated code as a section. + let fenced = format!("```\n{}\n```", buf.trim_end_matches('\n')); + for chunk in split_section(&fenced) { + blocks.push(section_block(&chunk)); + } + }, + None => { + // Opening fence: flush any pending paragraph first. + flush_paragraph(&mut paragraph, &mut blocks); + code = Some(String::new()); + }, + } + continue; + } + + if let Some(buf) = code.as_mut() { + buf.push_str(line); + buf.push('\n'); + continue; + } + + let trimmed = line.trim(); + + // Thematic break β†’ divider. + if matches!(trimmed, "---" | "***" | "___") { + flush_paragraph(&mut paragraph, &mut blocks); + blocks.push(json!({ "type": "divider" })); + continue; + } + + // ATX heading β†’ header block. + if let Some(text) = heading_text(trimmed) { + flush_paragraph(&mut paragraph, &mut blocks); + blocks.push(header_block(&text)); + continue; + } + + // Blank line β†’ paragraph boundary. + if trimmed.is_empty() { + flush_paragraph(&mut paragraph, &mut blocks); + continue; + } + + if !paragraph.is_empty() { + paragraph.push('\n'); + } + paragraph.push_str(line); + } + + // Flush trailing state. + if let Some(buf) = code.take() { + // Unterminated fence: still emit what we have. + let fenced = format!("```\n{}\n```", buf.trim_end_matches('\n')); + for chunk in split_section(&fenced) { + blocks.push(section_block(&chunk)); + } + } + flush_paragraph(&mut paragraph, &mut blocks); + + // Never lose a message: bail to plain text on empty or over-limit output. + if blocks.is_empty() || blocks.len() > MAX_BLOCKS { + return None; + } + Some(blocks) +} + +/// Extract heading text from an ATX heading line (`# …` … `###### …`). +fn heading_text(line: &str) -> Option { + let hashes = line.chars().take_while(|c| *c == '#').count(); + if (1..=6).contains(&hashes) && line[hashes..].starts_with(' ') { + Some(line[hashes..].trim().to_string()) + } else { + None + } +} + +fn header_block(text: &str) -> Value { + let truncated = truncate_chars(text, MAX_HEADER_CHARS); + json!({ + "type": "header", + "text": { "type": "plain_text", "text": truncated, "emoji": true }, + }) +} + +fn section_block(mrkdwn: &str) -> Value { + json!({ + "type": "section", + "text": { "type": "mrkdwn", "text": mrkdwn }, + }) +} + +/// Split a section body into pieces no longer than [`MAX_SECTION_CHARS`], +/// preferring line boundaries. +fn split_section(text: &str) -> Vec { + if text.chars().count() <= MAX_SECTION_CHARS { + return vec![text.to_string()]; + } + let mut out = Vec::new(); + let mut current = String::new(); + for line in text.split_inclusive('\n') { + if current.chars().count() + line.chars().count() > MAX_SECTION_CHARS && !current.is_empty() + { + out.push(std::mem::take(&mut current)); + } + // A single line longer than the limit is hard-split by char boundary. + if line.chars().count() > MAX_SECTION_CHARS { + let mut buf = line; + while buf.chars().count() > MAX_SECTION_CHARS { + let cut = buf + .char_indices() + .nth(MAX_SECTION_CHARS) + .map_or(buf.len(), |(i, _)| i); + out.push(buf[..cut].to_string()); + buf = &buf[cut..]; + } + current.push_str(buf); + } else { + current.push_str(line); + } + } + if !current.is_empty() { + out.push(current); + } + out +} + +fn truncate_chars(text: &str, max: usize) -> String { + if text.chars().count() <= max { + return text.to_string(); + } + let cut = text + .char_indices() + .nth(max.saturating_sub(1)) + .map_or(text.len(), |(i, _)| i); + format!("{}…", &text[..cut]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_returns_none() { + assert!(markdown_to_blocks(" \n ").is_none()); + } + + #[test] + fn heading_becomes_header_block() { + let blocks = markdown_to_blocks("# Title\n\nbody text").unwrap(); + assert_eq!(blocks[0]["type"], "header"); + assert_eq!(blocks[0]["text"]["text"], "Title"); + assert_eq!(blocks[1]["type"], "section"); + } + + #[test] + fn thematic_break_becomes_divider() { + let blocks = markdown_to_blocks("a\n\n---\n\nb").unwrap(); + assert!(blocks.iter().any(|b| b["type"] == "divider")); + } + + #[test] + fn fenced_code_is_preserved() { + let blocks = markdown_to_blocks("intro\n\n```\nlet x = 1;\n```").unwrap(); + let has_code = blocks.iter().any(|b| { + b["text"]["text"] + .as_str() + .is_some_and(|t| t.contains("```")) + }); + assert!(has_code); + } + + #[test] + fn non_heading_hashtag_is_not_a_header() { + // `#nospace` is not an ATX heading. + let blocks = markdown_to_blocks("#notaheading").unwrap(); + assert_eq!(blocks[0]["type"], "section"); + } + + #[test] + fn over_limit_falls_back_to_none() { + // Many dividers exceed the 50-block cap β†’ fall back to plain text. + let md = (0..60).map(|_| "---").collect::>().join("\n\n"); + assert!(markdown_to_blocks(&md).is_none()); + } + + #[test] + fn long_section_is_split() { + let long = "x".repeat(7000); + let blocks = markdown_to_blocks(&long).unwrap(); + assert!(blocks.len() >= 2); + for b in &blocks { + assert!(b["text"]["text"].as_str().unwrap().chars().count() <= MAX_SECTION_CHARS); + } + } +} diff --git a/crates/slack/src/config.rs b/crates/slack/src/config.rs index 1f14be250d..bd35e3ef36 100644 --- a/crates/slack/src/config.rs +++ b/crates/slack/src/config.rs @@ -120,6 +120,17 @@ pub struct SlackAccountConfig { /// Reply in threads (default: true). pub thread_replies: bool, + /// Render replies as Block Kit blocks (headings, dividers, code, sections) + /// instead of a single flat mrkdwn message. Falls back to plain text when + /// content cannot be represented within Slack's limits. Default: false. + pub rich_blocks: bool, + + /// Show a live "is thinking…" status via `assistant.threads.setStatus` + /// while a turn runs. Only works when the Slack app is configured as an + /// AI/Assistant app and the message is in an assistant thread; a no-op + /// otherwise. Default: false. + pub assistant_status: bool, + /// Acknowledge inbound messages with emoji reactions (πŸ‘€ on receipt, βœ… on /// success, ❌ on error). Only applied when the bot is directly addressed /// (DM or @mention). Default: true. @@ -173,6 +184,8 @@ impl std::fmt::Debug for SlackAccountConfig { .field("stream_mode", &self.stream_mode) .field("edit_throttle_ms", &self.edit_throttle_ms) .field("thread_replies", &self.thread_replies) + .field("rich_blocks", &self.rich_blocks) + .field("assistant_status", &self.assistant_status) .field("ack_reactions", &self.ack_reactions) .field("reaction_triggers", &self.reaction_triggers) .field("reaction_trigger_emojis", &self.reaction_trigger_emojis) @@ -203,6 +216,8 @@ impl Default for SlackAccountConfig { stream_mode: StreamMode::EditInPlace, edit_throttle_ms: 500, thread_replies: true, + rich_blocks: false, + assistant_status: false, ack_reactions: true, reaction_triggers: false, reaction_trigger_emojis: Vec::new(), @@ -274,7 +289,7 @@ pub struct RedactedConfig<'a>(pub &'a SlackAccountConfig); impl Serialize for RedactedConfig<'_> { fn serialize(&self, serializer: S) -> Result { let c = self.0; - let mut count = 16; // always-present fields + let mut count = 18; // always-present fields count += c.signing_secret.is_some() as usize; count += !c.reaction_trigger_emojis.is_empty() as usize; count += c.model.is_some() as usize; @@ -307,6 +322,8 @@ impl Serialize for RedactedConfig<'_> { s.serialize_field("stream_mode", &c.stream_mode)?; s.serialize_field("edit_throttle_ms", &c.edit_throttle_ms)?; s.serialize_field("thread_replies", &c.thread_replies)?; + s.serialize_field("rich_blocks", &c.rich_blocks)?; + s.serialize_field("assistant_status", &c.assistant_status)?; s.serialize_field("ack_reactions", &c.ack_reactions)?; s.serialize_field("reaction_triggers", &c.reaction_triggers)?; if !c.reaction_trigger_emojis.is_empty() { @@ -428,6 +445,8 @@ mod tests { assert_eq!(cfg.edit_throttle_ms, 500); assert!(cfg.thread_replies); assert!(cfg.ack_reactions); + assert!(!cfg.rich_blocks); + assert!(!cfg.assistant_status); assert!(cfg.otp_self_approval); assert_eq!(cfg.otp_cooldown_secs, 300); assert_eq!(cfg.mention_mode, MentionMode::Mention); diff --git a/crates/slack/src/lib.rs b/crates/slack/src/lib.rs index 0481da9035..40453c0f66 100644 --- a/crates/slack/src/lib.rs +++ b/crates/slack/src/lib.rs @@ -4,6 +4,7 @@ //! Handles inbound DMs and channel messages, applies access control //! policies, and dispatches messages to the chat session. +pub mod blocks; pub mod channel_webhook_verifier; pub mod client; pub mod commands; diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 9185b76696..c07868cd8e 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -73,6 +73,15 @@ impl SlackOutbound { .map(|(_, ts)| ts.clone()) } + /// Whether Block Kit rich rendering is enabled for the given account. + fn get_rich_blocks(&self, account_id: &str) -> bool { + let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); + accounts + .get(account_id) + .map(|s| s.config.rich_blocks) + .unwrap_or(false) + } + /// Get the stream mode for the given account. fn get_stream_mode(&self, account_id: &str) -> StreamMode { let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); @@ -318,6 +327,44 @@ async fn post_message( Ok(resp.ts) } +/// Post a message rendered as Block Kit blocks, with `fallback_text` used for +/// notifications and clients that cannot render blocks. +async fn post_message_with_blocks( + client: &SlackClient, + token: &SlackApiToken, + channel: &str, + fallback_text: &str, + blocks: &[serde_json::Value], + thread_ts: Option<&str>, +) -> ChannelResult<()> { + let session = client.open_session(token); + let mut body = serde_json::json!({ + "channel": channel, + "text": fallback_text, + "blocks": blocks, + }); + if let Some(ts) = thread_ts { + body["thread_ts"] = serde_json::json!(ts); + } + + let resp: serde_json::Value = session + .http_session_api + .http_post("chat.postMessage", &body, None) + .await + .map_err(|e| ChannelError::unavailable(format!("chat.postMessage (blocks) failed: {e}")))?; + + if resp.get("ok") == Some(&serde_json::Value::Bool(false)) { + let err = resp + .get("error") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + return Err(ChannelError::unavailable(format!( + "chat.postMessage (blocks) error: {err}" + ))); + } + Ok(()) +} + /// Update an existing message. async fn update_message( client: &SlackClient, @@ -473,9 +520,25 @@ impl ChannelOutbound for SlackOutbound { let thread_ts = self.get_thread_ts(account_id, to, reply_to); let slack_text = markdown_to_slack(text); - let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); - for chunk in chunks { - post_message(&client, &token, to, chunk, thread_ts.as_deref()).await?; + // Rich Block Kit rendering (opt-in). Falls back to plain chunked text + // when disabled or when the content can't be represented as blocks. + let rendered_blocks = self + .get_rich_blocks(account_id) + .then(|| crate::blocks::markdown_to_blocks(text)) + .flatten(); + + if let Some(blocks) = rendered_blocks { + let fallback = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN) + .first() + .copied() + .unwrap_or(&slack_text); + post_message_with_blocks(&client, &token, to, fallback, &blocks, thread_ts.as_deref()) + .await?; + } else { + let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); + for chunk in chunks { + post_message(&client, &token, to, chunk, thread_ts.as_deref()).await?; + } } #[cfg(feature = "metrics")] @@ -550,8 +613,56 @@ impl ChannelOutbound for SlackOutbound { } } - async fn send_typing(&self, _account_id: &str, _to: &str) -> ChannelResult<()> { - // Slack bots cannot show typing indicators. + async fn send_typing(&self, account_id: &str, to: &str) -> ChannelResult<()> { + // Slack bots have no classic typing indicator. When the account opts in + // and the message is in an assistant thread, surface a live status via + // `assistant.threads.setStatus` instead. Best-effort: any error (e.g. + // the app is not an AI/Assistant app) is swallowed at debug. + let params = { + let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); + let Some(state) = accounts.get(account_id) else { + return Ok(()); + }; + if !state.config.assistant_status { + return Ok(()); + } + // Resolve the thread ts for this channel (assistant threads only). + let thread_ts = state + .pending_threads + .iter() + .find(|(k, _)| k.starts_with(&format!("{to}:"))) + .map(|(_, ts)| ts.clone()); + thread_ts.map(|ts| { + ( + state.config.bot_token.expose_secret().clone(), + state.config.api_base_url.clone(), + ts, + ) + }) + }; + let Some((bot_token, api_base_url, thread_ts)) = params else { + return Ok(()); + }; + + let http = moltis_common::http_client::build_default_http_client(); + let body = serde_json::json!({ + "channel_id": to, + "thread_ts": thread_ts, + "status": "is thinking…", + }); + match http + .post(slack_api_method_url( + &api_base_url, + "assistant.threads.setStatus", + )?) + .bearer_auth(&bot_token) + .json(&body) + .send() + .await + { + Ok(_) => {}, + Err(e) => debug!(account_id, to, "assistant.threads.setStatus failed: {e}"), + } Ok(()) } diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index ba8b535679..116e69dd66 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -119,7 +119,21 @@ pub async fn start_socket_mode( Ok(()) } -/// Run the Socket Mode listener until cancelled. +/// Initial reconnect backoff after an unexpected socket disconnect. +const RECONNECT_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_secs(1); +/// Maximum reconnect backoff. +const RECONNECT_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_secs(30); +/// A connection that stayed up at least this long is considered healthy, so the +/// backoff resets to its initial value on the next disconnect. +const RECONNECT_STABLE_AFTER: std::time::Duration = std::time::Duration::from_secs(60); + +/// Run the Socket Mode listener until cancelled, reconnecting on disconnect. +/// +/// slack-morphism's `serve()` returns when the socket drops; on its own that +/// silently kills inbound delivery. This supervises the connection: on any +/// unexpected exit (or a failed `listen_for`) it rebuilds the listener and +/// retries with exponential backoff + jitter, resetting the backoff after a +/// healthy connection. Cancellation always wins the race and shuts down cleanly. async fn run_socket_listener( account_id: &str, client: Arc>, @@ -127,44 +141,105 @@ async fn run_socket_listener( accounts: AccountStateMap, cancel: tokio_util::sync::CancellationToken, ) -> Result<(), Box> { - let listener_state = ListenerState { - account_id: account_id.to_string(), - accounts, - }; + let mut backoff = RECONNECT_INITIAL_BACKOFF; - // Socket Mode callbacks β€” must be function pointers, so we use user state. - let callbacks = SlackSocketModeListenerCallbacks::new() - .with_push_events(push_events_callback) - .with_command_events(command_events_callback) - .with_interaction_events(interaction_events_callback); + while !cancel.is_cancelled() { + let listener_state = ListenerState { + account_id: account_id.to_string(), + accounts: accounts.clone(), + }; - let listener_environment = Arc::new( - SlackClientEventsListenerEnvironment::new(Arc::clone(&client)) - .with_error_handler(error_handler) - .with_user_state(listener_state), - ); + // Callbacks must be function pointers, so per-account data rides on the + // listener environment's user state. + let callbacks = SlackSocketModeListenerCallbacks::new() + .with_push_events(push_events_callback) + .with_command_events(command_events_callback) + .with_interaction_events(interaction_events_callback); + + let listener_environment = Arc::new( + SlackClientEventsListenerEnvironment::new(Arc::clone(&client)) + .with_error_handler(error_handler) + .with_user_state(listener_state), + ); - let config = SlackClientSocketModeConfig::new(); - let socket_listener = - SlackClientSocketModeListener::new(&config, listener_environment, callbacks); + let config = SlackClientSocketModeConfig::new(); + let socket_listener = + SlackClientSocketModeListener::new(&config, listener_environment, callbacks); - socket_listener.listen_for(&app_token).await?; + if let Err(e) = socket_listener.listen_for(&app_token).await { + warn!(account_id, "slack socket mode connect failed: {e}"); + if !backoff_sleep(&cancel, &mut backoff).await { + break; + } + continue; + } - info!(account_id, "slack socket mode listener started"); + info!(account_id, "slack socket mode listener started"); + let connected_at = tokio::time::Instant::now(); - tokio::select! { - () = cancel.cancelled() => { - info!(account_id, "slack socket mode shutting down"); - socket_listener.shutdown().await; - } - _code = socket_listener.serve() => { - warn!(account_id, "slack socket mode listener unexpectedly stopped"); + tokio::select! { + () = cancel.cancelled() => { + info!(account_id, "slack socket mode shutting down"); + socket_listener.shutdown().await; + break; + } + _code = socket_listener.serve() => { + socket_listener.shutdown().await; + // A connection that lasted a while was healthy; reset backoff. + if connected_at.elapsed() >= RECONNECT_STABLE_AFTER { + backoff = RECONNECT_INITIAL_BACKOFF; + } + warn!( + account_id, + backoff_secs = backoff.as_secs(), + "slack socket mode disconnected; reconnecting" + ); + if !backoff_sleep(&cancel, &mut backoff).await { + break; + } + } } } Ok(()) } +/// Sleep for the current backoff (with jitter), then double it up to the cap. +/// +/// Returns `false` if cancellation fired during the sleep (caller should stop). +async fn backoff_sleep( + cancel: &tokio_util::sync::CancellationToken, + backoff: &mut std::time::Duration, +) -> bool { + let delay = jittered(*backoff); + tokio::select! { + () = cancel.cancelled() => return false, + () = tokio::time::sleep(delay) => {}, + } + *backoff = next_backoff(*backoff); + true +} + +/// Double the backoff, capped at [`RECONNECT_MAX_BACKOFF`]. +fn next_backoff(current: std::time::Duration) -> std::time::Duration { + (current * 2).min(RECONNECT_MAX_BACKOFF) +} + +/// Apply +/-25% jitter to a backoff duration to avoid reconnect thundering herds. +/// +/// Uses the wall-clock nanosecond fraction as a cheap entropy source (no `rand` +/// dependency); jitter quality is not security-relevant here. +fn jittered(base: std::time::Duration) -> std::time::Duration { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos()) + .unwrap_or(0); + // Map nanos into [-0.25, +0.25]. + let frac = (f64::from(nanos) / f64::from(u32::MAX)) - 0.5; + let factor = 1.0 + frac * 0.5; + base.mul_f64(factor.clamp(0.5, 1.5)) +} + /// Error handler for Socket Mode. fn error_handler( err: Box, @@ -1021,6 +1096,27 @@ async fn send_otp_status( mod tests { use super::*; + #[test] + fn backoff_doubles_and_caps() { + use std::time::Duration; + assert_eq!(next_backoff(Duration::from_secs(1)), Duration::from_secs(2)); + assert_eq!(next_backoff(Duration::from_secs(2)), Duration::from_secs(4)); + // Caps at RECONNECT_MAX_BACKOFF (30s). + assert_eq!(next_backoff(Duration::from_secs(20)), RECONNECT_MAX_BACKOFF); + assert_eq!(next_backoff(RECONNECT_MAX_BACKOFF), RECONNECT_MAX_BACKOFF); + } + + #[test] + fn jitter_stays_within_bounds() { + use std::time::Duration; + let base = Duration::from_secs(10); + for _ in 0..100 { + let j = jittered(base); + assert!(j >= base.mul_f64(0.5), "jitter {j:?} below lower bound"); + assert!(j <= base.mul_f64(1.5), "jitter {j:?} above upper bound"); + } + } + #[test] fn ack_target_dm_is_acknowledged() { assert_eq!( diff --git a/crates/web/ui/e2e/specs/channels-slack.spec.js b/crates/web/ui/e2e/specs/channels-slack.spec.js index 0d0accece4..dd4df8bff2 100644 --- a/crates/web/ui/e2e/specs/channels-slack.spec.js +++ b/crates/web/ui/e2e/specs/channels-slack.spec.js @@ -77,20 +77,25 @@ test.describe("Slack channel settings", () => { .locator("label", { hasText: "Acknowledge with reactions" }) .locator('input[type="checkbox"]'); const triggerCheckbox = modal.locator("label", { hasText: "Reaction triggers" }).locator('input[type="checkbox"]'); + const richBlocksCheckbox = modal + .locator("label", { hasText: "Rich Block Kit rendering" }) + .locator('input[type="checkbox"]'); - // Reflects current config: ack on, triggers off. + // Reflects current config: ack on, triggers off, rich blocks off. await expect(ackCheckbox).toBeChecked(); await expect(triggerCheckbox).not.toBeChecked(); + await expect(richBlocksCheckbox).not.toBeChecked(); - // Flip both. + // Flip all three. await ackCheckbox.uncheck(); await triggerCheckbox.check(); + await richBlocksCheckbox.check(); await modal.getByRole("button", { name: "Save Changes", exact: true }).click(); await expect .poll(() => page.evaluate(() => window.__slackUpdateRequest)) - .toMatchObject({ config: { ack_reactions: false, reaction_triggers: true } }); + .toMatchObject({ config: { ack_reactions: false, reaction_triggers: true, rich_blocks: true } }); expect(pageErrors).toEqual([]); }); diff --git a/crates/web/ui/src/pages/ChannelsPage.tsx b/crates/web/ui/src/pages/ChannelsPage.tsx index 2b244600c6..1873f89a73 100644 --- a/crates/web/ui/src/pages/ChannelsPage.tsx +++ b/crates/web/ui/src/pages/ChannelsPage.tsx @@ -91,6 +91,8 @@ export interface ChannelConfig { channel_allowlist?: string[]; ack_reactions?: boolean; reaction_triggers?: boolean; + rich_blocks?: boolean; + assistant_status?: boolean; // Matrix homeserver?: string; user_id?: string; diff --git a/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx b/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx index 76a21319dc..5a9f5c6030 100644 --- a/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx +++ b/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx @@ -49,6 +49,8 @@ export function EditChannelModal(): VNode | null { const editSlackApiBaseUrl = useSignal("https://slack.com/api"); const editSlackAckReactions = useSignal(true); const editSlackReactionTriggers = useSignal(false); + const editSlackRichBlocks = useSignal(false); + const editSlackAssistantStatus = useSignal(false); const editChannelNamePatterns = useSignal([]); const editCategoryAllowlist = useSignal([]); const editAdvancedConfigPatch = useSignal(""); @@ -79,6 +81,8 @@ export function EditChannelModal(): VNode | null { editSlackApiBaseUrl.value = (ch?.config?.api_base_url as string) || "https://slack.com/api"; editSlackAckReactions.value = ch?.config?.ack_reactions !== false; editSlackReactionTriggers.value = ch?.config?.reaction_triggers === true; + editSlackRichBlocks.value = ch?.config?.rich_blocks === true; + editSlackAssistantStatus.value = ch?.config?.assistant_status === true; editChannelNamePatterns.value = (ch?.config?.channel_name_patterns || []) as string[]; editCategoryAllowlist.value = (ch?.config?.category_allowlist || []) as string[]; editAdvancedConfigPatch.value = ""; @@ -199,6 +203,8 @@ export function EditChannelModal(): VNode | null { updateConfig.api_base_url = editSlackApiBaseUrl.value.trim() || "https://slack.com/api"; updateConfig.ack_reactions = editSlackAckReactions.value; updateConfig.reaction_triggers = editSlackReactionTriggers.value; + updateConfig.rich_blocks = editSlackRichBlocks.value; + updateConfig.assistant_status = editSlackAssistantStatus.value; } addChannelCredentials(updateConfig, form); addModelToConfig(updateConfig); @@ -438,6 +444,38 @@ export function EditChannelModal(): VNode | null { + + )} {isNostr && ( diff --git a/docs/src/slack.md b/docs/src/slack.md index 86d7b6bf49..d50baca821 100644 --- a/docs/src/slack.md +++ b/docs/src/slack.md @@ -107,6 +107,8 @@ offered = ["slack"] | `ack_reactions` | no | `true` | Acknowledge inbound messages with emoji reactions (πŸ‘€ β†’ βœ…/❌). Only applied when the bot is directly addressed (DM or @mention). | | `reaction_triggers` | no | `false` | Route inbound user reactions into the agent as messages (e.g. react βœ… to approve). | | `reaction_trigger_emojis` | no | `[]` | When `reaction_triggers` is on, only these emoji shortcodes trigger the agent. Empty = any emoji. | +| `rich_blocks` | no | `false` | Render replies as Block Kit blocks (headings, dividers, code) with a plain-text fallback. | +| `assistant_status` | no | `false` | Show a live "is thinking…" status via `assistant.threads.setStatus`. Requires an AI/Assistant app. | | `channel_overrides` | no | `{}` | Per-channel model/provider overrides (see below) | | `user_overrides` | no | `{}` | Per-user model/provider overrides (see below) | @@ -273,8 +275,11 @@ with emoji reactions so you know your message was received and is being worked on: - πŸ‘€ (`eyes`) is added as soon as the message starts processing. +- While the agent runs, the reaction swaps to a **phase** emoji reflecting the + current tool β€” 🌐 web, πŸ’» shell, ✏️ file edits, πŸ› οΈ other tools β€” and shows ⏳ if + a step runs long. Rapid changes are debounced so the reaction doesn't flicker. - βœ… (`white_check_mark`) replaces it when the reply is delivered. -- ❌ (`x`) replaces it if the turn fails. +- ❌ (`x`) replaces it if the turn fails. A cancelled turn just removes πŸ‘€. Reactions are only added when the bot is **directly addressed** β€” a direct message, or a channel message that @mentions the bot β€” never on general channel From 141f5739a8c5fa40998ca37a12848e702271ccfa Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 10:27:41 -0700 Subject: [PATCH 03/23] refactor(channels): extract activity types to keep files under size limit The new `ChannelActivity`/`ChannelAckOutcome` enums and the turn-finished emit pushed `channels/plugin.rs` and `chat/.../send.rs` past the 1500-line limit. Move the enums to `crates/channels/src/activity.rs` (re-exported unchanged) and inline the terminal emit compactly. No behavior change. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/channels/src/activity.rs | 32 +++++++++++++++++++++++ crates/channels/src/lib.rs | 16 +++++++----- crates/channels/src/plugin.rs | 28 -------------------- crates/chat/src/service/chat_impl/send.rs | 31 +++++++++------------- 4 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 crates/channels/src/activity.rs diff --git a/crates/channels/src/activity.rs b/crates/channels/src/activity.rs new file mode 100644 index 0000000000..24b23f5be4 --- /dev/null +++ b/crates/channels/src/activity.rs @@ -0,0 +1,32 @@ +//! Channel acknowledgment-reaction signals. +//! +//! Small value types shared across the chat layer (which emits them) and the +//! gateway (which forwards them to a per-session reaction controller). Kept in +//! their own module so the plugin module stays focused. + +/// Terminal outcome of a channel-dispatched agent turn, used to finalize +/// acknowledgment reactions (βœ… / ❌ / none). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChannelAckOutcome { + /// The reply was delivered successfully. + Success, + /// The turn errored (timeout, provider failure, etc.). + Failure, + /// The turn was cancelled/aborted β€” leave no terminal marker. + Cancelled, +} + +/// A mid-turn activity signal emitted by the agent run, used to drive channel +/// acknowledgment reactions (phase emojis) and, later, live status text. +/// +/// Kept intentionally small β€” richer phases are derived from the tool name at +/// the controller. +#[derive(Debug, Clone)] +pub enum ChannelActivity { + /// The model is reasoning/planning before or between tool calls. + Thinking, + /// A tool call started; carries the tool name for phase classification. + Tool(String), + /// The turn finished with the given terminal outcome. + Finished(ChannelAckOutcome), +} diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index 6b098e8401..58c0a4d712 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -4,6 +4,7 @@ //! ChannelPlugin trait with sub-traits for config, auth, inbound/outbound //! messaging, status, and gateway lifecycle. +pub mod activity; pub mod channel_webhook_middleware; pub mod commands; pub mod config_view; @@ -19,6 +20,7 @@ pub mod slack_api_url; pub mod store; pub use { + activity::{ChannelAckOutcome, ChannelActivity}, channel_webhook_middleware::{ ChannelWebhookDedupeResult, ChannelWebhookRatePolicy, ChannelWebhookRejection, ChannelWebhookVerifier, TimestampGuard, VerifiedChannelWebhook, @@ -27,13 +29,13 @@ pub use { error::{Error, Result}, media_download::{InboundMediaDownloader, InboundMediaSource}, plugin::{ - ButtonRow, ButtonStyle, ChannelAckOutcome, ChannelActivity, ChannelAttachment, - ChannelCapabilities, ChannelDescriptor, ChannelDocumentFile, ChannelEvent, - ChannelEventSink, ChannelHealthSnapshot, ChannelMessageKind, ChannelMessageMeta, - ChannelOtpProvider, ChannelOutbound, ChannelPlugin, ChannelReplyTarget, ChannelStatus, - ChannelStreamOutbound, ChannelThreadContext, ChannelType, InboundMode, InteractiveButton, - InteractiveMessage, SavedChannelFile, StreamEvent, StreamReceiver, StreamSender, - ThreadMessage, resolve_session_channel_binding, web_session_channel_binding, + ButtonRow, ButtonStyle, ChannelAttachment, ChannelCapabilities, ChannelDescriptor, + ChannelDocumentFile, ChannelEvent, ChannelEventSink, ChannelHealthSnapshot, + ChannelMessageKind, ChannelMessageMeta, ChannelOtpProvider, ChannelOutbound, ChannelPlugin, + ChannelReplyTarget, ChannelStatus, ChannelStreamOutbound, ChannelThreadContext, + ChannelType, InboundMode, InteractiveButton, InteractiveMessage, SavedChannelFile, + StreamEvent, StreamReceiver, StreamSender, ThreadMessage, resolve_session_channel_binding, + web_session_channel_binding, }, registry::{ChannelRegistry, RegistryOutboundRouter}, slack_api_url::{normalize_slack_api_base_url, validate_slack_api_base_url}, diff --git a/crates/channels/src/plugin.rs b/crates/channels/src/plugin.rs index a8a5ebd70f..e7212befcf 100644 --- a/crates/channels/src/plugin.rs +++ b/crates/channels/src/plugin.rs @@ -671,34 +671,6 @@ impl ChannelReplyTarget { } } -/// Terminal outcome of a channel-dispatched agent turn, used to finalize -/// acknowledgment reactions (βœ… / ❌ / none). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ChannelAckOutcome { - /// The reply was delivered successfully. - Success, - /// The turn errored (timeout, provider failure, etc.). - Failure, - /// The turn was cancelled/aborted β€” leave no terminal marker. - Cancelled, -} - -/// A mid-turn activity signal emitted by the agent run, used to drive channel -/// acknowledgment reactions (phase emojis) and, later, live status text. -/// -/// The chat layer emits these; the gateway forwards them to a per-session -/// reaction controller. Kept intentionally small β€” richer phases are derived -/// from the tool name at the controller. -#[derive(Debug, Clone)] -pub enum ChannelActivity { - /// The model is reasoning/planning before or between tool calls. - Thinking, - /// A tool call started; carries the tool name for phase classification. - Tool(String), - /// The turn finished with the given terminal outcome. - Finished(ChannelAckOutcome), -} - impl From<&ChannelReplyTarget> for ChannelBinding { fn from(target: &ChannelReplyTarget) -> Self { let channel_type = target.channel_type.as_str().to_string(); diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index d0cf5d792a..9b4963201e 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -1271,24 +1271,19 @@ impl LiveChatService { agent_fut.await }; - // Signal the channel acknowledgment reactions that the turn finished. - // By this point the reply (if any) has already been delivered inside - // `agent_fut`, so a present assistant response means success. Aborted - // runs never reach here (the task is cancelled); the abort path emits - // `Cancelled` instead. - { - let outcome = if assistant_text.is_some() { - moltis_channels::ChannelAckOutcome::Success - } else { - moltis_channels::ChannelAckOutcome::Failure - }; - state - .note_channel_activity( - &session_key_clone, - moltis_channels::ChannelActivity::Finished(outcome), - ) - .await; - } + // Finalize channel ack reactions (aborted runs emit Cancelled on the + // abort path and never reach here). + let ack_outcome = if assistant_text.is_some() { + moltis_channels::ChannelAckOutcome::Success + } else { + moltis_channels::ChannelAckOutcome::Failure + }; + state + .note_channel_activity( + &session_key_clone, + moltis_channels::ChannelActivity::Finished(ack_outcome), + ) + .await; // Persist assistant response (even empty ones β€” needed for LLM history coherence). if let Some(assistant_output) = assistant_text { From b729dea2c88c656695f3bea24f5c5a1a10fab39e Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 10:31:53 -0700 Subject: [PATCH 04/23] test(slack): allow unwrap in blocks test module for --all-targets clippy `just lint` runs clippy with `--all-targets`, which lints test code where the workspace denies `unwrap()`. Match the existing convention used by the other Slack test modules. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/slack/src/blocks.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index 3c42daf517..994cb776ec 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -179,6 +179,7 @@ fn truncate_chars(text: &str, max: usize) -> String { } #[cfg(test)] +#[allow(clippy::unwrap_used)] mod tests { use super::*; From 31098585ffeb1ee53ef3577b12e24219da92363e Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 13:12:53 -0700 Subject: [PATCH 05/23] fix(slack): split long fenced code into balanced Block Kit fences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address Greptile P1 on #1166. A fenced code block longer than the 3000-char section limit was wrapped as ```…``` and then split at line boundaries, so the chunks had unbalanced fences (opening with no close, later chunks closing with no open) and Slack rendered them as broken code blocks. A single large code block stays under the 50-block cap, so the plain-text fallback never fired. Split the *raw* code first (reserving room for the delimiters) and re-wrap each chunk in its own ```…``` fence, so every emitted section is a self-contained, balanced code block. Adds a test asserting each chunk has matched fences and stays within the section limit. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/slack/src/blocks.rs | 66 +++++++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index 994cb776ec..da20bda353 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -29,7 +29,7 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { let flush_paragraph = |paragraph: &mut String, blocks: &mut Vec| { let trimmed = paragraph.trim(); if !trimmed.is_empty() { - for chunk in split_section(&markdown_to_slack(trimmed)) { + for chunk in split_by_limit(&markdown_to_slack(trimmed), MAX_SECTION_CHARS) { blocks.push(section_block(&chunk)); } } @@ -41,11 +41,8 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { if line.trim_start().starts_with("```") { match code.take() { Some(buf) => { - // Closing fence: emit the accumulated code as a section. - let fenced = format!("```\n{}\n```", buf.trim_end_matches('\n')); - for chunk in split_section(&fenced) { - blocks.push(section_block(&chunk)); - } + // Closing fence: emit the accumulated code as fenced section(s). + push_code_blocks(&buf, &mut blocks); }, None => { // Opening fence: flush any pending paragraph first. @@ -93,10 +90,7 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { // Flush trailing state. if let Some(buf) = code.take() { // Unterminated fence: still emit what we have. - let fenced = format!("```\n{}\n```", buf.trim_end_matches('\n')); - for chunk in split_section(&fenced) { - blocks.push(section_block(&chunk)); - } + push_code_blocks(&buf, &mut blocks); } flush_paragraph(&mut paragraph, &mut blocks); @@ -132,27 +126,35 @@ fn section_block(mrkdwn: &str) -> Value { }) } -/// Split a section body into pieces no longer than [`MAX_SECTION_CHARS`], -/// preferring line boundaries. -fn split_section(text: &str) -> Vec { - if text.chars().count() <= MAX_SECTION_CHARS { +/// Emit a code block as one or more fenced section blocks. Oversized code is +/// split on the *raw* content and each chunk re-wrapped in its own ``` fence, +/// so a split never produces unbalanced/mismatched code delimiters. +fn push_code_blocks(code: &str, blocks: &mut Vec) { + // Reserve room for the fence delimiters ("```\n" + "\n```") in each chunk. + let overhead = "```\n\n```".chars().count(); + let body_limit = MAX_SECTION_CHARS.saturating_sub(overhead).max(1); + for piece in split_by_limit(code.trim_end_matches('\n'), body_limit) { + blocks.push(section_block(&format!("```\n{piece}\n```"))); + } +} + +/// Split a body into pieces no longer than `limit` chars, preferring line +/// boundaries and hard-splitting any single line that exceeds the limit. +fn split_by_limit(text: &str, limit: usize) -> Vec { + if text.chars().count() <= limit { return vec![text.to_string()]; } let mut out = Vec::new(); let mut current = String::new(); for line in text.split_inclusive('\n') { - if current.chars().count() + line.chars().count() > MAX_SECTION_CHARS && !current.is_empty() - { + if current.chars().count() + line.chars().count() > limit && !current.is_empty() { out.push(std::mem::take(&mut current)); } // A single line longer than the limit is hard-split by char boundary. - if line.chars().count() > MAX_SECTION_CHARS { + if line.chars().count() > limit { let mut buf = line; - while buf.chars().count() > MAX_SECTION_CHARS { - let cut = buf - .char_indices() - .nth(MAX_SECTION_CHARS) - .map_or(buf.len(), |(i, _)| i); + while buf.chars().count() > limit { + let cut = buf.char_indices().nth(limit).map_or(buf.len(), |(i, _)| i); out.push(buf[..cut].to_string()); buf = &buf[cut..]; } @@ -236,4 +238,24 @@ mod tests { assert!(b["text"]["text"].as_str().unwrap().chars().count() <= MAX_SECTION_CHARS); } } + + #[test] + fn long_fenced_code_splits_into_balanced_fences() { + // A code block far larger than MAX_SECTION_CHARS, many lines. + let code: String = (0..500).map(|i| format!("line {i} of code\n")).collect(); + let md = format!("```\n{code}```"); + let blocks = markdown_to_blocks(&md).unwrap(); + assert!(blocks.len() >= 2, "expected the code block to split"); + for b in &blocks { + let text = b["text"]["text"].as_str().unwrap(); + // Every chunk is a self-contained, balanced code fence within limits. + assert!( + text.starts_with("```\n"), + "chunk missing opening fence: {text:.40}" + ); + assert!(text.ends_with("\n```"), "chunk missing closing fence"); + assert_eq!(text.matches("```").count(), 2, "unbalanced fences in chunk"); + assert!(text.chars().count() <= MAX_SECTION_CHARS); + } + } } From ffe054869f2658cc76498d1a50dffa65ec2f0aa7 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 19:49:08 -0700 Subject: [PATCH 06/23] refactor(slack): bound reaction worker lifetime and reuse the HTTP client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the two edge cases Greptile noted on #1166 (5/5, non-blocking): - Reaction controller worker now has a hard lifetime cap. Normally the run signals completion long before it, but if a run never finalizes (e.g. the spawned task panics and skips the terminal signal) the cap ensures the worker task β€” and the πŸ‘€ reaction β€” cannot leak; it strips the marker and exits. - `send_typing` (and native streaming) reuse a shared `reqwest::Client` instead of building one per call. reqwest pools connections; `send_typing` runs on a repeating loop while a turn is active, so per-call construction wasted the pool. Claude-Session: https://claude.ai/code/session_016mWjaPu4o3E5dBhZU6tPUL --- crates/gateway/src/channel_reactions.rs | 16 ++++++++++++++++ crates/slack/src/outbound.rs | 15 +++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index eaf5ee9f76..0d777ca2c6 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -32,6 +32,11 @@ const STALL_EMOJI: &str = "hourglass_flowing_sand"; const PHASE_DEBOUNCE: Duration = Duration::from_millis(700); /// After this idle time with no activity, show a "still working" marker. const STALL_AFTER: Duration = Duration::from_secs(20); +/// Hard cap on a worker's lifetime. Normally the run signals completion long +/// before this; the cap is a safety net so a run that never finalizes (e.g. a +/// panicked task that skips the terminal signal) can't leave the worker task β€” +/// and the πŸ‘€ reaction β€” alive forever. +const MAX_LIFETIME: Duration = Duration::from_secs(900); /// Registry of active controllers, keyed by session key (runs serialize per /// session via the send permit, so at most one is active per session). @@ -150,8 +155,19 @@ async fn run_worker( let mut current: Option = Some(RECEIVED_EMOJI.to_string()); let mut pending: Option = None; let mut stalled = false; + let started = tokio::time::Instant::now(); loop { + // Safety net: never outlive the hard cap. If a run failed to signal + // completion (e.g. it panicked), strip the in-progress marker and exit + // so neither the task nor the πŸ‘€ reaction leaks. + if started.elapsed() >= MAX_LIFETIME { + if let Some(cur) = current.take() { + remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur).await; + } + return; + } + // Wait for the next command. If a phase change is pending, apply it once // the debounce window elapses; otherwise fall back to the stall timer. let wait = if pending.is_some() { diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index c07868cd8e..b73ea79b3a 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -28,6 +28,17 @@ use crate::{ /// Minimum chars before the first message is sent during streaming. const STREAM_MIN_INITIAL_CHARS: usize = 30; +/// Shared HTTP client for raw Slack Web API calls (native streaming, assistant +/// status). `reqwest::Client` pools connections and is cheap to clone, so build +/// it once instead of per call β€” `send_typing` in particular is invoked on a +/// repeating loop while a turn runs. +fn shared_http_client() -> reqwest::Client { + static CLIENT: std::sync::OnceLock = std::sync::OnceLock::new(); + CLIENT + .get_or_init(moltis_common::http_client::build_default_http_client) + .clone() +} + /// Slack outbound message sender. pub struct SlackOutbound { pub(crate) accounts: AccountStateMap, @@ -125,7 +136,7 @@ impl SlackOutbound { stream: &mut StreamReceiver, ) -> ChannelResult<()> { let (bot_token, api_base_url, throttle) = self.get_native_stream_config(account_id)?; - let http = moltis_common::http_client::build_default_http_client(); + let http = shared_http_client(); let stream_id = start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; @@ -644,7 +655,7 @@ impl ChannelOutbound for SlackOutbound { return Ok(()); }; - let http = moltis_common::http_client::build_default_http_client(); + let http = shared_http_client(); let body = serde_json::json!({ "channel_id": to, "thread_ts": thread_ts, From 8193085ac842863e51f5ae28af45d4e00629e73b Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 23:22:20 -0700 Subject: [PATCH 07/23] fix(slack): finalize replaced reaction controller and test the state machine MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two improvements to the reaction controller found in self-review: - Same-session collision leak: when a second message for a session arrived while the first was still running, registering the new controller dropped the old handle, so its worker exited on the closed channel and left the previous message's πŸ‘€ stuck forever. Finalize the replaced controller (strip its πŸ‘€) before installing the new one. - Add state-machine tests with a recording mock outbound covering the full lifecycle: πŸ‘€ β†’ βœ… (success), πŸ‘€ β†’ ❌ (failure), πŸ‘€ β†’ stripped (cancelled, no terminal), and πŸ‘€ β†’ tool phase β†’ βœ… across the debounce window. The controller logic was previously only covered by pure-function tests. --- crates/gateway/src/channel_events.rs | 13 ++- crates/gateway/src/channel_reactions.rs | 118 +++++++++++++++++++++++- 2 files changed, 128 insertions(+), 3 deletions(-) diff --git a/crates/gateway/src/channel_events.rs b/crates/gateway/src/channel_events.rs index bd609ddf1c..109edd1d6b 100644 --- a/crates/gateway/src/channel_events.rs +++ b/crates/gateway/src/channel_events.rs @@ -360,11 +360,22 @@ async fn register_channel_reaction_controller( reply_to.chat_id.clone(), message_id, ); - state + // If a controller was already registered for this session (e.g. the user + // sent another message before the previous turn finished), finalize the old + // one first so its in-progress πŸ‘€ is stripped rather than left stuck when + // its handle is dropped. + let previous = state .channel_reaction_controllers .lock() .await .insert(session_key.to_string(), controller); + if let Some(previous) = previous { + previous + .note(moltis_channels::ChannelActivity::Finished( + ChannelAckOutcome::Cancelled, + )) + .await; + } } /// Finalize the acknowledgment reaction only when `chat.send` returned before diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index 0d777ca2c6..d7385f999a 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -38,8 +38,12 @@ const STALL_AFTER: Duration = Duration::from_secs(20); /// and the πŸ‘€ reaction β€” alive forever. const MAX_LIFETIME: Duration = Duration::from_secs(900); -/// Registry of active controllers, keyed by session key (runs serialize per -/// session via the send permit, so at most one is active per session). +/// Registry of active controllers, keyed by session key. +/// +/// Agent runs serialize per session (via the send permit), so at most one run +/// is executing at a time. If a second message for the same session is received +/// while the first is still running, registering its controller replaces (and +/// finalizes) the first β€” see `register_channel_reaction_controller`. pub type ReactionControllerRegistry = Arc>>>; /// Classify a tool name into a phase emoji shortcode. @@ -284,8 +288,118 @@ async fn remove_reaction( #[cfg(test)] mod tests { + use std::sync::Mutex as StdMutex; + + use moltis_channels::{Result as ChannelResult, plugin::ChannelOutbound}; + use super::*; + /// A mock outbound that records the reaction add/remove operations in order. + struct RecordingOutbound { + ops: Arc>>, + } + + #[async_trait::async_trait] + impl ChannelOutbound for RecordingOutbound { + async fn send_text(&self, _: &str, _: &str, _: &str, _: Option<&str>) -> ChannelResult<()> { + Ok(()) + } + + async fn send_media( + &self, + _: &str, + _: &str, + _: &moltis_common::types::ReplyPayload, + _: Option<&str>, + ) -> ChannelResult<()> { + Ok(()) + } + + async fn add_reaction(&self, _: &str, _: &str, _: &str, emoji: &str) -> ChannelResult<()> { + self.ops + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("+{emoji}")); + Ok(()) + } + + async fn remove_reaction( + &self, + _: &str, + _: &str, + _: &str, + emoji: &str, + ) -> ChannelResult<()> { + self.ops + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(format!("-{emoji}")); + Ok(()) + } + } + + async fn run_lifecycle(outcome: ChannelAckOutcome) -> Vec { + let ops = Arc::new(StdMutex::new(Vec::new())); + let outbound = Arc::new(RecordingOutbound { ops: ops.clone() }); + let controller = + ChannelReactionController::start(outbound, "a".into(), "c".into(), "m".into()); + // Let the worker apply the initial πŸ‘€ before signalling completion. + tokio::time::sleep(Duration::from_millis(50)).await; + controller.note(ChannelActivity::Finished(outcome)).await; + tokio::time::sleep(Duration::from_millis(50)).await; + ops.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + #[tokio::test] + async fn lifecycle_success_swaps_eyes_for_check() { + assert_eq!(run_lifecycle(ChannelAckOutcome::Success).await, vec![ + "+eyes", + "-eyes", + "+white_check_mark" + ]); + } + + #[tokio::test] + async fn lifecycle_failure_swaps_eyes_for_x() { + assert_eq!(run_lifecycle(ChannelAckOutcome::Failure).await, vec![ + "+eyes", "-eyes", "+x" + ]); + } + + #[tokio::test] + async fn lifecycle_cancelled_strips_eyes_with_no_terminal() { + // Cancelled removes the in-progress marker and adds nothing. + assert_eq!(run_lifecycle(ChannelAckOutcome::Cancelled).await, vec![ + "+eyes", "-eyes" + ]); + } + + #[tokio::test] + async fn tool_phase_swaps_marker_then_terminal() { + let ops = Arc::new(StdMutex::new(Vec::new())); + let outbound = Arc::new(RecordingOutbound { ops: ops.clone() }); + let controller = + ChannelReactionController::start(outbound, "a".into(), "c".into(), "m".into()); + tokio::time::sleep(Duration::from_millis(50)).await; + controller + .note(ChannelActivity::Tool("web_search".into())) + .await; + // Wait past the debounce window so the phase is applied. + tokio::time::sleep(PHASE_DEBOUNCE + Duration::from_millis(100)).await; + controller + .note(ChannelActivity::Finished(ChannelAckOutcome::Success)) + .await; + tokio::time::sleep(Duration::from_millis(50)).await; + let recorded = ops.lock().unwrap_or_else(|e| e.into_inner()).clone(); + assert_eq!(recorded, vec![ + "+eyes", + "-eyes", + "+globe_with_meridians", + "-globe_with_meridians", + "+white_check_mark", + ]); + } + #[test] fn classifies_web_tools() { assert_eq!(classify_tool_emoji("web_search"), "globe_with_meridians"); From b31ccfc63708e7e87f4802166d2d4e51dd262f63 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Fri, 24 Jul 2026 23:43:26 -0700 Subject: [PATCH 08/23] refactor(channels): share a neutral ack-emoji vocabulary and add before removing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two improvements from re-comparing against openclaw and hermes. Channel-neutral emoji vocabulary (DRY + layering). The reaction controller lives in the gateway and drives a generic `ChannelOutbound`, but it hardcoded Slack shortcodes ("eyes", "white_check_mark", …). Matrix uses the emoji glyph itself as the reaction key and Teams has its own vocabulary, so any non-Slack channel that populated `ack_message_id` would have reacted with the literal text "eyes". Per the cross-crate DRY rule, the canonical set now lives once in `moltis_channels::activity::ack_emoji` as Unicode glyphs, and each channel translates at its own boundary β€” Slack's existing `normalize_reaction_name` maps them to shortcodes, Matrix passes them straight through. This is the same split openclaw gets from its `StatusReactionAdapter`. A new Slack test walks `ack_emoji::ALL` and fails if any canonical emoji lacks a shortcode mapping, so the two crates cannot drift. Add-then-remove ordering, matching openclaw's `finishWithEmoji` (which applies the terminal emoji before clearing the rest). Removing first left a window with no reaction at all, and if the subsequent add failed the message was left bare β€” indistinguishable from the bot ignoring it. Adding first means the message always carries a marker and a failed call degrades to the previous one. Cancel still just strips the marker. Tests assert the exact emitted sequence. --- crates/channels/src/activity.rs | 36 ++++++++ crates/gateway/src/channel_reactions.rs | 104 ++++++++++++++---------- crates/slack/src/emoji.rs | 20 +++++ 3 files changed, 119 insertions(+), 41 deletions(-) diff --git a/crates/channels/src/activity.rs b/crates/channels/src/activity.rs index 24b23f5be4..875b211228 100644 --- a/crates/channels/src/activity.rs +++ b/crates/channels/src/activity.rs @@ -4,6 +4,42 @@ //! gateway (which forwards them to a per-session reaction controller). Kept in //! their own module so the plugin module stays focused. +/// Canonical acknowledgment-reaction emoji, as Unicode glyphs. +/// +/// These are the platform-neutral names the reaction controller emits. Each +/// channel adapts them at its own boundary: Slack normalizes glyphs to +/// shortcodes (its Web API rejects raw glyphs), while Matrix uses the glyph +/// itself as the reaction key. Keeping the vocabulary here β€” rather than +/// hardcoding one platform's spelling in the controller β€” is what lets the +/// controller stay channel-agnostic. +pub mod ack_emoji { + /// Message received, work starting. + pub const RECEIVED: &str = "πŸ‘€"; + /// Turn completed successfully. + pub const SUCCESS: &str = "βœ…"; + /// Turn failed. + pub const ERROR: &str = "❌"; + /// Turn is taking a while with no activity. + pub const STALL: &str = "⏳"; + /// Phase: web search / fetch / browsing. + pub const WEB: &str = "🌐"; + /// Phase: shell / process execution. + pub const SHELL: &str = "πŸ’»"; + /// Phase: editing or writing files. + pub const EDIT: &str = "✏️"; + /// Phase: deploying / releasing. + pub const DEPLOY: &str = "πŸ›«"; + /// Phase: building / compiling. + pub const BUILD: &str = "πŸ—οΈ"; + /// Phase: any other tool. + pub const TOOL: &str = "πŸ› οΈ"; + + /// Every canonical emoji, for exhaustive per-channel mapping tests. + pub const ALL: &[&str] = &[ + RECEIVED, SUCCESS, ERROR, STALL, WEB, SHELL, EDIT, DEPLOY, BUILD, TOOL, + ]; +} + /// Terminal outcome of a channel-dispatched agent turn, used to finalize /// acknowledgment reactions (βœ… / ❌ / none). #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index d7385f999a..91a9f1a4c6 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -7,10 +7,15 @@ //! loop, and finalized (βœ…/❌ or nothing on cancel) when the run completes. //! //! The controller owns a single reaction "slot" on the inbound message: each -//! transition removes the current emoji and adds the next, so at most one -//! reaction from the bot is visible at a time. All Slack API calls run on one -//! serialized worker task (no concurrent add/remove races), phase changes are -//! debounced to avoid flicker, and terminal transitions win over late phases. +//! transition adds the next emoji and then removes the previous one, so the +//! message is never momentarily bare and a failed add leaves the old marker +//! rather than clearing the acknowledgment entirely. All reaction API calls run +//! on one serialized worker task (no concurrent add/remove races), phase changes +//! are debounced to avoid flicker, and terminals win over late phases. +//! +//! Emoji come from the channel-neutral [`ack_emoji`] vocabulary; each channel +//! translates at its own boundary (Slack maps glyphs to shortcodes, Matrix uses +//! glyphs directly), so this controller stays platform-agnostic. //! //! Design ported from openclaw's `status-reactions.ts`. @@ -22,11 +27,15 @@ use { tracing::debug, }; -/// Emoji shortcodes for acknowledgment/phase reactions (Slack-style names). -const RECEIVED_EMOJI: &str = "eyes"; -const SUCCESS_EMOJI: &str = "white_check_mark"; -const ERROR_EMOJI: &str = "x"; -const STALL_EMOJI: &str = "hourglass_flowing_sand"; +/// Acknowledgment/phase emoji, from the shared channel-neutral vocabulary. +/// Channels translate these at their own boundary (Slack maps glyphs to +/// shortcodes; Matrix uses the glyph directly). +use moltis_channels::activity::ack_emoji; + +const RECEIVED_EMOJI: &str = ack_emoji::RECEIVED; +const SUCCESS_EMOJI: &str = ack_emoji::SUCCESS; +const ERROR_EMOJI: &str = ack_emoji::ERROR; +const STALL_EMOJI: &str = ack_emoji::STALL; /// Coalesce rapid phase changes so the reaction does not flicker. const PHASE_DEBOUNCE: Duration = Duration::from_millis(700); @@ -65,9 +74,9 @@ pub fn classify_tool_emoji(tool_name: &str) -> &'static str { "http", "url", ]) { - "globe_with_meridians" + ack_emoji::WEB } else if has(&["bash", "exec", "shell", "process", "command", "run"]) { - "computer" + ack_emoji::SHELL } else if has(&[ "edit", "write", @@ -77,13 +86,13 @@ pub fn classify_tool_emoji(tool_name: &str) -> &'static str { "create_file", "code", ]) { - "pencil2" + ack_emoji::EDIT } else if has(&["deploy", "fly", "release", "publish"]) { - "airplane_departure" + ack_emoji::DEPLOY } else if has(&["build", "compile", "cargo", "npm", "make"]) { - "building_construction" + ack_emoji::BUILD } else { - "hammer_and_wrench" + ack_emoji::TOOL } } @@ -188,11 +197,24 @@ async fn run_worker( } }, Ok(Some(Command::Finish(outcome))) => { - if let Some(cur) = current.take() { - remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur).await; - } - if let Some(term) = terminal_emoji(outcome) { - add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; + match terminal_emoji(outcome) { + // Add the terminal marker *before* removing the in-progress + // one so the message is never momentarily bare, and a failed + // add still leaves the previous marker rather than nothing. + Some(term) => { + add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; + if let Some(cur) = current.take().filter(|cur| cur != term) { + remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur) + .await; + } + }, + // Cancelled: strip the in-progress marker, leave nothing. + None => { + if let Some(cur) = current.take() { + remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur) + .await; + } + }, } return; }, @@ -243,10 +265,13 @@ async fn swap( if current.as_deref() == Some(emoji) { return; } + // Add first, then remove the previous marker: the message always shows at + // least one reaction, and a failed add leaves the old marker in place + // instead of clearing the acknowledgment entirely. + add_reaction(outbound, account_id, chat_id, message_id, emoji).await; if let Some(cur) = current.take() { remove_reaction(outbound, account_id, chat_id, message_id, &cur).await; } - add_reaction(outbound, account_id, chat_id, message_id, emoji).await; *current = Some(emoji.to_string()); } @@ -353,16 +378,14 @@ mod tests { #[tokio::test] async fn lifecycle_success_swaps_eyes_for_check() { assert_eq!(run_lifecycle(ChannelAckOutcome::Success).await, vec![ - "+eyes", - "-eyes", - "+white_check_mark" + "+πŸ‘€", "+βœ…", "-πŸ‘€" ]); } #[tokio::test] async fn lifecycle_failure_swaps_eyes_for_x() { assert_eq!(run_lifecycle(ChannelAckOutcome::Failure).await, vec![ - "+eyes", "-eyes", "+x" + "+πŸ‘€", "+❌", "-πŸ‘€" ]); } @@ -370,7 +393,7 @@ mod tests { async fn lifecycle_cancelled_strips_eyes_with_no_terminal() { // Cancelled removes the in-progress marker and adds nothing. assert_eq!(run_lifecycle(ChannelAckOutcome::Cancelled).await, vec![ - "+eyes", "-eyes" + "+πŸ‘€", "-πŸ‘€" ]); } @@ -391,40 +414,39 @@ mod tests { .await; tokio::time::sleep(Duration::from_millis(50)).await; let recorded = ops.lock().unwrap_or_else(|e| e.into_inner()).clone(); - assert_eq!(recorded, vec![ - "+eyes", - "-eyes", - "+globe_with_meridians", - "-globe_with_meridians", - "+white_check_mark", - ]); + // Each transition adds the new marker before removing the old one, so + // the message is never left without a reaction. + assert_eq!(recorded, vec!["+πŸ‘€", "+🌐", "-πŸ‘€", "+βœ…", "-🌐"]); } #[test] fn classifies_web_tools() { - assert_eq!(classify_tool_emoji("web_search"), "globe_with_meridians"); - assert_eq!(classify_tool_emoji("web_fetch"), "globe_with_meridians"); + assert_eq!(classify_tool_emoji("web_search"), ack_emoji::WEB); + assert_eq!(classify_tool_emoji("web_fetch"), ack_emoji::WEB); } #[test] fn classifies_shell_and_edit_tools() { - assert_eq!(classify_tool_emoji("exec"), "computer"); - assert_eq!(classify_tool_emoji("bash"), "computer"); - assert_eq!(classify_tool_emoji("str_replace_editor"), "pencil2"); + assert_eq!(classify_tool_emoji("exec"), ack_emoji::SHELL); + assert_eq!(classify_tool_emoji("bash"), ack_emoji::SHELL); + assert_eq!(classify_tool_emoji("str_replace_editor"), ack_emoji::EDIT); } #[test] fn unknown_tool_uses_generic_emoji() { - assert_eq!(classify_tool_emoji("some_mcp_tool"), "hammer_and_wrench"); + assert_eq!(classify_tool_emoji("some_mcp_tool"), ack_emoji::TOOL); } #[test] fn terminal_emoji_maps_outcomes() { assert_eq!( terminal_emoji(ChannelAckOutcome::Success), - Some("white_check_mark") + Some(ack_emoji::SUCCESS) + ); + assert_eq!( + terminal_emoji(ChannelAckOutcome::Failure), + Some(ack_emoji::ERROR) ); - assert_eq!(terminal_emoji(ChannelAckOutcome::Failure), Some("x")); assert_eq!(terminal_emoji(ChannelAckOutcome::Cancelled), None); } } diff --git a/crates/slack/src/emoji.rs b/crates/slack/src/emoji.rs index 1b8e60e6d6..2010bbff92 100644 --- a/crates/slack/src/emoji.rs +++ b/crates/slack/src/emoji.rs @@ -57,6 +57,7 @@ fn glyph_to_shortcode(glyph: &str) -> Option<&'static str> { "πŸ› οΈ" | "πŸ› " => "hammer_and_wrench", "🌐" => "globe_with_meridians", "πŸ’»" => "computer", + "✏️" | "✏" => "pencil2", "πŸ—οΈ" | "πŸ—" => "building_construction", "πŸ›«" => "airplane_departure", "πŸ—œοΈ" | "πŸ—œ" => "clamp", @@ -107,6 +108,25 @@ mod tests { assert_eq!(normalize_reaction_name(" eyes "), "eyes"); } + #[test] + fn every_canonical_ack_emoji_maps_to_a_shortcode() { + // The reaction controller emits the channel-neutral glyph vocabulary; + // Slack's Web API silently drops raw glyphs, so every one of them must + // normalize to a shortcode here. This is the cross-crate contract β€” + // adding a new canonical emoji without a mapping fails this test. + for glyph in moltis_channels::activity::ack_emoji::ALL { + let normalized = normalize_reaction_name(glyph); + assert_ne!( + &normalized, glyph, + "canonical ack emoji {glyph} has no Slack shortcode mapping" + ); + assert!( + normalized.is_ascii(), + "{glyph} normalized to a non-shortcode value: {normalized}" + ); + } + } + #[test] fn unknown_glyph_passes_through() { // Not in the map β€” returned unchanged for Slack to accept or reject. From 1b46b5d08600a69afaa2f5193f549b155863f5b8 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Sat, 25 Jul 2026 00:37:06 -0700 Subject: [PATCH 09/23] fix(slack): route acknowledgments per message, not per session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External review found the acknowledgment reactions were routed by session key, which is wrong whenever a session handles more than one inbound message. Queued messages hijacked the running turn. The controller was registered before `chat.send` decided whether the message had to queue. A second message replaced the first's controller, then returned early without starting a run β€” so the first run's phases and terminal landed on the second message, the first was left unresolved, and a replayed message never registered anything at all. Terminal cleanup had a matching race: it removed whatever was stored for the session without checking identity, so a slow terminal could release a newer turn's controller and strand its marker. Acknowledgments are now keyed by the message itself (account:chat:ts) and parked on receipt, so πŸ‘€ still appears immediately even while queued. A run claims its acknowledgments only once it starts executing β€” after the queue decision β€” and releases them under an identity check against a turn id. In collect mode one run legitimately owns several messages and the terminal fans out to all of them. Keys travel through queueing and replay in the request params. Also from the review: - Reaction triggers now require the reacted-to message to be bot-authored (via item_user, failing closed), so a member cannot aim the agent at an unrelated person's message. - Socket Mode acknowledges envelopes immediately and processes off the callback path, with bounded event-id dedup so a Slack retry cannot start a second turn. - /sh runs and hook-rejected messages now finalize their acknowledgment instead of leaving a marker until the lifetime cap; a reply that fails to deliver resolves as ❌ rather than βœ…. - Block Kit no longer truncates overlong headings (rendered as sections) and is skipped for replies that need chunking, since the plain-text fallback must carry the whole message. rich_blocks now disables streaming, which otherwise silently bypassed block rendering entirely. - Reconnect jitter was always negative (0.75-0.87x) because nanoseconds were divided by u32::MAX instead of a second. - Reaction state only advances when the API call actually succeeded. - Setup UI and docs list the reaction scopes and event subscriptions. assistant_status is removed: it is unverifiable without an AI/Assistant app and its thread matching needs an exact per-turn identity to be correct. Adds registry tests for queued-message isolation, collect fan-out, superseded turns, no-run finalization, and stray activity. Extracts the queue-drain logic, which was duplicated verbatim across the two run paths. --- crates/chat/src/channel_acks.rs | 101 ++++++ crates/chat/src/channels.rs | 2 + crates/chat/src/lib.rs | 1 + crates/chat/src/runtime.rs | 14 + crates/chat/src/service/chat_impl.rs | 1 + .../chat/src/service/chat_impl/queue_drain.rs | 93 ++++++ crates/chat/src/service/chat_impl/send.rs | 161 +++------- crates/config/src/template.rs | 1 - crates/gateway/src/channel_events.rs | 71 +++-- crates/gateway/src/channel_events/dispatch.rs | 11 +- crates/gateway/src/channel_reactions.rs | 289 ++++++++++++++++-- crates/gateway/src/chat.rs | 41 +-- crates/gateway/src/state.rs | 6 +- crates/slack/src/blocks.rs | 42 ++- crates/slack/src/config.rs | 12 +- crates/slack/src/outbound.rs | 78 ++--- crates/slack/src/plugin.rs | 1 + crates/slack/src/socket.rs | 88 +++++- crates/slack/src/state.rs | 34 ++- crates/slack/src/webhook.rs | 9 + crates/web/ui/src/pages/ChannelsPage.tsx | 1 - .../pages/channels/modals/AddSlackModal.tsx | 15 +- .../channels/modals/EditChannelModal.tsx | 19 -- docs/src/slack.md | 11 +- 24 files changed, 772 insertions(+), 330 deletions(-) create mode 100644 crates/chat/src/channel_acks.rs create mode 100644 crates/chat/src/service/chat_impl/queue_drain.rs diff --git a/crates/chat/src/channel_acks.rs b/crates/chat/src/channel_acks.rs new file mode 100644 index 0000000000..b32bc2d11d --- /dev/null +++ b/crates/chat/src/channel_acks.rs @@ -0,0 +1,101 @@ +//! Channel acknowledgment-reaction plumbing for chat runs. +//! +//! An inbound channel message carries an acknowledgment identity (its "ack +//! key") so its reaction follows the message itself through queueing, replay +//! and collection, rather than following whatever else shares the session. + +use std::sync::Arc; + +use serde_json::Value; + +use crate::runtime::ChatRuntime; + +/// Params key carrying the acknowledgment identities a call is responsible for. +pub(crate) const ACK_KEYS_PARAM: &str = "_ack_keys"; + +/// Read the acknowledgment identities from `chat.send` params. +pub(crate) fn ack_keys_from_params(params: &Value) -> Vec { + params + .get(ACK_KEYS_PARAM) + .and_then(Value::as_array) + .map(|a| { + a.iter() + .filter_map(|v| v.as_str().map(String::from)) + .collect() + }) + .unwrap_or_default() +} + +/// Union the acknowledgment identities across several queued messages. +/// +/// In `collect` queue mode one run answers all of them, so it owns every one of +/// their acknowledgments and each message receives the terminal reaction. +pub(crate) fn merged_ack_keys<'a>(params: impl Iterator) -> Vec { + let mut keys = Vec::new(); + for p in params { + for key in ack_keys_from_params(p) { + if !keys.contains(&key) { + keys.push(key); + } + } + } + keys +} + +/// Mark the channel acknowledgment as failed because the reply never reached +/// the user. Terminal protection means this wins over the run's later Success. +pub(crate) async fn note_delivery_failed(state: &Arc, session_key: &str) { + state + .note_channel_activity( + session_key, + moltis_channels::ChannelActivity::Finished(moltis_channels::ChannelAckOutcome::Failure), + ) + .await; +} + +/// Finalize the acknowledgment for a completed run. +pub(crate) async fn note_turn_finished( + state: &Arc, + session_key: &str, + succeeded: bool, +) { + let outcome = if succeeded { + moltis_channels::ChannelAckOutcome::Success + } else { + moltis_channels::ChannelAckOutcome::Failure + }; + state + .note_channel_activity( + session_key, + moltis_channels::ChannelActivity::Finished(outcome), + ) + .await; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_and_defaults_ack_keys() { + assert_eq!( + ack_keys_from_params(&serde_json::json!({})), + Vec::::new() + ); + assert_eq!( + ack_keys_from_params(&serde_json::json!({ "_ack_keys": ["a", "b"] })), + vec!["a".to_string(), "b".to_string()] + ); + } + + #[test] + fn merges_and_dedupes_across_queued_messages() { + let a = serde_json::json!({ "_ack_keys": ["k1"] }); + let b = serde_json::json!({ "_ack_keys": ["k2", "k1"] }); + let c = serde_json::json!({}); + assert_eq!(merged_ack_keys([&a, &b, &c].into_iter()), vec![ + "k1".to_string(), + "k2".to_string() + ]); + } +} diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 4cd7b464bf..a86a5646c9 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -633,6 +633,7 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel reply: {e}" ); + crate::channel_acks::note_delivery_failed(&state, &session_key).await; } }, }, @@ -689,6 +690,7 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel reply: {e}" ); + crate::channel_acks::note_delivery_failed(&state, &session_key).await; } }, }, diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index 63bcd570bd..6a444f37c9 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,4 +1,5 @@ mod agent_loop; +pub(crate) mod channel_acks; mod channels; mod compaction; mod compaction_run; diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index f265d580fd..8991a73532 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -141,6 +141,20 @@ pub trait ChatRuntime: Send + Sync { /// completion (Finished). async fn note_channel_activity(&self, _session_key: &str, _activity: ChannelActivity) {} + /// Bind the given acknowledgment keys to this session's now-executing run, + /// so activity routes to exactly the inbound message(s) this run handles. + /// Called once the queue decision is known and the run actually starts. + async fn activate_channel_acks(&self, _session_key: &str, _ack_keys: Vec) {} + + /// Finalize acknowledgment keys directly, for paths where no run executes + /// (rejected hook, early error) and nothing else will signal a terminal. + async fn finalize_channel_acks( + &self, + _ack_keys: Vec, + _outcome: moltis_channels::ChannelAckOutcome, + ) { + } + // ── Push notifications ─────────────────────────────────────────────── /// Send a push notification to all subscribed devices. diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index 0706d8bab6..c89fc3a668 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -1,5 +1,6 @@ //! `ChatService` trait implementation for `LiveChatService`. +mod queue_drain; mod send; use std::{ diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs new file mode 100644 index 0000000000..8f77322cf1 --- /dev/null +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -0,0 +1,93 @@ +//! Replay of messages that queued behind an active run. +//! +//! Shared by both run paths (model turns and explicit `/sh` commands), which +//! previously carried identical copies of this logic. + +use std::{collections::HashMap, sync::Arc}; + +use { + serde_json::Value, + tokio::sync::RwLock, + tracing::{info, warn}, +}; + +use moltis_config::MessageQueueMode; + +use crate::{runtime::ChatRuntime, service::types::QueuedMessage}; + +/// Drain this session's queue and replay it according to `mode`. +/// +/// `Followup` replays the oldest message and puts the rest back, so the +/// replayed run drains them in turn. `Collect` merges every queued message into +/// one turn β€” which therefore owns all of their acknowledgment identities, so a +/// reaction is resolved on each of the combined messages rather than only the +/// last. +pub(super) async fn drain_and_replay( + message_queue: &Arc>>>, + session_key: &str, + mode: MessageQueueMode, + state: &Arc, +) { + let queued = message_queue + .write() + .await + .remove(session_key) + .unwrap_or_default(); + if queued.is_empty() { + return; + } + let chat = state.chat_service().await; + match mode { + MessageQueueMode::Followup => { + let mut iter = queued.into_iter(); + let Some(first) = iter.next() else { + return; + }; + // Put remaining messages back so the replayed run's own drain loop + // picks them up after it completes. + let rest: Vec = iter.collect(); + if !rest.is_empty() { + message_queue + .write() + .await + .entry(session_key.to_string()) + .or_default() + .extend(rest); + } + info!(session = %session_key, "replaying queued message (followup)"); + let mut replay_params = first.params; + replay_params["_queued_replay"] = Value::Bool(true); + if let Err(e) = chat.send(replay_params).await { + warn!(session = %session_key, error = %e, "failed to replay queued message"); + } + }, + MessageQueueMode::Collect => { + let combined: Vec<&str> = queued + .iter() + .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) + .collect(); + if combined.is_empty() { + return; + } + info!( + session = %session_key, + count = combined.len(), + "replaying collected messages" + ); + // Use the last queued message as the base params, override text. + let Some(last) = queued.last() else { + return; + }; + let mut merged = last.params.clone(); + merged["text"] = Value::String(combined.join("\n\n")); + merged["_queued_replay"] = Value::Bool(true); + // One run answers all of them, so it owns every acknowledgment. + merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( + crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) + ); + if let Err(e) = chat.send(merged).await { + warn!(session = %session_key, error = %e, "failed to replay collected messages"); + } + }, + } +} diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index 9b4963201e..23cea3a3a0 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -8,7 +8,7 @@ use { tracing::{debug, info, warn}, }; -use {moltis_config::MessageQueueMode, moltis_service_traits::ServiceResult}; +use moltis_service_traits::ServiceResult; #[cfg(feature = "local-llm")] use moltis_providers::model_id::raw_model_id; @@ -32,6 +32,8 @@ use crate::{ use {super::*, crate::service::build_persisted_assistant_message}; +use super::queue_drain; + use { crate::memory_tools::AgentScopedMemoryWriter, moltis_agents::model::values_to_chat_messages_with_tool_result_limit, @@ -125,6 +127,9 @@ impl LiveChatService { .and_then(|v| v.as_bool()) .unwrap_or(false); + // Carried through queueing/replay so reactions follow the message. + let ack_keys = crate::channel_acks::ack_keys_from_params(¶ms); + // Track client-side sequence number for ordering diagnostics. // Note: seq resets to 1 on page reload, so a drop from a high value // back to 1 is normal (new browser session) β€” only flag issues within @@ -196,6 +201,10 @@ impl LiveChatService { queued_replay, "chat.send: acquired session permit" ); + // This call owns the session and will execute: claim its acks. + self.state + .activate_channel_acks(&session_key, ack_keys.clone()) + .await; p }, Err(_) => { @@ -425,6 +434,10 @@ impl LiveChatService { session_metadata.touch(&session_key_clone, count).await; } + // Explicit /sh runs finalize their own acknowledgment: they + // never reach the model path that emits the terminal below. + crate::channel_acks::note_turn_finished(&state, &session_key_clone, true).await; + active_runs.write().await.remove(&run_id_clone); let mut runs_by_session = active_runs_by_session.write().await; if runs_by_session.get(&session_key_clone) == Some(&run_id_clone) { @@ -445,61 +458,14 @@ impl LiveChatService { drop(permit); - // Drain queued messages for this session. - let queued = message_queue - .write() - .await - .remove(&session_key_clone) - .unwrap_or_default(); - if !queued.is_empty() { - let queue_mode = message_queue_mode; - let chat = state_for_drain.chat_service().await; - match queue_mode { - MessageQueueMode::Followup => { - let mut iter = queued.into_iter(); - let Some(first) = iter.next() else { - return; - }; - let rest: Vec = iter.collect(); - if !rest.is_empty() { - message_queue - .write() - .await - .entry(session_key_clone.clone()) - .or_default() - .extend(rest); - } - info!(session = %session_key_clone, "replaying queued message (followup)"); - let mut replay_params = first.params; - replay_params["_queued_replay"] = serde_json::json!(true); - if let Err(e) = chat.send(replay_params).await { - warn!(session = %session_key_clone, error = %e, "failed to replay queued message"); - } - }, - MessageQueueMode::Collect => { - let combined: Vec<&str> = queued - .iter() - .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) - .collect(); - if !combined.is_empty() { - info!( - session = %session_key_clone, - count = combined.len(), - "replaying collected messages" - ); - let Some(last) = queued.last() else { - return; - }; - let mut merged = last.params.clone(); - merged["text"] = serde_json::json!(combined.join("\n\n")); - merged["_queued_replay"] = serde_json::json!(true); - if let Err(e) = chat.send(merged).await { - warn!(session = %session_key_clone, error = %e, "failed to replay collected messages"); - } - } - }, - } - } + // Replay anything that queued behind this run. + queue_drain::drain_and_replay( + &message_queue, + &session_key_clone, + message_queue_mode, + &state_for_drain, + ) + .await; }); self.active_runs @@ -1273,17 +1239,12 @@ impl LiveChatService { // Finalize channel ack reactions (aborted runs emit Cancelled on the // abort path and never reach here). - let ack_outcome = if assistant_text.is_some() { - moltis_channels::ChannelAckOutcome::Success - } else { - moltis_channels::ChannelAckOutcome::Failure - }; - state - .note_channel_activity( - &session_key_clone, - moltis_channels::ChannelActivity::Finished(ack_outcome), - ) - .await; + crate::channel_acks::note_turn_finished( + &state, + &session_key_clone, + assistant_text.is_some(), + ) + .await; // Persist assistant response (even empty ones β€” needed for LLM history coherence). if let Some(assistant_output) = assistant_text { @@ -1417,64 +1378,14 @@ impl LiveChatService { // fail `try_acquire_owned()` and re-queue the message forever. drop(permit); - // Drain queued messages for this session. - let queued = message_queue - .write() - .await - .remove(&session_key_clone) - .unwrap_or_default(); - if !queued.is_empty() { - let queue_mode = message_queue_mode; - let chat = state_for_drain.chat_service().await; - match queue_mode { - MessageQueueMode::Followup => { - let mut iter = queued.into_iter(); - let Some(first) = iter.next() else { - return; - }; - // Put remaining messages back so the replayed run's - // own drain loop picks them up after it completes. - let rest: Vec = iter.collect(); - if !rest.is_empty() { - message_queue - .write() - .await - .entry(session_key_clone.clone()) - .or_default() - .extend(rest); - } - info!(session = %session_key_clone, "replaying queued message (followup)"); - let mut replay_params = first.params; - replay_params["_queued_replay"] = serde_json::json!(true); - if let Err(e) = chat.send(replay_params).await { - warn!(session = %session_key_clone, error = %e, "failed to replay queued message"); - } - }, - MessageQueueMode::Collect => { - let combined: Vec<&str> = queued - .iter() - .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) - .collect(); - if !combined.is_empty() { - info!( - session = %session_key_clone, - count = combined.len(), - "replaying collected messages" - ); - // Use the last queued message as the base params, override text. - let Some(last) = queued.last() else { - return; - }; - let mut merged = last.params.clone(); - merged["text"] = serde_json::json!(combined.join("\n\n")); - merged["_queued_replay"] = serde_json::json!(true); - if let Err(e) = chat.send(merged).await { - warn!(session = %session_key_clone, error = %e, "failed to replay collected messages"); - } - } - }, - } - } + // Replay anything that queued behind this run. + queue_drain::drain_and_replay( + &message_queue, + &session_key_clone, + message_queue_mode, + &state_for_drain, + ) + .await; }); self.active_runs diff --git a/crates/config/src/template.rs b/crates/config/src/template.rs index 8e6bf7f74c..b17e80abe0 100644 --- a/crates/config/src/template.rs +++ b/crates/config/src/template.rs @@ -819,7 +819,6 @@ port = {port} # Port number (auto-generated for this i # ack_reactions = true # πŸ‘€ on receipt, phase emoji while working, βœ…/❌ on completion # reaction_triggers = false # route user reactions into the agent (react βœ… to approve) # rich_blocks = false # render replies as Block Kit blocks (fallback to plain text) -# assistant_status = false # live "is thinking…" status (needs an AI/Assistant app) # otp_self_approval = true # otp_cooldown_secs = 300 diff --git a/crates/gateway/src/channel_events.rs b/crates/gateway/src/channel_events.rs index 109edd1d6b..288dbbd6aa 100644 --- a/crates/gateway/src/channel_events.rs +++ b/crates/gateway/src/channel_events.rs @@ -345,37 +345,26 @@ fn start_channel_typing_loop( /// and reactions enabled) and an outbound implementation is available. async fn register_channel_reaction_controller( state: &Arc, - session_key: &str, reply_to: &ChannelReplyTarget, -) { - let Some(message_id) = reply_to.ack_message_id.clone() else { - return; - }; - let Some(outbound) = state.services.channel_outbound_arc() else { - return; - }; +) -> Option { + let message_id = reply_to.ack_message_id.clone()?; + let outbound = state.services.channel_outbound_arc()?; + let key = + crate::channel_reactions::ack_key(&reply_to.account_id, &reply_to.chat_id, &message_id); let controller = ChannelReactionController::start( outbound, reply_to.account_id.clone(), reply_to.chat_id.clone(), message_id, ); - // If a controller was already registered for this session (e.g. the user - // sent another message before the previous turn finished), finalize the old - // one first so its in-progress πŸ‘€ is stripped rather than left stuck when - // its handle is dropped. - let previous = state + // Park it against this message's own identity. A run claims it once the + // queue decision is known, so a message that queues behind an active run + // keeps its own πŸ‘€ instead of hijacking the running turn's reactions. + state .channel_reaction_controllers - .lock() - .await - .insert(session_key.to_string(), controller); - if let Some(previous) = previous { - previous - .note(moltis_channels::ChannelActivity::Finished( - ChannelAckOutcome::Cancelled, - )) - .await; - } + .register_pending(key.clone(), controller) + .await; + Some(key) } /// Finalize the acknowledgment reaction only when `chat.send` returned before @@ -385,30 +374,38 @@ async fn register_channel_reaction_controller( /// this leaves their controller in place. async fn finalize_reaction_on_early_return( state: &Arc, - session_key: &str, + ack_key: Option<&String>, send_result: &Result, ) { + let Some(ack_key) = ack_key else { + return; + }; let outcome = match send_result { Err(_) => Some(ChannelAckOutcome::Failure), - Ok(payload) => match payload.get("state").and_then(|v| v.as_str()) { - Some("rejected" | "error" | "blocked") => Some(ChannelAckOutcome::Failure), - Some("aborted") => Some(ChannelAckOutcome::Cancelled), - _ => None, + Ok(payload) => { + // A queued message keeps its acknowledgment: it has not run yet and + // will claim it on replay. + if payload.get("queued").and_then(serde_json::Value::as_bool) == Some(true) { + None + } else if payload.get("rejected").and_then(serde_json::Value::as_bool) == Some(true) { + // Blocked by a MessageReceived hook β€” no run will ever start. + Some(ChannelAckOutcome::Failure) + } else { + match payload.get("state").and_then(|v| v.as_str()) { + Some("rejected" | "error" | "blocked") => Some(ChannelAckOutcome::Failure), + Some("aborted") => Some(ChannelAckOutcome::Cancelled), + _ => None, + } + } }, }; let Some(outcome) = outcome else { return; }; - let controller = state + state .channel_reaction_controllers - .lock() - .await - .remove(session_key); - if let Some(controller) = controller { - controller - .note(moltis_channels::ChannelActivity::Finished(outcome)) - .await; - } + .finalize_keys(std::slice::from_ref(ack_key), outcome) + .await; } async fn resolve_channel_agent_id( diff --git a/crates/gateway/src/channel_events/dispatch.rs b/crates/gateway/src/channel_events/dispatch.rs index bab7355bae..6f4d36be66 100644 --- a/crates/gateway/src/channel_events/dispatch.rs +++ b/crates/gateway/src/channel_events/dispatch.rs @@ -24,7 +24,7 @@ pub(in crate::channel_events) async fn dispatch_to_chat( // (bot directly addressed and reactions enabled). Because the run is // fire-and-forget, the terminal is applied from the run's completion β€” // NOT from `chat.send` returning below. - register_channel_reaction_controller(state, &session_key, &reply_to).await; + let ack_key = register_channel_reaction_controller(state, &reply_to).await; let effective_text = if state.is_channel_command_mode_enabled(&session_key).await { rewrite_for_shell_mode(text).unwrap_or_else(|| text.to_string()) } else { @@ -142,6 +142,13 @@ pub(in crate::channel_events) async fn dispatch_to_chat( "_channel_reply_target": &reply_to, }); + // Carry this message's acknowledgment identity into the run so the + // reaction follows the message itself β€” through queueing and replay β€” + // rather than whatever else shares the session. + if let Some(ref key) = ack_key { + params["_ack_keys"] = serde_json::json!([key]); + } + // Attach thread context if available. if let Some(thread_history) = thread_context { params["_thread_context"] = serde_json::json!(thread_history); @@ -235,7 +242,7 @@ pub(in crate::channel_events) async fn dispatch_to_chat( // Only finalize the reaction here for early returns where the run never // executes (rejection/error before spawn). Normal runs finalize from the // run's completion via `note_channel_activity`. - finalize_reaction_on_early_return(state, &session_key, &send_result).await; + finalize_reaction_on_early_return(state, ack_key.as_ref(), &send_result).await; if let Err(e) = send_result { error!("channel dispatch_to_chat failed: {e}"); diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index 91a9f1a4c6..d47ec39acc 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -47,13 +47,143 @@ const STALL_AFTER: Duration = Duration::from_secs(20); /// and the πŸ‘€ reaction β€” alive forever. const MAX_LIFETIME: Duration = Duration::from_secs(900); -/// Registry of active controllers, keyed by session key. +/// Opaque per-message acknowledgment identity: `account:chat:message_id`. /// -/// Agent runs serialize per session (via the send permit), so at most one run -/// is executing at a time. If a second message for the same session is received -/// while the first is still running, registering its controller replaces (and -/// finalizes) the first β€” see `register_channel_reaction_controller`. -pub type ReactionControllerRegistry = Arc>>>; +/// One inbound message == one ack key == one reaction slot, independent of +/// which agent run eventually handles it. +#[must_use] +pub fn ack_key(account_id: &str, chat_id: &str, message_id: &str) -> String { + format!("{account_id}:{chat_id}:{message_id}") +} + +/// Upper bound on parked (received but not yet running) acknowledgments, so a +/// message that is never claimed by a run cannot grow the map without bound. +/// Controllers additionally self-expire via [`MAX_LIFETIME`]. +const MAX_PENDING_ACKS: usize = 512; + +/// The acknowledgments a currently-executing run owns. +/// +/// `id` distinguishes successive turns on the same session so a slow terminal +/// from turn N can never clear turn N+1's state. +struct ActiveTurn { + id: u64, + keys: Vec, +} + +#[derive(Default)] +struct RegistryInner { + /// Received messages with a live reaction, not yet claimed by a run. + pending: HashMap>, + /// Insertion order of `pending`, for oldest-first eviction. + pending_order: Vec, + /// Which acknowledgments the running turn owns, per session. + active: HashMap, +} + +/// Routes acknowledgment-reaction activity to the right inbound message(s). +/// +/// A message's controller is created when it is *received* (so πŸ‘€ appears +/// immediately, even while queued) and keyed by its own ack key. A run claims +/// its acknowledgments when it actually starts executing β€” which is also when +/// the queue decision is known β€” so activity is never misrouted to a message +/// that merely happens to share the session. In `collect` queue mode a single +/// run legitimately owns several inbound messages, and every one of them +/// receives the phase and terminal reactions. +#[derive(Default)] +pub struct ReactionRegistry { + inner: Mutex, + next_turn_id: std::sync::atomic::AtomicU64, +} + +impl ReactionRegistry { + /// Park a freshly received message's controller until a run claims it. + pub async fn register_pending(&self, key: String, controller: Arc) { + let mut inner = self.inner.lock().await; + if inner.pending.insert(key.clone(), controller).is_none() { + inner.pending_order.push(key); + } + while inner.pending_order.len() > MAX_PENDING_ACKS { + let oldest = inner.pending_order.remove(0); + inner.pending.remove(&oldest); + } + } + + /// Bind the given acknowledgments to this session's now-executing run. + /// + /// Any previously active turn for the session is dropped from routing (its + /// controllers keep their own lifetime cap), so a superseded run cannot + /// keep driving reactions. + pub async fn activate(&self, session_key: &str, keys: Vec) { + if keys.is_empty() { + return; + } + let id = self + .next_turn_id + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut inner = self.inner.lock().await; + inner + .active + .insert(session_key.to_string(), ActiveTurn { id, keys }); + } + + /// Forward an activity to every acknowledgment the session's run owns. + /// + /// Terminal activities finalize and release those acknowledgments; the + /// release is identity-checked against the turn id so a late terminal from + /// a superseded run cannot clear the current one. + pub async fn note(&self, session_key: &str, activity: ChannelActivity) { + let is_terminal = matches!(activity, ChannelActivity::Finished(_)); + let (turn_id, controllers) = { + let inner = self.inner.lock().await; + let Some(turn) = inner.active.get(session_key) else { + return; + }; + let controllers: Vec<_> = turn + .keys + .iter() + .filter_map(|k| inner.pending.get(k).cloned()) + .collect(); + (turn.id, controllers) + }; + + for controller in &controllers { + controller.note(activity.clone()).await; + } + + if is_terminal { + let mut inner = self.inner.lock().await; + // Only release if this session's active turn is still ours. + let still_ours = inner + .active + .get(session_key) + .is_some_and(|turn| turn.id == turn_id); + if still_ours && let Some(turn) = inner.active.remove(session_key) { + for key in turn.keys { + inner.pending.remove(&key); + inner.pending_order.retain(|k| k != &key); + } + } + } + } + + /// Finalize specific acknowledgments directly, for paths where no run ever + /// executes (a rejected hook, an early error) and nothing will signal a + /// terminal for them. + pub async fn finalize_keys(&self, keys: &[String], outcome: ChannelAckOutcome) { + let controllers = { + let mut inner = self.inner.lock().await; + keys.iter() + .filter_map(|k| { + inner.pending_order.retain(|pk| pk != k); + inner.pending.remove(k) + }) + .collect::>() + }; + for controller in controllers { + controller.note(ChannelActivity::Finished(outcome)).await; + } + } +} /// Classify a tool name into a phase emoji shortcode. /// @@ -156,8 +286,8 @@ async fn run_worker( chat_id: String, message_id: String, ) { - // Initial acknowledgment: πŸ‘€. - add_reaction( + // Initial acknowledgment: πŸ‘€. Only track it if it actually landed. + let acked = add_reaction( &outbound, &account_id, &chat_id, @@ -165,7 +295,7 @@ async fn run_worker( RECEIVED_EMOJI, ) .await; - let mut current: Option = Some(RECEIVED_EMOJI.to_string()); + let mut current: Option = acked.then(|| RECEIVED_EMOJI.to_string()); let mut pending: Option = None; let mut stalled = false; let started = tokio::time::Instant::now(); @@ -202,8 +332,10 @@ async fn run_worker( // one so the message is never momentarily bare, and a failed // add still leaves the previous marker rather than nothing. Some(term) => { - add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; - if let Some(cur) = current.take().filter(|cur| cur != term) { + let landed = + add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; + let stale = current.take().filter(|cur| cur != term); + if let Some(cur) = stale.filter(|_| landed) { remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur) .await; } @@ -268,28 +400,38 @@ async fn swap( // Add first, then remove the previous marker: the message always shows at // least one reaction, and a failed add leaves the old marker in place // instead of clearing the acknowledgment entirely. - add_reaction(outbound, account_id, chat_id, message_id, emoji).await; + if !add_reaction(outbound, account_id, chat_id, message_id, emoji).await { + // The new marker never landed β€” keep tracking the old one so it is + // still cleaned up at the terminal instead of being orphaned. + return; + } if let Some(cur) = current.take() { remove_reaction(outbound, account_id, chat_id, message_id, &cur).await; } *current = Some(emoji.to_string()); } +/// Returns whether the reaction is believed to be on the message afterwards, so +/// callers only advance their tracked state when the call actually landed. async fn add_reaction( outbound: &Arc, account_id: &str, chat_id: &str, message_id: &str, emoji: &str, -) { - if let Err(e) = outbound +) -> bool { + match outbound .add_reaction(account_id, chat_id, message_id, emoji) .await { - debug!( - account_id, - chat_id, emoji, "channel add_reaction failed: {e}" - ); + Ok(()) => true, + Err(e) => { + debug!( + account_id, + chat_id, emoji, "channel add_reaction failed: {e}" + ); + false + }, } } @@ -419,6 +561,117 @@ mod tests { assert_eq!(recorded, vec!["+πŸ‘€", "+🌐", "-πŸ‘€", "+βœ…", "-🌐"]); } + /// Build a registry with one parked acknowledgment; returns its ops log. + async fn park(registry: &ReactionRegistry, key: &str) -> Arc>> { + let ops = Arc::new(StdMutex::new(Vec::new())); + let outbound = Arc::new(RecordingOutbound { ops: ops.clone() }); + let controller = + ChannelReactionController::start(outbound, "a".into(), "c".into(), key.into()); + registry.register_pending(key.to_string(), controller).await; + tokio::time::sleep(Duration::from_millis(40)).await; + ops + } + + fn ops_of(ops: &Arc>>) -> Vec { + ops.lock().unwrap_or_else(|e| e.into_inner()).clone() + } + + #[tokio::test] + async fn queued_message_keeps_its_own_ack_and_is_not_driven_by_the_active_run() { + // A is running; B arrives and queues behind it. B must keep its own πŸ‘€ + // and must not receive A's terminal. + let registry = ReactionRegistry::default(); + let a = park(®istry, "msg-a").await; + let b = park(®istry, "msg-b").await; + registry.activate("sess", vec!["msg-a".into()]).await; + + registry + .note( + "sess", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + + assert_eq!(ops_of(&a), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"], "A should complete"); + assert_eq!( + ops_of(&b), + vec!["+πŸ‘€"], + "B must still be waiting, untouched" + ); + } + + #[tokio::test] + async fn collect_mode_fans_terminal_out_to_every_combined_message() { + let registry = ReactionRegistry::default(); + let a = park(®istry, "msg-a").await; + let b = park(®istry, "msg-b").await; + // One run answers both messages. + registry + .activate("sess", vec!["msg-a".into(), "msg-b".into()]) + .await; + + registry + .note( + "sess", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + + for ops in [&a, &b] { + assert_eq!(ops_of(ops), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"]); + } + } + + #[tokio::test] + async fn superseded_turn_terminal_does_not_clear_the_current_turn() { + // Turn 1 activates, is superseded by turn 2, then turn 1's terminal + // arrives late. It must not release turn 2's acknowledgment. + let registry = ReactionRegistry::default(); + let b = park(®istry, "msg-b").await; + registry.activate("sess", vec!["msg-b".into()]).await; + registry.activate("sess", vec!["msg-b".into()]).await; // turn 2 + + registry + .note( + "sess", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + + // The still-current turn resolved exactly once. + assert_eq!(ops_of(&b), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"]); + } + + #[tokio::test] + async fn finalize_keys_resolves_acks_that_never_ran() { + // Hook rejection / early error: no run ever claims the message. + let registry = ReactionRegistry::default(); + let a = park(®istry, "msg-a").await; + registry + .finalize_keys(&["msg-a".to_string()], ChannelAckOutcome::Failure) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + assert_eq!(ops_of(&a), vec!["+πŸ‘€", "+❌", "-πŸ‘€"]); + } + + #[tokio::test] + async fn activity_for_unknown_session_is_a_noop() { + let registry = ReactionRegistry::default(); + let a = park(®istry, "msg-a").await; + // Never activated β€” a stray activity must not touch the parked ack. + registry + .note( + "other-session", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(40)).await; + assert_eq!(ops_of(&a), vec!["+πŸ‘€"]); + } + #[test] fn classifies_web_tools() { assert_eq!(classify_tool_emoji("web_search"), ack_emoji::WEB); diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index ac27862e85..2d7ec6e175 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -59,25 +59,28 @@ impl ChatRuntime for GatewayChatRuntime { // ── Channel acknowledgment reactions ──────────────────────────────────── async fn note_channel_activity(&self, session_key: &str, activity: ChannelActivity) { - // A terminal activity finalizes and removes the controller; phase - // activities just forward. Look up without holding the lock across the - // (awaiting) forward to avoid contention. - let is_terminal = matches!(activity, ChannelActivity::Finished(_)); - let controller = { - let map = self.state.channel_reaction_controllers.lock().await; - map.get(session_key).cloned() - }; - let Some(controller) = controller else { - return; - }; - controller.note(activity).await; - if is_terminal { - self.state - .channel_reaction_controllers - .lock() - .await - .remove(session_key); - } + self.state + .channel_reaction_controllers + .note(session_key, activity) + .await; + } + + async fn activate_channel_acks(&self, session_key: &str, ack_keys: Vec) { + self.state + .channel_reaction_controllers + .activate(session_key, ack_keys) + .await; + } + + async fn finalize_channel_acks( + &self, + ack_keys: Vec, + outcome: moltis_channels::ChannelAckOutcome, + ) { + self.state + .channel_reaction_controllers + .finalize_keys(&ack_keys, outcome) + .await; } // ── Channel status log ────────────────────────────────────────────────── diff --git a/crates/gateway/src/state.rs b/crates/gateway/src/state.rs index 3b3211ab99..549938e933 100644 --- a/crates/gateway/src/state.rs +++ b/crates/gateway/src/state.rs @@ -502,7 +502,7 @@ pub struct GatewayState { /// Active per-turn channel acknowledgment reaction controllers, keyed by /// session key. Created when a channel message is received (adds πŸ‘€), /// driven by the agent run, and removed when the turn finalizes. - pub channel_reaction_controllers: crate::channel_reactions::ReactionControllerRegistry, + pub channel_reaction_controllers: Arc, } impl GatewayState { @@ -606,7 +606,9 @@ impl GatewayState { broadcaster: Arc::new(Broadcaster::new()), client_registry: RwLock::new(ClientRegistryInner::new()), inner: RwLock::new(GatewayInner::new(hook_registry)), - channel_reaction_controllers: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + channel_reaction_controllers: Arc::new( + crate::channel_reactions::ReactionRegistry::default(), + ), }) } diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index da20bda353..94489eb936 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -68,10 +68,19 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { continue; } - // ATX heading β†’ header block. + // ATX heading β†’ header block. A header cannot hold more than + // MAX_HEADER_CHARS and Slack has no continuation, so an overlong + // heading is rendered as a bold section instead of being truncated β€” + // the renderer must never drop content. if let Some(text) = heading_text(trimmed) { flush_paragraph(&mut paragraph, &mut blocks); - blocks.push(header_block(&text)); + if text.chars().count() > MAX_HEADER_CHARS { + for chunk in split_by_limit(&format!("*{text}*"), MAX_SECTION_CHARS) { + blocks.push(section_block(&chunk)); + } + } else { + blocks.push(header_block(&text)); + } continue; } @@ -111,11 +120,12 @@ fn heading_text(line: &str) -> Option { } } +/// Build a header block. Callers must ensure `text` fits [`MAX_HEADER_CHARS`]; +/// longer headings are rendered as sections so nothing is lost. fn header_block(text: &str) -> Value { - let truncated = truncate_chars(text, MAX_HEADER_CHARS); json!({ "type": "header", - "text": { "type": "plain_text", "text": truncated, "emoji": true }, + "text": { "type": "plain_text", "text": text, "emoji": true }, }) } @@ -169,17 +179,6 @@ fn split_by_limit(text: &str, limit: usize) -> Vec { out } -fn truncate_chars(text: &str, max: usize) -> String { - if text.chars().count() <= max { - return text.to_string(); - } - let cut = text - .char_indices() - .nth(max.saturating_sub(1)) - .map_or(text.len(), |(i, _)| i); - format!("{}…", &text[..cut]) -} - #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -222,6 +221,19 @@ mod tests { assert_eq!(blocks[0]["type"], "section"); } + #[test] + fn overlong_heading_becomes_a_section_not_truncated() { + let heading = "H".repeat(400); + let blocks = markdown_to_blocks(&format!("# {heading}")).unwrap(); + // Rendered as a section, and the full text survives. + assert_eq!(blocks[0]["type"], "section"); + let rendered: String = blocks + .iter() + .filter_map(|b| b["text"]["text"].as_str()) + .collect(); + assert!(rendered.contains(&heading), "heading content was lost"); + } + #[test] fn over_limit_falls_back_to_none() { // Many dividers exceed the 50-block cap β†’ fall back to plain text. diff --git a/crates/slack/src/config.rs b/crates/slack/src/config.rs index bd35e3ef36..d6f2b20cd9 100644 --- a/crates/slack/src/config.rs +++ b/crates/slack/src/config.rs @@ -125,12 +125,6 @@ pub struct SlackAccountConfig { /// content cannot be represented within Slack's limits. Default: false. pub rich_blocks: bool, - /// Show a live "is thinking…" status via `assistant.threads.setStatus` - /// while a turn runs. Only works when the Slack app is configured as an - /// AI/Assistant app and the message is in an assistant thread; a no-op - /// otherwise. Default: false. - pub assistant_status: bool, - /// Acknowledge inbound messages with emoji reactions (πŸ‘€ on receipt, βœ… on /// success, ❌ on error). Only applied when the bot is directly addressed /// (DM or @mention). Default: true. @@ -185,7 +179,6 @@ impl std::fmt::Debug for SlackAccountConfig { .field("edit_throttle_ms", &self.edit_throttle_ms) .field("thread_replies", &self.thread_replies) .field("rich_blocks", &self.rich_blocks) - .field("assistant_status", &self.assistant_status) .field("ack_reactions", &self.ack_reactions) .field("reaction_triggers", &self.reaction_triggers) .field("reaction_trigger_emojis", &self.reaction_trigger_emojis) @@ -217,7 +210,6 @@ impl Default for SlackAccountConfig { edit_throttle_ms: 500, thread_replies: true, rich_blocks: false, - assistant_status: false, ack_reactions: true, reaction_triggers: false, reaction_trigger_emojis: Vec::new(), @@ -289,7 +281,7 @@ pub struct RedactedConfig<'a>(pub &'a SlackAccountConfig); impl Serialize for RedactedConfig<'_> { fn serialize(&self, serializer: S) -> Result { let c = self.0; - let mut count = 18; // always-present fields + let mut count = 17; // always-present fields count += c.signing_secret.is_some() as usize; count += !c.reaction_trigger_emojis.is_empty() as usize; count += c.model.is_some() as usize; @@ -323,7 +315,6 @@ impl Serialize for RedactedConfig<'_> { s.serialize_field("edit_throttle_ms", &c.edit_throttle_ms)?; s.serialize_field("thread_replies", &c.thread_replies)?; s.serialize_field("rich_blocks", &c.rich_blocks)?; - s.serialize_field("assistant_status", &c.assistant_status)?; s.serialize_field("ack_reactions", &c.ack_reactions)?; s.serialize_field("reaction_triggers", &c.reaction_triggers)?; if !c.reaction_trigger_emojis.is_empty() { @@ -446,7 +437,6 @@ mod tests { assert!(cfg.thread_replies); assert!(cfg.ack_reactions); assert!(!cfg.rich_blocks); - assert!(!cfg.assistant_status); assert!(cfg.otp_self_approval); assert_eq!(cfg.otp_cooldown_secs, 300); assert_eq!(cfg.mention_mode, MentionMode::Mention); diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index b73ea79b3a..49c644f639 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -531,22 +531,22 @@ impl ChannelOutbound for SlackOutbound { let thread_ts = self.get_thread_ts(account_id, to, reply_to); let slack_text = markdown_to_slack(text); - // Rich Block Kit rendering (opt-in). Falls back to plain chunked text - // when disabled or when the content can't be represented as blocks. - let rendered_blocks = self - .get_rich_blocks(account_id) + let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); + + // Rich Block Kit rendering (opt-in). Only used when the whole reply fits + // a single message: the `text` fallback must carry the *entire* content + // for notifications and clients that cannot render blocks, so a reply + // that needs chunking is sent as plain text instead of silently + // delivering only its first chunk. + let rendered_blocks = (self.get_rich_blocks(account_id) && chunks.len() == 1) .then(|| crate::blocks::markdown_to_blocks(text)) .flatten(); if let Some(blocks) = rendered_blocks { - let fallback = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN) - .first() - .copied() - .unwrap_or(&slack_text); + let fallback = chunks.first().copied().unwrap_or(&slack_text); post_message_with_blocks(&client, &token, to, fallback, &blocks, thread_ts.as_deref()) .await?; } else { - let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); for chunk in chunks { post_message(&client, &token, to, chunk, thread_ts.as_deref()).await?; } @@ -624,56 +624,11 @@ impl ChannelOutbound for SlackOutbound { } } - async fn send_typing(&self, account_id: &str, to: &str) -> ChannelResult<()> { - // Slack bots have no classic typing indicator. When the account opts in - // and the message is in an assistant thread, surface a live status via - // `assistant.threads.setStatus` instead. Best-effort: any error (e.g. - // the app is not an AI/Assistant app) is swallowed at debug. - let params = { - let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); - let Some(state) = accounts.get(account_id) else { - return Ok(()); - }; - if !state.config.assistant_status { - return Ok(()); - } - // Resolve the thread ts for this channel (assistant threads only). - let thread_ts = state - .pending_threads - .iter() - .find(|(k, _)| k.starts_with(&format!("{to}:"))) - .map(|(_, ts)| ts.clone()); - thread_ts.map(|ts| { - ( - state.config.bot_token.expose_secret().clone(), - state.config.api_base_url.clone(), - ts, - ) - }) - }; - let Some((bot_token, api_base_url, thread_ts)) = params else { - return Ok(()); - }; - - let http = shared_http_client(); - let body = serde_json::json!({ - "channel_id": to, - "thread_ts": thread_ts, - "status": "is thinking…", - }); - match http - .post(slack_api_method_url( - &api_base_url, - "assistant.threads.setStatus", - )?) - .bearer_auth(&bot_token) - .json(&body) - .send() - .await - { - Ok(_) => {}, - Err(e) => debug!(account_id, to, "assistant.threads.setStatus failed: {e}"), - } + async fn send_typing(&self, _account_id: &str, _to: &str) -> ChannelResult<()> { + // Slack bots have no typing indicator. `assistant.threads.setStatus` + // can show a live status, but only for apps configured as Slack AI/ + // Assistant apps and only inside assistant threads; that needs an exact + // thread identity per turn, so it is intentionally not wired here. Ok(()) } @@ -953,7 +908,10 @@ impl ChannelStreamOutbound for SlackOutbound { } async fn is_stream_enabled(&self, account_id: &str) -> bool { - self.get_stream_mode(account_id) != StreamMode::Off + // Streaming sends incremental plain text, which cannot carry Block Kit. + // When rich rendering is requested it wins: the reply is delivered once, + // complete, through `send_text` so it actually renders as blocks. + self.get_stream_mode(account_id) != StreamMode::Off && !self.get_rich_blocks(account_id) } } diff --git a/crates/slack/src/plugin.rs b/crates/slack/src/plugin.rs index 3bd903ddcf..54d93d0540 100644 --- a/crates/slack/src/plugin.rs +++ b/crates/slack/src/plugin.rs @@ -342,6 +342,7 @@ mod tests { bot_user_id: Some("Ubot".into()), pending_threads: HashMap::new(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), } } diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index 116e69dd66..70379dbd34 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -90,6 +90,7 @@ pub async fn start_socket_mode( bot_user_id: Some(bot_user_id), pending_threads: std::collections::HashMap::new(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(otp_cooldown_secs)), + dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), }); } @@ -234,10 +235,13 @@ fn jittered(base: std::time::Duration) -> std::time::Duration { .duration_since(std::time::UNIX_EPOCH) .map(|d| d.subsec_nanos()) .unwrap_or(0); - // Map nanos into [-0.25, +0.25]. - let frac = (f64::from(nanos) / f64::from(u32::MAX)) - 0.5; - let factor = 1.0 + frac * 0.5; - base.mul_f64(factor.clamp(0.5, 1.5)) + // Nanoseconds within the current second map to [0, 1), then to a + // symmetric +/-25% factor. Dividing by anything but NANOS_PER_SEC would + // bias the jitter to one side. + const NANOS_PER_SEC: f64 = 1_000_000_000.0; + let unit = f64::from(nanos) / NANOS_PER_SEC; + let factor = 0.75 + unit * 0.5; + base.mul_f64(factor.clamp(0.75, 1.25)) } /// Error handler for Socket Mode. @@ -263,6 +267,39 @@ async fn push_events_callback( }; drop(guard); + // Slack retries an envelope that is not acknowledged quickly. Drop repeats + // so a retry cannot start a second agent turn for one user message. + { + let accts = listener_state + .accounts + .read() + .unwrap_or_else(|e| e.into_inner()); + if let Some(state) = accts.get(&listener_state.account_id) { + let mut dedup = state.dedup.lock().unwrap_or_else(|e| e.into_inner()); + if !dedup.insert_new(event.event_id.as_ref()) { + debug!( + account_id = %listener_state.account_id, + event_id = %event.event_id, + "dropping duplicate slack event (retry)" + ); + return Ok(()); + } + } + } + + // Process off the callback path: the SDK only acknowledges the envelope + // once this callback returns, and handling can involve Web API calls that + // would otherwise risk exceeding Slack's acknowledgment deadline and + // trigger a retry. + tokio::spawn(async move { + handle_push_event(event, listener_state).await; + }); + + Ok(()) +} + +/// Handle a push event body. Runs off the Socket Mode acknowledgment path. +async fn handle_push_event(event: SlackPushEventCallback, listener_state: ListenerState) { match event.event { SlackEventCallbackBody::Message(msg_event) => { handle_message_event( @@ -303,6 +340,7 @@ async fn push_events_callback( reaction_event.user.as_ref(), reaction_event.reaction.as_ref(), &reaction_event.item, + reaction_event.item_user.as_ref().map(|u| u.to_string()), true, &listener_state.accounts, ) @@ -314,6 +352,7 @@ async fn push_events_callback( reaction_event.user.as_ref(), reaction_event.reaction.as_ref(), &reaction_event.item, + reaction_event.item_user.as_ref().map(|u| u.to_string()), false, &listener_state.accounts, ) @@ -323,8 +362,6 @@ async fn push_events_callback( debug!("unhandled slack push event"); }, } - - Ok(()) } /// Command events callback (slash commands). @@ -698,11 +735,13 @@ pub(crate) async fn handle_inbound( } /// Handle a reaction_added or reaction_removed event. +#[allow(clippy::too_many_arguments)] pub(crate) async fn handle_reaction_event( account_id: &str, user_id: &str, emoji: &str, item: &SlackReactionsItem, + item_user: Option, added: bool, accounts: &AccountStateMap, ) { @@ -720,7 +759,7 @@ pub(crate) async fn handle_reaction_event( }; dispatch_reaction( - account_id, user_id, emoji, channel_id, message_ts, added, accounts, + account_id, user_id, emoji, channel_id, message_ts, item_user, added, accounts, ) .await; } @@ -730,12 +769,14 @@ pub(crate) async fn handle_reaction_event( /// Always emits a [`ChannelEvent::ReactionChange`] for observers; additionally /// routes the reaction into the agent as a synthetic message when /// `reaction_triggers` is enabled and the reaction is eligible. +#[allow(clippy::too_many_arguments)] pub(crate) async fn dispatch_reaction( account_id: &str, user_id: &str, emoji: &str, channel_id: String, message_ts: String, + item_user: Option, added: bool, accounts: &AccountStateMap, ) { @@ -783,10 +824,18 @@ pub(crate) async fn dispatch_reaction( true }, }; + // Only reactions on the bot's *own* messages drive the agent. Without this + // any member could point the agent at an arbitrary third party's message + // simply by reacting to it. Fail closed when authorship is unknown. + let target_is_bot = match (bot_user_id.as_deref(), item_user.as_deref()) { + (Some(bot_id), Some(author)) => bot_id == author, + _ => false, + }; if !reaction_should_trigger( config.reaction_triggers, added, is_self, + target_is_bot, emoji, &config.reaction_trigger_emojis, ) { @@ -846,16 +895,22 @@ pub(crate) async fn dispatch_reaction( /// Decide whether an inbound reaction should be routed to the agent. /// /// Triggers only when: the feature is enabled, the reaction was *added* (not -/// removed), it is not the bot's own reaction, and β€” if an emoji allowlist is -/// configured β€” the emoji is on it. An empty allowlist matches any emoji. +/// removed), it is not the bot's own reaction, the reacted-to message was +/// authored by the bot, and β€” if an emoji allowlist is configured β€” the emoji +/// is on it. An empty allowlist matches any emoji. fn reaction_should_trigger( enabled: bool, added: bool, is_self: bool, + target_is_bot: bool, emoji: &str, allowlist: &[String], ) -> bool { - enabled && added && !is_self && (allowlist.is_empty() || allowlist.iter().any(|e| e == emoji)) + enabled + && added + && !is_self + && target_is_bot + && (allowlist.is_empty() || allowlist.iter().any(|e| e == emoji)) } /// Decide which inbound message (if any) to acknowledge with reactions. @@ -1152,13 +1207,13 @@ mod tests { #[test] fn reaction_trigger_requires_enabled_added_and_not_self() { // Disabled β†’ never. - assert!(!reaction_should_trigger(false, true, false, "x", &[])); + assert!(!reaction_should_trigger(false, true, false, true, "x", &[])); // Removal β†’ never. - assert!(!reaction_should_trigger(true, false, false, "x", &[])); + assert!(!reaction_should_trigger(true, false, false, true, "x", &[])); // Bot's own reaction β†’ never. - assert!(!reaction_should_trigger(true, true, true, "eyes", &[])); + assert!(!reaction_should_trigger(true, true, true, true, "eyes", &[])); // Enabled add by another user with no allowlist β†’ trigger. - assert!(reaction_should_trigger(true, true, false, "x", &[])); + assert!(reaction_should_trigger(true, true, false, true, "x", &[])); } #[test] @@ -1168,10 +1223,13 @@ mod tests { true, true, false, + true, "white_check_mark", &allow )); - assert!(!reaction_should_trigger(true, true, false, "x", &allow)); + assert!(!reaction_should_trigger( + true, true, false, true, "x", &allow + )); } #[test] diff --git a/crates/slack/src/state.rs b/crates/slack/src/state.rs index af27f880a1..59d54f8d1a 100644 --- a/crates/slack/src/state.rs +++ b/crates/slack/src/state.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashMap, + collections::{HashMap, HashSet, VecDeque}, sync::{Arc, Mutex, RwLock}, }; @@ -13,6 +13,36 @@ use crate::config::SlackAccountConfig; /// Shared account state map. pub type AccountStateMap = Arc>>; +/// Bounded set of recently seen Slack event ids. +/// +/// Slack retries an envelope when it is not acknowledged in time, so the same +/// event can arrive more than once. Without this a retry would start a second +/// agent turn for one user message. +#[derive(Default)] +pub struct EventDedup { + seen: HashSet, + order: VecDeque, +} + +impl EventDedup { + /// Maximum retained ids. Slack retries within minutes, so this is ample. + const MAX: usize = 2048; + + /// Record an event id, returning `true` if it had not been seen before. + pub fn insert_new(&mut self, event_id: &str) -> bool { + if !self.seen.insert(event_id.to_string()) { + return false; + } + self.order.push_back(event_id.to_string()); + while self.order.len() > Self::MAX { + if let Some(old) = self.order.pop_front() { + self.seen.remove(&old); + } + } + true + } +} + /// Per-account runtime state. pub struct AccountState { pub account_id: String, @@ -26,4 +56,6 @@ pub struct AccountState { /// Used to route replies into the correct thread. pub pending_threads: HashMap, pub otp: Mutex, + /// Recently processed event ids, for retry idempotency. + pub dedup: Mutex, } diff --git a/crates/slack/src/webhook.rs b/crates/slack/src/webhook.rs index d40af4c12f..82a6813a2b 100644 --- a/crates/slack/src/webhook.rs +++ b/crates/slack/src/webhook.rs @@ -79,6 +79,7 @@ pub async fn register_events_api_account( bot_user_id: Some(bot_user_id), pending_threads: std::collections::HashMap::new(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(otp_cooldown_secs)), + dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), }); } @@ -390,6 +391,10 @@ async fn dispatch_event_callback( if let Some(item) = item { // Extract channel and message_ts from the item. let item_channel = item.get("channel").and_then(|v| v.as_str()).unwrap_or(""); + let item_user = event + .get("item_user") + .and_then(|v| v.as_str()) + .map(String::from); let message_ts = item.get("ts").and_then(|v| v.as_str()).unwrap_or(""); if !user.is_empty() && !reaction.is_empty() && !item_channel.is_empty() { @@ -400,6 +405,7 @@ async fn dispatch_event_callback( reaction, item_channel.to_string(), message_ts.to_string(), + item_user, added, accounts, ) @@ -726,6 +732,7 @@ mod tests { bot_user_id: Some("B123".to_string()), pending_threads: std::collections::HashMap::new(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), }); } @@ -756,6 +763,7 @@ mod tests { bot_user_id: Some("B123".to_string()), pending_threads: std::collections::HashMap::new(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), }); } @@ -791,6 +799,7 @@ mod tests { bot_user_id: Some("B123".to_string()), pending_threads: std::collections::HashMap::new(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), }); } diff --git a/crates/web/ui/src/pages/ChannelsPage.tsx b/crates/web/ui/src/pages/ChannelsPage.tsx index 1873f89a73..666c2418e0 100644 --- a/crates/web/ui/src/pages/ChannelsPage.tsx +++ b/crates/web/ui/src/pages/ChannelsPage.tsx @@ -92,7 +92,6 @@ export interface ChannelConfig { ack_reactions?: boolean; reaction_triggers?: boolean; rich_blocks?: boolean; - assistant_status?: boolean; // Matrix homeserver?: string; user_id?: string; diff --git a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx index 2580788c70..7d36e83363 100644 --- a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx +++ b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx @@ -125,7 +125,9 @@ export function AddSlackModal(): VNode { 2. Under OAuth & Permissions, add bot scopes: chat:write,{" "} channels:history,{" "} im:history,{" "} - app_mentions:read + app_mentions:read,{" "} + reactions:write (acknowledgment reactions),{" "} + reactions:read (only for reaction triggers)
3. Install the app to your workspace and copy the Bot User OAuth Token @@ -135,7 +137,16 @@ export function AddSlackModal(): VNode { connections:write scope
- 5. For Events API: set the Request URL to your server's webhook endpoint + 5. Under Event Subscriptions, subscribe to bot events:{" "} + message.im,{" "} + app_mention, and{" "} + reaction_added if you use reaction triggers +
+
+ 6. For Events API: set the Request URL to{" "} + + https://your-host/api/channels/slack/<account_id>/webhook +
diff --git a/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx b/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx index 5a9f5c6030..c86f7c4c16 100644 --- a/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx +++ b/crates/web/ui/src/pages/channels/modals/EditChannelModal.tsx @@ -50,7 +50,6 @@ export function EditChannelModal(): VNode | null { const editSlackAckReactions = useSignal(true); const editSlackReactionTriggers = useSignal(false); const editSlackRichBlocks = useSignal(false); - const editSlackAssistantStatus = useSignal(false); const editChannelNamePatterns = useSignal([]); const editCategoryAllowlist = useSignal([]); const editAdvancedConfigPatch = useSignal(""); @@ -82,7 +81,6 @@ export function EditChannelModal(): VNode | null { editSlackAckReactions.value = ch?.config?.ack_reactions !== false; editSlackReactionTriggers.value = ch?.config?.reaction_triggers === true; editSlackRichBlocks.value = ch?.config?.rich_blocks === true; - editSlackAssistantStatus.value = ch?.config?.assistant_status === true; editChannelNamePatterns.value = (ch?.config?.channel_name_patterns || []) as string[]; editCategoryAllowlist.value = (ch?.config?.category_allowlist || []) as string[]; editAdvancedConfigPatch.value = ""; @@ -204,7 +202,6 @@ export function EditChannelModal(): VNode | null { updateConfig.ack_reactions = editSlackAckReactions.value; updateConfig.reaction_triggers = editSlackReactionTriggers.value; updateConfig.rich_blocks = editSlackRichBlocks.value; - updateConfig.assistant_status = editSlackAssistantStatus.value; } addChannelCredentials(updateConfig, form); addModelToConfig(updateConfig); @@ -460,22 +457,6 @@ export function EditChannelModal(): VNode | null { - )} {isNostr && ( diff --git a/docs/src/slack.md b/docs/src/slack.md index d50baca821..75cf578532 100644 --- a/docs/src/slack.md +++ b/docs/src/slack.md @@ -107,8 +107,7 @@ offered = ["slack"] | `ack_reactions` | no | `true` | Acknowledge inbound messages with emoji reactions (πŸ‘€ β†’ βœ…/❌). Only applied when the bot is directly addressed (DM or @mention). | | `reaction_triggers` | no | `false` | Route inbound user reactions into the agent as messages (e.g. react βœ… to approve). | | `reaction_trigger_emojis` | no | `[]` | When `reaction_triggers` is on, only these emoji shortcodes trigger the agent. Empty = any emoji. | -| `rich_blocks` | no | `false` | Render replies as Block Kit blocks (headings, dividers, code) with a plain-text fallback. | -| `assistant_status` | no | `false` | Show a live "is thinking…" status via `assistant.threads.setStatus`. Requires an AI/Assistant app. | +| `rich_blocks` | no | `false` | Render replies as Block Kit blocks (headings, dividers, code) with a plain-text fallback. Disables streaming, since streamed text cannot carry blocks. | | `channel_overrides` | no | `{}` | Per-channel model/provider overrides (see below) | | `user_overrides` | no | `{}` | Per-user model/provider overrides (see below) | @@ -285,6 +284,12 @@ Reactions are only added when the bot is **directly addressed** β€” a direct message, or a channel message that @mentions the bot β€” never on general channel chatter. Set `ack_reactions = false` to disable them. +Each inbound message owns its own reaction for its whole life, including while +it waits behind an in-flight turn: a message queued behind another keeps its πŸ‘€ +and is resolved when its own turn runs. In `collect` queue mode one reply +answers several messages, and every one of them receives the terminal reaction. +A reply that fails to deliver is marked ❌ rather than βœ…. + The bot needs the `reactions:write` OAuth scope (and `reactions:read` if you also use inbound reaction triggers). Reaction failures are non-fatal and never block the reply. @@ -300,6 +305,8 @@ flows like "react βœ… to approve" or "react πŸ‘ to continue". triggers never loop. - Only *added* reactions trigger (not removals), and only from senders who pass the normal DM/channel access policy. +- Only reactions on the **bot's own messages** trigger. Without this, any member + could point the agent at an unrelated person's message just by reacting to it. - Restrict which emoji count with `reaction_trigger_emojis` (shortcodes without colons, e.g. `["white_check_mark", "thumbsup"]`). Empty means any emoji. From 5d5386abb4c3388a16f3664ccf247ad25081c630 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Sat, 25 Jul 2026 01:19:08 -0700 Subject: [PATCH 10/23] test(slack): assert jitter is two-sided; resolve acks for dropped queued messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the Greptile review, which flagged both spots. The jitter bounds test asserted 0.5x-1.5x while the implementation produces 0.75x-1.25x, so it would have passed against the original one-sided bug it was meant to guard. Tighten it to the documented range and extract the pure factor so a second test can assert the distribution reaches both sides of the base delay directly, instead of sampling the clock and hoping. Collect-mode replay returned early when nothing was replayable β€” every queued message non-text, or an empty queue β€” abandoning those messages without resolving their acknowledgments, so their πŸ‘€ lingered until the lifetime cap. Those paths now finalize the acknowledgments, since no reply is coming. --- .../chat/src/service/chat_impl/queue_drain.rs | 15 +++++++++ crates/slack/src/socket.rs | 31 ++++++++++++++++--- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs index 8f77322cf1..dbc7f944ec 100644 --- a/crates/chat/src/service/chat_impl/queue_drain.rs +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -67,6 +67,10 @@ pub(super) async fn drain_and_replay( .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) .collect(); if combined.is_empty() { + // Nothing replayable (e.g. every queued message was non-text), + // so no reply will ever come. Resolve their acknowledgments + // instead of leaving markers to expire. + abandon(state, &queued).await; return; } info!( @@ -76,6 +80,7 @@ pub(super) async fn drain_and_replay( ); // Use the last queued message as the base params, override text. let Some(last) = queued.last() else { + abandon(state, &queued).await; return; }; let mut merged = last.params.clone(); @@ -91,3 +96,13 @@ pub(super) async fn drain_and_replay( }, } } + +/// Resolve the acknowledgments of queued messages that will never be answered. +async fn abandon(state: &Arc, queued: &[QueuedMessage]) { + let keys = crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)); + if !keys.is_empty() { + state + .finalize_channel_acks(keys, moltis_channels::ChannelAckOutcome::Failure) + .await; + } +} diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index 70379dbd34..d4234c601b 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -238,10 +238,17 @@ fn jittered(base: std::time::Duration) -> std::time::Duration { // Nanoseconds within the current second map to [0, 1), then to a // symmetric +/-25% factor. Dividing by anything but NANOS_PER_SEC would // bias the jitter to one side. + base.mul_f64(jitter_factor(nanos)) +} + +/// Map sub-second nanoseconds onto a symmetric +/-25% delay factor. +/// +/// Split out from [`jittered`] so the distribution can be asserted directly +/// rather than sampled from the clock. +fn jitter_factor(nanos: u32) -> f64 { const NANOS_PER_SEC: f64 = 1_000_000_000.0; let unit = f64::from(nanos) / NANOS_PER_SEC; - let factor = 0.75 + unit * 0.5; - base.mul_f64(factor.clamp(0.75, 1.25)) + (0.75 + unit * 0.5).clamp(0.75, 1.25) } /// Error handler for Socket Mode. @@ -1165,13 +1172,27 @@ mod tests { fn jitter_stays_within_bounds() { use std::time::Duration; let base = Duration::from_secs(10); - for _ in 0..100 { + for _ in 0..200 { let j = jittered(base); - assert!(j >= base.mul_f64(0.5), "jitter {j:?} below lower bound"); - assert!(j <= base.mul_f64(1.5), "jitter {j:?} above upper bound"); + // Exactly the documented +/-25%. Looser bounds would also accept a + // one-sided distribution, which is the bug this guards against. + assert!(j >= base.mul_f64(0.75), "jitter {j:?} below lower bound"); + assert!(j <= base.mul_f64(1.25), "jitter {j:?} above upper bound"); } } + #[test] + fn jitter_is_two_sided() { + // The factor must be able to land on both sides of the base delay; + // dividing nanoseconds by the wrong constant silently made it + // always-negative, which no range assertion alone would catch. + assert!(jitter_factor(0) < 1.0, "minimum should shorten the delay"); + assert!( + jitter_factor(999_999_999) > 1.0, + "maximum should lengthen the delay" + ); + } + #[test] fn ack_target_dm_is_acknowledged() { assert_eq!( From 3caded117f661f3a37a45b677eb961b65de4fad3 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Sat, 25 Jul 2026 10:19:44 -0700 Subject: [PATCH 11/23] fix(slack): let Slack callbacks reach their HMAC check, and settle stranded acks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Events API was unusable whenever gateway auth was on. is_public_path exempted /api/channels/msteams/ but never Slack, so Slack's callbacks to /events, /interactions and /commands were rejected with 401 before their handlers β€” and therefore before signature verification β€” could run. Only those three exact suffixes are exempted, and they remain protected by Slack's HMAC and timestamp checks; anything else under the Slack namespace stays authenticated. A test pins both halves of that boundary. The setup modal also advertised /webhook, which is not a route. It and the docs now list the real Event Subscriptions, Interactivity and Slash Commands URLs. Acknowledgments that could previously linger until the lifetime cap: - A replay that fails, or returns a terminal payload with no run behind it, now settles its keys instead of leaving them owned by a run that never started. - Cancelling a queued message clears its marker. - A controller displaced by a duplicate registration or by the pending cap is finalized rather than dropped; dropping the handle closed the worker channel, whose exit path deliberately leaves the message untouched. - finalize_keys now also drops active-turn ownership, so a later terminal cannot address controllers that are already resolved. Block Kit fence parsing is delimiter-aware: previously any line starting with three backticks toggled the fence, so a longer fence used as content truncated the block, and trailing newlines inside a snippet were stripped. Closing now requires a bare run of at least the opening length, and only the parser's own appended newline is removed. Round-trip tests cover nested fences, info strings and interior blank lines. --- crates/chat/src/service/chat_impl.rs | 13 +++ .../chat/src/service/chat_impl/queue_drain.rs | 46 ++++++++-- crates/gateway/src/channel_reactions.rs | 58 ++++++++++-- crates/httpd/src/auth_middleware.rs | 59 ++++++++++++ crates/slack/src/blocks.rs | 89 ++++++++++++++++--- .../pages/channels/modals/AddSlackModal.tsx | 11 ++- docs/src/slack.md | 14 +++ 7 files changed, 261 insertions(+), 29 deletions(-) diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index c89fc3a668..f95e5cfa06 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -426,6 +426,19 @@ impl ChatService for LiveChatService { let count = removed.len(); info!(session = %session_key, count, "cancel_queued: cleared message queue"); + // These messages will never be answered, so clear their channel + // acknowledgment markers rather than leaving them to expire. + let cancelled_acks = + crate::channel_acks::merged_ack_keys(removed.iter().map(|m| &m.params)); + if !cancelled_acks.is_empty() { + self.state + .finalize_channel_acks( + cancelled_acks, + moltis_channels::ChannelAckOutcome::Cancelled, + ) + .await; + } + broadcast( &self.state, "chat", diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs index dbc7f944ec..0a6752311d 100644 --- a/crates/chat/src/service/chat_impl/queue_drain.rs +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -55,11 +55,11 @@ pub(super) async fn drain_and_replay( .extend(rest); } info!(session = %session_key, "replaying queued message (followup)"); + let keys = crate::channel_acks::ack_keys_from_params(&first.params); let mut replay_params = first.params; replay_params["_queued_replay"] = Value::Bool(true); - if let Err(e) = chat.send(replay_params).await { - warn!(session = %session_key, error = %e, "failed to replay queued message"); - } + let result = chat.send(replay_params).await; + settle_replay(state, session_key, keys, &result).await; }, MessageQueueMode::Collect => { let combined: Vec<&str> = queued @@ -90,10 +90,46 @@ pub(super) async fn drain_and_replay( merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) ); - if let Err(e) = chat.send(merged).await { - warn!(session = %session_key, error = %e, "failed to replay collected messages"); + let keys = crate::channel_acks::ack_keys_from_params(&merged); + let result = chat.send(merged).await; + settle_replay(state, session_key, keys, &result).await; + }, + } +} + +/// Resolve acknowledgments for a replay that never became a run. +/// +/// A replay can fail outright, or return a terminal payload (a rejected hook, +/// an error) with no run behind it. Either way nothing else will finalize the +/// reaction, so it is settled here. A replay that queues again keeps its +/// acknowledgment: it will be claimed when it finally runs. +async fn settle_replay( + state: &Arc, + session_key: &str, + keys: Vec, + result: &moltis_service_traits::ServiceResult, +) { + let failed = match result { + Err(e) => { + warn!(session = %session_key, error = %e, "failed to replay queued message"); + true + }, + Ok(payload) => { + if payload.get("queued").and_then(Value::as_bool) == Some(true) { + false + } else { + payload.get("rejected").and_then(Value::as_bool) == Some(true) + || matches!( + payload.get("state").and_then(Value::as_str), + Some("rejected" | "error" | "blocked") + ) } }, + }; + if failed && !keys.is_empty() { + state + .finalize_channel_acks(keys, moltis_channels::ChannelAckOutcome::Failure) + .await; } } diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index d47ec39acc..6f0443d296 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -97,14 +97,33 @@ pub struct ReactionRegistry { impl ReactionRegistry { /// Park a freshly received message's controller until a run claims it. + /// + /// A controller displaced by a duplicate registration or by the pending cap + /// is finalized rather than dropped: dropping its handle would close the + /// worker's channel, which exits without touching the message and would + /// leave πŸ‘€ on it until the lifetime cap. pub async fn register_pending(&self, key: String, controller: Arc) { - let mut inner = self.inner.lock().await; - if inner.pending.insert(key.clone(), controller).is_none() { - inner.pending_order.push(key); - } - while inner.pending_order.len() > MAX_PENDING_ACKS { - let oldest = inner.pending_order.remove(0); - inner.pending.remove(&oldest); + let displaced = { + let mut inner = self.inner.lock().await; + let mut displaced = Vec::new(); + if let Some(previous) = inner.pending.insert(key.clone(), controller) { + displaced.push(previous); + } else { + inner.pending_order.push(key.clone()); + } + // Evict oldest entries that no active turn still owns. + while inner.pending_order.len() > MAX_PENDING_ACKS { + let oldest = inner.pending_order.remove(0); + if let Some(evicted) = inner.pending.remove(&oldest) { + displaced.push(evicted); + } + } + displaced + }; + for controller in displaced { + controller + .note(ChannelActivity::Finished(ChannelAckOutcome::Cancelled)) + .await; } } @@ -172,6 +191,11 @@ impl ReactionRegistry { pub async fn finalize_keys(&self, keys: &[String], outcome: ChannelAckOutcome) { let controllers = { let mut inner = self.inner.lock().await; + // Drop any active-turn ownership of these keys too, so a later + // terminal cannot address controllers that are already resolved. + inner + .active + .retain(|_, turn| !turn.keys.iter().any(|tk| keys.contains(tk))); keys.iter() .filter_map(|k| { inner.pending_order.retain(|pk| pk != k); @@ -657,6 +681,26 @@ mod tests { assert_eq!(ops_of(&a), vec!["+πŸ‘€", "+❌", "-πŸ‘€"]); } + #[tokio::test] + async fn duplicate_registration_resolves_the_displaced_controller() { + // A retry of the same message must not strand the first πŸ‘€. + let registry = ReactionRegistry::default(); + let first = park(®istry, "msg-a").await; + let second_ops = Arc::new(StdMutex::new(Vec::new())); + let outbound = Arc::new(RecordingOutbound { + ops: second_ops.clone(), + }); + let replacement = + ChannelReactionController::start(outbound, "a".into(), "c".into(), "msg-a".into()); + registry + .register_pending("msg-a".to_string(), replacement) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + // Displaced controller cleared its marker instead of leaking it. + assert_eq!(ops_of(&first), vec!["+πŸ‘€", "-πŸ‘€"]); + assert_eq!(ops_of(&second_ops), vec!["+πŸ‘€"]); + } + #[tokio::test] async fn activity_for_unknown_session_is_a_noop() { let registry = ReactionRegistry::default(); diff --git a/crates/httpd/src/auth_middleware.rs b/crates/httpd/src/auth_middleware.rs index 7c9a824305..612041ce5b 100644 --- a/crates/httpd/src/auth_middleware.rs +++ b/crates/httpd/src/auth_middleware.rs @@ -237,6 +237,24 @@ pub async fn auth_gate( /// Paths that never require authentication. #[cfg(feature = "web-ui")] +/// Exactly the three Slack callback endpoints, and nothing else under the +/// Slack namespace. +/// +/// Matching the whole `/api/channels/slack/` prefix would also expose any +/// future management route added there, so the suffix is checked explicitly. +#[cfg(feature = "slack")] +fn is_slack_callback_path(path: &str) -> bool { + let Some(rest) = path.strip_prefix("/api/channels/slack/") else { + return false; + }; + matches!( + rest.split_once('/'), + Some((account_id, suffix)) + if !account_id.is_empty() + && matches!(suffix, "events" | "interactions" | "commands") + ) +} + fn is_public_path(path: &str) -> bool { matches!( path, @@ -259,6 +277,20 @@ fn is_public_path(path: &str) -> bool { false } } + // Slack calls these endpoints itself and cannot present a Moltis + // session, so gateway auth would reject every callback before the + // handler runs. They are not unauthenticated: each verifies Slack's + // HMAC signature (and timestamp freshness) before doing any work. + || { + #[cfg(feature = "slack")] + { + is_slack_callback_path(path) + } + #[cfg(not(feature = "slack"))] + { + false + } + } || path.starts_with("/api/webhooks/ingest/") || path.starts_with("/assets/") || path.starts_with("/share/") @@ -431,6 +463,33 @@ pub fn parse_cookie<'a>(header: &'a str, name: &str) -> Option<&'a str> { #[cfg(test)] mod tests { + + #[cfg(feature = "slack")] + #[test] + fn slack_callback_paths_bypass_gateway_auth_but_nothing_else_does() { + // Slack cannot present a Moltis session; these three verify Slack's + // HMAC signature themselves. + for p in [ + "/api/channels/slack/my-bot/events", + "/api/channels/slack/my-bot/interactions", + "/api/channels/slack/my-bot/commands", + ] { + assert!( + is_public_path(p), + "{p} must reach its HMAC-verifying handler" + ); + } + // Everything else in the namespace stays authenticated. + for p in [ + "/api/channels/slack/my-bot/config", + "/api/channels/slack/my-bot", + "/api/channels/slack//events", + "/api/channels/slack/my-bot/events/extra", + "/api/channels/slack", + ] { + assert!(!is_public_path(p), "{p} must stay authenticated"); + } + } use {super::*, sqlx::SqlitePool}; #[test] diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index 94489eb936..6127b96df5 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -24,7 +24,8 @@ const MAX_HEADER_CHARS: usize = 150; pub fn markdown_to_blocks(markdown: &str) -> Option> { let mut blocks: Vec = Vec::new(); let mut paragraph = String::new(); - let mut code: Option = None; + // Accumulated fenced-code body plus the backtick count that opened it. + let mut code: Option<(String, usize)> = None; let flush_paragraph = |paragraph: &mut String, blocks: &mut Vec| { let trimmed = paragraph.trim(); @@ -37,23 +38,28 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { }; for line in markdown.lines() { - // Fenced code block toggling. - if line.trim_start().starts_with("```") { - match code.take() { - Some(buf) => { - // Closing fence: emit the accumulated code as fenced section(s). - push_code_blocks(&buf, &mut blocks); + // Fence handling. A closing fence must be a bare run of at least as + // many backticks as the opener; anything else (a longer fence used as + // content, or ```rust) is body text. Treating every ```-prefixed line + // as a toggle silently truncated code blocks that contained one. + if let Some(ticks) = opening_fence_len(line) { + match &code { + Some((buf, open_ticks)) => { + if is_closing_fence(line, *open_ticks) { + push_code_blocks(buf, &mut blocks); + code = None; + continue; + } }, None => { - // Opening fence: flush any pending paragraph first. flush_paragraph(&mut paragraph, &mut blocks); - code = Some(String::new()); + code = Some((String::new(), ticks)); + continue; }, } - continue; } - if let Some(buf) = code.as_mut() { + if let Some((buf, _)) = code.as_mut() { buf.push_str(line); buf.push('\n'); continue; @@ -97,7 +103,7 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { } // Flush trailing state. - if let Some(buf) = code.take() { + if let Some((buf, _)) = code.take() { // Unterminated fence: still emit what we have. push_code_blocks(&buf, &mut blocks); } @@ -110,6 +116,21 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { Some(blocks) } +/// Backtick count if this line opens a fence (three or more backticks, +/// optionally followed by an info string such as ```rust). +fn opening_fence_len(line: &str) -> Option { + let trimmed = line.trim_start(); + let ticks = trimmed.chars().take_while(|c| *c == '`').count(); + (ticks >= 3).then_some(ticks) +} + +/// A fence closes a block only if it is a bare run of at least `open_ticks` +/// backticks with no trailing content. +fn is_closing_fence(line: &str, open_ticks: usize) -> bool { + let trimmed = line.trim(); + trimmed.len() >= open_ticks && trimmed.chars().all(|c| c == '`') +} + /// Extract heading text from an ATX heading line (`# …` … `###### …`). fn heading_text(line: &str) -> Option { let hashes = line.chars().take_while(|c| *c == '#').count(); @@ -143,7 +164,10 @@ fn push_code_blocks(code: &str, blocks: &mut Vec) { // Reserve room for the fence delimiters ("```\n" + "\n```") in each chunk. let overhead = "```\n\n```".chars().count(); let body_limit = MAX_SECTION_CHARS.saturating_sub(overhead).max(1); - for piece in split_by_limit(code.trim_end_matches('\n'), body_limit) { + // Only the single newline the parser appends per line is dropped; further + // trailing blank lines are part of the snippet. + let body = code.strip_suffix('\n').unwrap_or(code); + for piece in split_by_limit(body, body_limit) { blocks.push(section_block(&format!("```\n{piece}\n```"))); } } @@ -234,6 +258,45 @@ mod tests { assert!(rendered.contains(&heading), "heading content was lost"); } + #[test] + fn code_block_containing_a_longer_fence_is_not_truncated() { + // A ```` run inside a ``` block is content, not a terminator. + let md = "```\nbefore\n````\nafter\n```"; + let blocks = markdown_to_blocks(md).unwrap(); + let rendered: String = blocks + .iter() + .filter_map(|b| b["text"]["text"].as_str()) + .collect(); + assert!(rendered.contains("before"), "opening content lost"); + assert!(rendered.contains("after"), "content after inner fence lost"); + } + + #[test] + fn info_string_fence_round_trips_content() { + let md = "```rust\nlet x = 1;\n```"; + let blocks = markdown_to_blocks(md).unwrap(); + let rendered: String = blocks + .iter() + .filter_map(|b| b["text"]["text"].as_str()) + .collect(); + assert!(rendered.contains("let x = 1;"), "code body lost"); + assert_eq!(rendered.matches("```").count(), 2, "unbalanced fences"); + } + + #[test] + fn interior_blank_lines_in_code_are_preserved() { + let md = "```\na\n\n\nb\n```"; + let blocks = markdown_to_blocks(md).unwrap(); + let rendered: String = blocks + .iter() + .filter_map(|b| b["text"]["text"].as_str()) + .collect(); + assert!( + rendered.contains("a\n\n\nb"), + "blank lines collapsed: {rendered:?}" + ); + } + #[test] fn over_limit_falls_back_to_none() { // Many dividers exceed the 50-block cap β†’ fall back to plain text. diff --git a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx index 7d36e83363..3085da9bb5 100644 --- a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx +++ b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx @@ -143,10 +143,13 @@ export function AddSlackModal(): VNode { reaction_added if you use reaction triggers
- 6. For Events API: set the Request URL to{" "} - - https://your-host/api/channels/slack/<account_id>/webhook - + 6. For Events API, set these Request URLs (replace{" "} + <id> with the Account ID below): Event Subscriptions{" "} + https://your-host/api/channels/slack/<id>/events, + Interactivity{" "} + https://your-host/api/channels/slack/<id>/interactions + , Slash Commands{" "} + https://your-host/api/channels/slack/<id>/commands
diff --git a/docs/src/slack.md b/docs/src/slack.md index 75cf578532..c49b9909af 100644 --- a/docs/src/slack.md +++ b/docs/src/slack.md @@ -153,6 +153,20 @@ model_provider = "anthropic" ### Events API Mode +Set these Request URLs in your Slack app (replace `` with the account ID you +configured in Moltis): + +| Slack setting | URL | +|---------------|-----| +| Event Subscriptions | `https://your-host/api/channels/slack//events` | +| Interactivity & Shortcuts | `https://your-host/api/channels/slack//interactions` | +| Slash Commands | `https://your-host/api/channels/slack//commands` | + +These endpoints are reachable without a Moltis session β€” Slack cannot present +one β€” and instead verify Slack's request signature (HMAC) and timestamp before +doing any work. + + If you prefer webhook-based delivery instead of Socket Mode: ```toml From c5868da568eaf3bab8678fa4a295d73faa763054 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Sat, 25 Jul 2026 10:22:36 -0700 Subject: [PATCH 12/23] fix(slack): surface final streaming delivery failures instead of swallowing them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A streamed reply whose final flush, stop, update or overflow post failed still returned Ok(()). The dispatcher then recorded the target as delivered, skipped the plain-text fallback, and the acknowledgment settled on βœ… β€” for a reply the user never fully received. Those final calls now propagate. The caller already treats a stream error as 'not delivered', so the reply falls back to a normal send and the acknowledgment reflects what actually happened. Mid-stream throttled edits stay best-effort, since a later edit supersedes them. --- crates/slack/src/outbound.rs | 49 +++++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 49c644f639..4a660a6aa8 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -178,20 +178,21 @@ impl SlackOutbound { } } - // Flush any remaining text. + // Flush any remaining text. A failure here means the tail of the reply + // never reached the user, so it must surface: the caller then treats + // the target as undelivered and falls back to a plain send, and the + // acknowledgment cannot claim success. if !pending.is_empty() { let text = markdown_to_slack(&pending); - if let Err(e) = - append_native_stream(&http, &api_base_url, &bot_token, &stream_id, &text).await - { - warn!(account_id, to, "final chat.appendStream failed: {e}"); - } + append_native_stream(&http, &api_base_url, &bot_token, &stream_id, &text) + .await + .inspect_err(|e| warn!(account_id, to, "final chat.appendStream failed: {e}"))?; } - // Finalize the stream. - if let Err(e) = stop_native_stream(&http, &api_base_url, &bot_token, &stream_id).await { - warn!(account_id, to, "chat.stopStream failed: {e}"); - } + // Finalize the stream. A stream left open renders as incomplete. + stop_native_stream(&http, &api_base_url, &bot_token, &stream_id) + .await + .inspect_err(|e| warn!(account_id, to, "chat.stopStream failed: {e}"))?; Ok(()) } @@ -284,24 +285,32 @@ impl SlackOutbound { let final_text = markdown_to_slack(&accumulated); let chunks = chunk_message(&final_text, SLACK_MAX_MESSAGE_LEN); + // Final delivery failures are returned, not swallowed: the caller uses + // the result to decide whether the reply actually landed. match &sent_ts { Some(ts) => { - if let Some(first) = chunks.first() - && let Err(e) = update_message(&client, &token, to, ts, first).await - { - warn!(account_id, to, "failed to finalize stream message: {e}"); + if let Some(first) = chunks.first() { + update_message(&client, &token, to, ts, first) + .await + .inspect_err(|e| { + warn!(account_id, to, "failed to finalize stream message: {e}"); + })?; } for chunk in chunks.iter().skip(1) { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send overflow chunk: {e}"); - } + post_message(&client, &token, to, chunk, thread_ts) + .await + .inspect_err(|e| { + warn!(account_id, to, "failed to send overflow chunk: {e}") + })?; } }, None => { for chunk in &chunks { - if let Err(e) = post_message(&client, &token, to, chunk, thread_ts).await { - warn!(account_id, to, "failed to send stream message: {e}"); - } + post_message(&client, &token, to, chunk, thread_ts) + .await + .inspect_err(|e| { + warn!(account_id, to, "failed to send stream message: {e}") + })?; } }, } From 8db430d8cceeea7422ba40472347ffa3214ed040 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Sat, 25 Jul 2026 12:15:09 -0700 Subject: [PATCH 13/23] fix(chat): make abort clean up its own turn, and stop collect dropping content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Abort could resolve the wrong turn. AbortHandle::abort() kills the run future outright, so it never emits its own terminal; the abort handler emitted one afterwards, addressed by session. Because killing the task also drops the session permit, a new turn could activate in between and receive the old cancellation. The handler now takes ownership of the active turn's acknowledgments *before* aborting, via a registry call that removes them under the lock β€” turn-addressed by construction, so a turn starting later cannot be caught by it. Abort also skipped queue draining entirely: the killed future never reached the drain step at its end, so anything queued behind it waited forever. The handler now drains after the aborted task has released the session permit (waiting for it first, or the replay would just re-queue itself). Collect mode silently discarded content. It merged only the "text" field while attaching acknowledgment keys from every queued message, and send() prefers "content" over "text" β€” so one multimodal message anywhere in the batch dropped every other message's words while still reporting them all successful. All messages are now flattened into a single ordered block list, preserving images and text alike. Also: - The terminal reaction retries once. It is the last thing the worker does, so unlike an intermediate phase there is no later transition to correct it. - Losing the outbound entirely, or a panicked delivery task, no longer settles as a success. - Slack interactions are deduplicated by trigger id, so a retried envelope cannot run a button's action twice. --- crates/chat/src/channels.rs | 3 + crates/chat/src/runtime.rs | 11 +++ crates/chat/src/service/chat_impl.rs | 35 +++++-- .../chat/src/service/chat_impl/queue_drain.rs | 97 +++++++++++++++++++ crates/gateway/src/channel_reactions.rs | 39 +++++++- crates/gateway/src/chat.rs | 11 +++ crates/slack/src/socket.rs | 20 ++++ 7 files changed, 205 insertions(+), 11 deletions(-) diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index a86a5646c9..b0470d3091 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -125,6 +125,7 @@ pub(crate) async fn deliver_channel_replies( "channel reply delivery skipped: outbound unavailable" ); } + crate::channel_acks::note_delivery_failed(state, session_key).await; return; }, }; @@ -701,6 +702,8 @@ async fn deliver_channel_replies_to_targets( for task in tasks { if let Err(e) = task.await { warn!(error = %e, "channel reply task join failed"); + // A panicked task may have sent nothing β€” do not claim success. + crate::channel_acks::note_delivery_failed(&state, &session_key).await; } } } diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index 8991a73532..f9d54a7ca6 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -148,6 +148,17 @@ pub trait ChatRuntime: Send + Sync { /// Finalize acknowledgment keys directly, for paths where no run executes /// (rejected hook, early error) and nothing else will signal a terminal. + /// Finalize whichever turn is currently active for this session. + /// + /// For abort, where the run future is killed and cannot emit its own + /// terminal. Turn-addressed, so it cannot resolve a turn that starts later. + async fn finalize_active_channel_acks( + &self, + _session_key: &str, + _outcome: moltis_channels::ChannelAckOutcome, + ) { + } + async fn finalize_channel_acks( &self, _ack_keys: Vec, diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index f95e5cfa06..64b3886899 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -357,6 +357,16 @@ impl ChatService for LiveChatService { Self::resolve_session_key_for_run(&self.active_runs_by_session, run_id, session_key) .await; + // Take ownership of this turn's acknowledgments *before* killing the + // run. Aborting drops the task and with it the session permit, so a new + // turn can activate immediately; finalizing afterwards could resolve + // that newer turn instead of the one being aborted. + if let Some(key) = resolved_session_key.as_deref() { + self.state + .finalize_active_channel_acks(key, moltis_channels::ChannelAckOutcome::Cancelled) + .await; + } + let (resolved_run_id, aborted) = Self::abort_run_handle( &self.active_runs, &self.active_runs_by_session, @@ -391,17 +401,22 @@ impl ChatService for LiveChatService { } } broadcast(&self.state, "chat", payload, BroadcastOpts::default()).await; + } - // Finalize channel acknowledgment reactions: cancelled leaves no - // βœ…/❌ terminal, only strips the in-progress marker. - self.state - .note_channel_activity( - key, - moltis_channels::ChannelActivity::Finished( - moltis_channels::ChannelAckOutcome::Cancelled, - ), - ) - .await; + // The aborted future was killed mid-flight, so it never ran its own + // drain step. Without this, anything queued behind it waits forever. + if aborted && let Some(key) = resolved_session_key.clone() { + let queue = Arc::clone(&self.message_queue); + let state = Arc::clone(&self.state); + let mode = self.config.chat.message_queue_mode; + let session_sem = self.session_semaphore(&key).await; + tokio::spawn(async move { + // Wait for the aborted task to release the session permit, + // otherwise the replay would simply re-queue itself. + let permit = session_sem.acquire_owned().await; + drop(permit); + queue_drain::drain_and_replay(&queue, &key, mode, &state).await; + }); } Ok(serde_json::json!({ diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs index 0a6752311d..7f3a91724f 100644 --- a/crates/chat/src/service/chat_impl/queue_drain.rs +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -62,10 +62,46 @@ pub(super) async fn drain_and_replay( settle_replay(state, session_key, keys, &result).await; }, MessageQueueMode::Collect => { + // Merge *all* content, not just text. `send` prefers `content` over + // `text` when both are present, so a multimodal message anywhere in + // the batch would otherwise discard every other message's words + // while their acknowledgments still reported success. + let multimodal = queued.iter().any(|m| m.params.get("content").is_some()); let combined: Vec<&str> = queued .iter() .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) .collect(); + if multimodal { + let blocks = merge_content_blocks(&queued); + if blocks.is_empty() { + abandon(state, &queued).await; + return; + } + let Some(last) = queued.last() else { + abandon(state, &queued).await; + return; + }; + info!( + session = %session_key, + count = queued.len(), + "replaying collected messages (multimodal)" + ); + let mut merged = last.params.clone(); + merged["content"] = Value::Array(blocks); + // `text` would be ignored anyway; drop it so the two cannot + // disagree about what was actually sent. + if let Some(obj) = merged.as_object_mut() { + obj.remove("text"); + } + merged["_queued_replay"] = Value::Bool(true); + merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( + crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) + ); + let keys = crate::channel_acks::ack_keys_from_params(&merged); + let result = chat.send(merged).await; + settle_replay(state, session_key, keys, &result).await; + return; + } if combined.is_empty() { // Nothing replayable (e.g. every queued message was non-text), // so no reply will ever come. Resolve their acknowledgments @@ -97,6 +133,27 @@ pub(super) async fn drain_and_replay( } } +/// Flatten every queued message into one ordered list of content blocks. +/// +/// Text-only messages contribute a single text block; multimodal messages +/// contribute their blocks verbatim, so images survive the merge. +fn merge_content_blocks(queued: &[QueuedMessage]) -> Vec { + let mut blocks = Vec::new(); + for m in queued { + match m.params.get("content").and_then(Value::as_array) { + Some(content) => blocks.extend(content.iter().cloned()), + None => { + if let Some(text) = m.params.get("text").and_then(Value::as_str) + && !text.is_empty() + { + blocks.push(serde_json::json!({ "type": "text", "text": text })); + } + }, + } + } + blocks +} + /// Resolve acknowledgments for a replay that never became a run. /// /// A replay can fail outright, or return a terminal payload (a rejected hook, @@ -142,3 +199,43 @@ async fn abandon(state: &Arc, queued: &[QueuedMessage]) { .await; } } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn msg(v: Value) -> QueuedMessage { + QueuedMessage { params: v } + } + + #[test] + fn merges_text_and_image_blocks_in_order() { + let queued = vec![ + msg(serde_json::json!({ "text": "first" })), + msg(serde_json::json!({ "content": [ + { "type": "text", "text": "second" }, + { "type": "image_url", "image_url": { "url": "data:image/png;base64,AAA" } } + ]})), + msg(serde_json::json!({ "text": "third" })), + ]; + let blocks = merge_content_blocks(&queued); + // Nothing is dropped: three texts plus the image. + assert_eq!(blocks.len(), 4); + assert_eq!(blocks[0]["text"], "first"); + assert_eq!(blocks[1]["text"], "second"); + assert_eq!(blocks[2]["type"], "image_url"); + assert_eq!(blocks[3]["text"], "third"); + } + + #[test] + fn skips_empty_text_messages() { + let queued = vec![ + msg(serde_json::json!({ "text": "" })), + msg(serde_json::json!({ "text": "kept" })), + ]; + let blocks = merge_content_blocks(&queued); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0]["text"], "kept"); + } +} diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index 6f0443d296..0bed3d1dc3 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -46,6 +46,9 @@ const STALL_AFTER: Duration = Duration::from_secs(20); /// panicked task that skips the terminal signal) can't leave the worker task β€” /// and the πŸ‘€ reaction β€” alive forever. const MAX_LIFETIME: Duration = Duration::from_secs(900); +/// Pause before the single terminal-reaction retry. The terminal is the last +/// thing the worker does, so losing it leaves a permanently stale marker. +const TERMINAL_RETRY_DELAY: Duration = Duration::from_millis(400); /// Opaque per-message acknowledgment identity: `account:chat:message_id`. /// @@ -185,6 +188,31 @@ impl ReactionRegistry { } } + /// Finalize whichever turn is active for this session *right now*. + /// + /// Used by abort, where the run future is killed and cannot signal its own + /// terminal. Taking the turn under the lock makes this turn-addressed: a + /// turn that starts afterwards can never be caught by this call, which a + /// session-addressed terminal sent after the fact could. + pub async fn finalize_active(&self, session_key: &str, outcome: ChannelAckOutcome) { + let controllers = { + let mut inner = self.inner.lock().await; + let Some(turn) = inner.active.remove(session_key) else { + return; + }; + turn.keys + .iter() + .filter_map(|k| { + inner.pending_order.retain(|pk| pk != k); + inner.pending.remove(k) + }) + .collect::>() + }; + for controller in controllers { + controller.note(ChannelActivity::Finished(outcome)).await; + } + } + /// Finalize specific acknowledgments directly, for paths where no run ever /// executes (a rejected hook, an early error) and nothing will signal a /// terminal for them. @@ -356,8 +384,17 @@ async fn run_worker( // one so the message is never momentarily bare, and a failed // add still leaves the previous marker rather than nothing. Some(term) => { - let landed = + // The worker exits immediately after this, so a failed + // terminal is permanent β€” unlike an intermediate phase, + // which the next transition would supersede. Retry once. + let mut landed = add_reaction(&outbound, &account_id, &chat_id, &message_id, term).await; + if !landed { + tokio::time::sleep(TERMINAL_RETRY_DELAY).await; + landed = + add_reaction(&outbound, &account_id, &chat_id, &message_id, term) + .await; + } let stale = current.take().filter(|cur| cur != term); if let Some(cur) = stale.filter(|_| landed) { remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur) diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index 2d7ec6e175..96abf5f76b 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -72,6 +72,17 @@ impl ChatRuntime for GatewayChatRuntime { .await; } + async fn finalize_active_channel_acks( + &self, + session_key: &str, + outcome: moltis_channels::ChannelAckOutcome, + ) { + self.state + .channel_reaction_controllers + .finalize_active(session_key, outcome) + .await; + } + async fn finalize_channel_acks( &self, ack_keys: Vec, diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index d4234c601b..0254904fc4 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -442,6 +442,26 @@ async fn interaction_events_callback( }; drop(guard); + // Interactions are retried like events, and a repeat would run the button's + // action twice. Dedup on the trigger id, which is unique per interaction. + if let SlackInteractionEvent::BlockActions(ref ba) = event { + let trigger = ba.trigger_id.as_ref(); + let accts = listener_state + .accounts + .read() + .unwrap_or_else(|e| e.into_inner()); + if let Some(state) = accts.get(&listener_state.account_id) { + let mut dedup = state.dedup.lock().unwrap_or_else(|e| e.into_inner()); + if !dedup.insert_new(trigger) { + debug!( + account_id = %listener_state.account_id, + "dropping duplicate slack interaction (retry)" + ); + return Ok(()); + } + } + } + // Extract the action_id from block_actions interaction type. let (action_id, channel_id) = match &event { SlackInteractionEvent::BlockActions(ba) => { From f2509e26e072b6919d5dc73eec052e59969550db Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 11:51:30 -0700 Subject: [PATCH 14/23] Move ACP selection into the chat model picker (#1171) --- crates/web/ui/e2e/specs/agents.spec.js | 393 ++++++++++++++---- crates/web/ui/e2e/specs/chat-input.spec.js | 2 +- .../web/ui/e2e/specs/reasoning-toggle.spec.js | 6 +- .../web/ui/src/components/SessionHeader.tsx | 153 ------- crates/web/ui/src/models.ts | 266 +++++++++--- crates/web/ui/src/onboarding-view.tsx | 2 +- crates/web/ui/src/onboarding/types.ts | 10 +- crates/web/ui/src/pages/ChatPage.tsx | 4 +- crates/web/ui/src/reasoning-toggle.ts | 10 +- crates/web/ui/src/stores/model-store.ts | 15 +- crates/web/ui/src/types/external-agent.ts | 8 + crates/web/ui/src/types/index.ts | 2 + crates/web/ui/src/types/rpc-methods.ts | 3 +- docs/src/external-agents.md | 14 +- 14 files changed, 582 insertions(+), 306 deletions(-) create mode 100644 crates/web/ui/src/types/external-agent.ts diff --git a/crates/web/ui/e2e/specs/agents.spec.js b/crates/web/ui/e2e/specs/agents.spec.js index b8cef3e524..cb83134790 100644 --- a/crates/web/ui/e2e/specs/agents.spec.js +++ b/crates/web/ui/e2e/specs/agents.spec.js @@ -46,50 +46,126 @@ async function deleteAgentByName(page, agentName) { await expect(testCard).toHaveCount(0, { timeout: 10_000 }); } -async function mockExternalAgentsRpc(page, listPayload) { - await page.addInitScript((externalAgentsListPayload) => { - if (window.__externalAgentE2EPatched) return; - window.__externalAgentE2EPatched = true; - window.__externalAgentE2ERequests = []; - window.__externalAgentE2EListPayload = externalAgentsListPayload || [ - { kind: "codex", name: "Codex", installed: true, isAcp: false, version: null }, - { kind: "claude-code", name: "Claude Code", installed: false, isAcp: false, version: null }, - ]; - const originalSend = WebSocket.prototype.send; - - function respond(socket, id, payload) { - queueMicrotask(() => { - const event = new MessageEvent("message", { - data: JSON.stringify({ type: "res", id, ok: true, payload }), +async function mockExternalAgentsRpc(page, listPayload, modelsPayload, bindFailures = 0, holdBackendSwitches = false) { + if (Array.isArray(modelsPayload)) { + await page.route( + "**/api/bootstrap?**", + async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ models: modelsPayload }), }); - if (typeof socket.onmessage === "function") socket.onmessage(event); - }); + }, + { times: 1 }, + ); + } + await page.route(/\/api\/sessions(?:\?.*)?$/, async (route) => { + const response = await route.fetch(); + const payload = await response.json(); + const bindings = await page.evaluate(() => window.__externalAgentE2EBindings || {}); + const sessions = Array.isArray(payload) ? payload : payload.sessions; + if (Array.isArray(sessions)) { + for (const session of sessions) { + if (Object.hasOwn(bindings, session.key)) session.external_agent_kind = bindings[session.key]; + } } + await route.fulfill({ response, json: payload }); + }); + await page.addInitScript( + ({ externalAgentsListPayload, modelListPayload, bindFailureCount, holdSwitches }) => { + if (window.__externalAgentE2EPatched) return; + window.__externalAgentE2EPatched = true; + window.__externalAgentE2ERequests = []; + window.__externalAgentE2EBindings = {}; + window.__externalAgentE2EPendingResponses = []; + window.__externalAgentE2EHoldSwitches = holdSwitches; + window.__releaseExternalAgentE2EResponses = () => { + const pending = window.__externalAgentE2EPendingResponses.splice(0); + for (const sendResponse of pending) sendResponse(); + }; + let failuresRemaining = bindFailureCount; + const agentsPayload = externalAgentsListPayload || [ + { kind: "codex", name: "Codex", installed: true, isAcp: false, version: null }, + { kind: "claude-code", name: "Claude Code", installed: false, isAcp: false, version: null }, + ]; + const originalSend = WebSocket.prototype.send; + + function respond(socket, id, payload) { + queueMicrotask(() => { + const event = new MessageEvent("message", { + data: JSON.stringify({ type: "res", id, ok: true, payload }), + }); + if (typeof socket.onmessage === "function") socket.onmessage(event); + }); + } - WebSocket.prototype.send = function (payload) { - try { - var parsed = JSON.parse(payload); - if (parsed?.method === "external_agents.list") { - window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); - respond(this, parsed.id, window.__externalAgentE2EListPayload); - return; - } - if (parsed?.method === "external_agents.bind") { - window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); - respond(this, parsed.id, { ok: true, sessionKey: parsed.params?.sessionKey, kind: parsed.params?.kind }); - return; - } - if (parsed?.method === "external_agents.unbind") { - window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); - respond(this, parsed.id, { ok: true, sessionKey: parsed.params?.sessionKey }); + function respondError(socket, id, message) { + queueMicrotask(() => { + const event = new MessageEvent("message", { + data: JSON.stringify({ type: "res", id, ok: false, error: { message } }), + }); + if (typeof socket.onmessage === "function") socket.onmessage(event); + }); + } + + function respondToBackendSwitch(sendResponse) { + if (window.__externalAgentE2EHoldSwitches) { + window.__externalAgentE2EPendingResponses.push(sendResponse); return; } - } catch (_err) { - // Fall through to the original sender. + sendResponse(); } - return originalSend.call(this, payload); - }; - }, listPayload); + + WebSocket.prototype.send = function (payload) { + try { + var parsed = JSON.parse(payload); + if (parsed?.method === "models.list" && Array.isArray(modelListPayload)) { + respond(this, parsed.id, modelListPayload); + return; + } + if (parsed?.method === "external_agents.list") { + window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); + respond(this, parsed.id, agentsPayload); + return; + } + if (parsed?.method === "external_agents.bind") { + window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); + if (failuresRemaining > 0) { + failuresRemaining--; + respondError(this, parsed.id, "simulated bind failure"); + return; + } + respondToBackendSwitch(() => { + window.__externalAgentE2EBindings[parsed.params?.sessionKey] = parsed.params?.kind; + respond(this, parsed.id, { ok: true }); + }); + return; + } + if (parsed?.method === "external_agents.unbind") { + window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); + respondToBackendSwitch(() => { + window.__externalAgentE2EBindings[parsed.params?.sessionKey] = null; + respond(this, parsed.id, { ok: true }); + }); + return; + } + if (parsed?.method === "sessions.patch") { + window.__externalAgentE2ERequests.push({ method: parsed.method, params: parsed.params || {} }); + } + } catch (_err) { + // Fall through to the original sender. + } + return originalSend.call(this, payload); + }; + }, + { + externalAgentsListPayload: listPayload, + modelListPayload: modelsPayload, + bindFailureCount: bindFailures, + holdSwitches: holdBackendSwitches, + }, + ); } async function expectActiveSessionExternalAgent(page, kind) { @@ -407,32 +483,51 @@ test.describe("Agents settings page", () => { expect(pageErrors).toEqual([]); }); - test("external-agent picker labels named ACP agents", async ({ page }) => { + test("composer selector lists and binds named ACP agents", async ({ page }) => { const pageErrors = watchPageErrors(page); - await mockExternalAgentsRpc(page, [ - { kind: "acp-copilot", name: "ACP: Copilot", installed: true, isAcp: true, version: null }, - { kind: "acp-codex", name: "ACP: Codex", installed: true, isAcp: true, version: null }, - { kind: "acp-claude", name: "ACP: Claude", installed: true, isAcp: true, version: null }, - { kind: "acp-pi", name: "ACP: Pi", installed: true, isAcp: true, version: null }, - { kind: "acp-opencode", name: "ACP: opencode", installed: true, isAcp: true, version: null }, - { kind: "acp-gemini", name: "ACP: Gemini", installed: true, isAcp: true, version: null }, - { kind: "acp-augment", name: "ACP: Augment", installed: true, isAcp: true, version: null }, - { kind: "acp-kiro", name: "ACP: Kiro", installed: true, isAcp: true, version: null }, - { kind: "acp-openclaw", name: "ACP: OpenClaw", installed: true, isAcp: true, version: null }, - { kind: "acp-openhands", name: "ACP: OpenHands", installed: true, isAcp: true, version: null }, - { kind: "acp-kimi", name: "ACP: Kimi", installed: true, isAcp: true, version: null }, - { kind: "acp-stakpak", name: "ACP: Stakpak", installed: true, isAcp: true, version: null }, - { kind: "acp-fast-agent", name: "ACP: fast-agent", installed: true, isAcp: true, version: null }, - ]); + await mockExternalAgentsRpc( + page, + [ + { kind: "acp-copilot", name: "ACP: Copilot", installed: true, isAcp: true, version: null }, + { kind: "acp-codex", name: "ACP: Codex", installed: true, isAcp: true, version: null }, + { kind: "acp-claude", name: "ACP: Claude", installed: true, isAcp: true, version: null }, + { kind: "acp-pi", name: "ACP: Pi", installed: true, isAcp: true, version: null }, + { kind: "acp-opencode", name: "ACP: opencode", installed: true, isAcp: true, version: null }, + { kind: "acp-gemini", name: "ACP: Gemini", installed: true, isAcp: true, version: null }, + { kind: "acp-augment", name: "ACP: Augment", installed: true, isAcp: true, version: null }, + { kind: "acp-kiro", name: "ACP: Kiro", installed: true, isAcp: true, version: null }, + { kind: "acp-openclaw", name: "ACP: OpenClaw", installed: true, isAcp: true, version: null }, + { kind: "acp-openhands", name: "ACP: OpenHands", installed: true, isAcp: true, version: null }, + { kind: "acp-kimi", name: "ACP: Kimi", installed: true, isAcp: true, version: null }, + { kind: "acp-stakpak", name: "ACP: Stakpak", installed: true, isAcp: true, version: null }, + { kind: "acp-fast-agent", name: "ACP: fast-agent", installed: true, isAcp: true, version: null }, + ], + [{ id: "e2e/model", displayName: "E2E Model", provider: "e2e", supportsReasoning: true }], + ); await page.goto("/chats"); await expectPageContentMounted(page); await waitForWsConnected(page); await createSession(page); + const sessionKey = await page.evaluate(() => window.__moltis_stores?.sessionStore?.activeSessionKey?.value || ""); + await page.evaluate(() => window.__moltis_stores?.modelStore?.select("e2e/model")); - const picker = page.getByTestId("external-agent-picker"); - await expect(picker).toBeVisible({ timeout: 10_000 }); - await picker.locator("button").click(); + await expect(page.getByTestId("external-agent-picker")).toHaveCount(0); + const picker = page.locator("#modelComboBtn"); + await expect(picker).toBeEnabled({ timeout: 10_000 }); + await expect(page.locator("#reasoningCombo")).toBeVisible(); + await page.locator("#reasoningComboBtn").click(); + await page + .locator("#reasoningDropdownList .model-dropdown-item") + .filter({ hasText: /^High$/ }) + .click(); + await expect(page.locator("#reasoningComboLabel")).toHaveText("High"); + await picker.click(); + const dropdown = page.locator("#modelDropdownList"); + await expect(dropdown.getByText("E2E Model", { exact: true })).toBeVisible(); await expect(page.getByText("ACP: Copilot", { exact: true })).toBeVisible(); + await expect( + dropdown.locator(".model-dropdown-item", { hasText: "ACP: Copilot" }).locator(".model-item-provider"), + ).toHaveText("ACP agent"); await expect(page.getByText("ACP: Codex", { exact: true })).toBeVisible(); await expect(page.getByText("ACP: Claude", { exact: true })).toBeVisible(); await expect(page.getByText("ACP: Pi", { exact: true })).toBeVisible(); @@ -450,10 +545,51 @@ test.describe("Agents settings page", () => { await expect .poll( async () => - page.evaluate(() => - (window.__externalAgentE2ERequests || []).some( - (req) => req.method === "external_agents.bind" && req.params?.kind === "acp-copilot", - ), + page.evaluate( + (key) => + (window.__externalAgentE2ERequests || []).some( + (req) => + req.method === "external_agents.bind" && + req.params?.sessionKey === key && + req.params?.kind === "acp-copilot", + ), + sessionKey, + ), + { timeout: 10_000 }, + ) + .toBe(true); + await expect(picker).toBeEnabled(); + await expect(page.locator("#modelComboLabel")).toHaveText("ACP: Copilot"); + await expect(page.locator("#reasoningCombo")).toBeHidden(); + + await picker.click(); + await dropdown.getByText("E2E Model", { exact: true }).click(); + await expect + .poll( + async () => + page.evaluate( + (key) => + (window.__externalAgentE2ERequests || []).some( + (req) => req.method === "external_agents.unbind" && req.params?.sessionKey === key, + ), + sessionKey, + ), + { timeout: 10_000 }, + ) + .toBe(true); + await expect(page.locator("#modelComboLabel")).toHaveText("E2E Model"); + await expect(page.locator("#reasoningCombo")).toBeVisible(); + await expect(page.locator("#reasoningComboLabel")).toHaveText("High"); + await expect + .poll( + async () => + page.evaluate( + (key) => + (window.__externalAgentE2ERequests || []).some( + (req) => + req.method === "sessions.patch" && req.params?.key === key && req.params?.model === "e2e/model", + ), + sessionKey, ), { timeout: 10_000 }, ) @@ -462,7 +598,7 @@ test.describe("Agents settings page", () => { expect(pageErrors).toEqual([]); }); - test("external-agent picker is hidden when no external agents are installed", async ({ page }) => { + test("composer selector hides unavailable ACP agents", async ({ page }) => { const pageErrors = watchPageErrors(page); await mockExternalAgentsRpc(page, [ { kind: "acp-copilot", name: "ACP: Copilot", installed: false, isAcp: true, version: null }, @@ -492,21 +628,132 @@ test.describe("Agents settings page", () => { ) .toBe(true); await expect(page.getByTestId("external-agent-picker")).toHaveCount(0); - await expect(page.getByText("ACP: Copilot (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Codex (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: opencode (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Gemini (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Augment (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Kiro (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: OpenClaw (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: OpenHands (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Kimi (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: Stakpak (unavailable)", { exact: true })).toHaveCount(0); - await expect(page.getByText("ACP: fast-agent (unavailable)", { exact: true })).toHaveCount(0); + await page.locator("#modelComboBtn").click(); + const dropdown = page.locator("#modelDropdownList"); + for (const name of [ + "ACP: Copilot", + "ACP: Codex", + "ACP: opencode", + "ACP: Gemini", + "ACP: Augment", + "ACP: Kiro", + "ACP: OpenClaw", + "ACP: OpenHands", + "ACP: Kimi", + "ACP: Stakpak", + "ACP: fast-agent", + ]) { + await expect(dropdown.getByText(name, { exact: true })).toHaveCount(0); + } expect(pageErrors).toEqual([]); }); + test("backend switch only disables the originating session selector", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await mockExternalAgentsRpc( + page, + [{ kind: "acp-copilot", name: "ACP: Copilot", installed: true, isAcp: true, version: null }], + [{ id: "e2e/model", displayName: "E2E Model", provider: "e2e", supportsReasoning: true }], + 0, + true, + ); + await page.goto("/chats"); + await expectPageContentMounted(page); + await waitForWsConnected(page); + + const firstSessionKey = await page.evaluate( + () => window.__moltis_stores?.sessionStore?.activeSessionKey?.value || "", + ); + const picker = page.locator("#modelComboBtn"); + await expect(picker).toBeEnabled({ timeout: 10_000 }); + await picker.click(); + await page.locator("#modelDropdownList").getByText("ACP: Copilot", { exact: true }).click(); + await expect(picker).toBeDisabled(); + + await createSession(page); + const secondSessionKey = await page.evaluate( + () => window.__moltis_stores?.sessionStore?.activeSessionKey?.value || "", + ); + expect(secondSessionKey).not.toBe(firstSessionKey); + await expect(picker).toBeEnabled(); + + await page.evaluate(() => window.__releaseExternalAgentE2EResponses?.()); + await expect + .poll( + async () => + page.evaluate( + (key) => window.__moltis_stores?.sessionStore?.getByKey?.(key)?.external_agent_kind || null, + firstSessionKey, + ), + { timeout: 10_000 }, + ) + .toBe("acp-copilot"); + await expect(picker).toBeEnabled(); + expect(pageErrors).toEqual([]); + }); + + test("ACP-only sessions auto-bind once", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await mockExternalAgentsRpc( + page, + [{ kind: "acp-copilot", name: "ACP: Copilot", installed: true, isAcp: true, version: null }], + [], + ); + await page.goto("/chats"); + await expectPageContentMounted(page); + await waitForWsConnected(page); + await createSession(page); + + const sessionKey = await page.evaluate(() => window.__moltis_stores?.sessionStore?.activeSessionKey?.value || ""); + await expect + .poll( + async () => + page.evaluate( + (key) => + (window.__externalAgentE2ERequests || []).filter( + (req) => req.method === "external_agents.bind" && req.params?.sessionKey === key, + ).length, + sessionKey, + ), + { timeout: 10_000 }, + ) + .toBe(1); + await expect(page.locator("#modelComboLabel")).toHaveText("ACP: Copilot"); + await expect(page.locator("#modelComboBtn")).toBeEnabled(); + expect(pageErrors).toEqual([]); + }); + + test("ACP-only sessions continue retrying failed auto-bind attempts", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await mockExternalAgentsRpc( + page, + [{ kind: "acp-copilot", name: "ACP: Copilot", installed: true, isAcp: true, version: null }], + [], + 3, + ); + await page.goto("/chats"); + await expectPageContentMounted(page); + await waitForWsConnected(page); + + const sessionKey = await page.evaluate(() => window.__moltis_stores?.sessionStore?.activeSessionKey?.value || ""); + await expect + .poll( + async () => + page.evaluate( + (key) => + (window.__externalAgentE2ERequests || []).filter( + (req) => req.method === "external_agents.bind" && req.params?.sessionKey === key, + ).length, + sessionKey, + ), + { timeout: 10_000 }, + ) + .toBe(4); + await expect(page.locator("#modelComboLabel")).toHaveText("ACP: Copilot"); + expect(pageErrors).toEqual([]); + }); + test("create form validates required fields", async ({ page }) => { const pageErrors = watchPageErrors(page); await navigateAndWait(page, "/settings/agents"); diff --git a/crates/web/ui/e2e/specs/chat-input.spec.js b/crates/web/ui/e2e/specs/chat-input.spec.js index bc780568f5..0694344a49 100644 --- a/crates/web/ui/e2e/specs/chat-input.spec.js +++ b/crates/web/ui/e2e/specs/chat-input.spec.js @@ -460,7 +460,7 @@ test.describe("Chat input and slash commands", () => { const box = await dropdown.boundingBox(); expect(box?.width || 0).toBeGreaterThan(360); - const item = page.locator("#modelDropdownList .model-dropdown-item").first(); + const item = page.locator("#modelDropdownList .model-dropdown-item", { hasText: displayName }); await expect(item).toHaveAttribute("title", fullTitle); await expect(item.locator(".model-item-label")).toHaveAttribute("title", fullTitle); await item.click(); diff --git a/crates/web/ui/e2e/specs/reasoning-toggle.spec.js b/crates/web/ui/e2e/specs/reasoning-toggle.spec.js index 92706354e4..8cb1c3dabe 100644 --- a/crates/web/ui/e2e/specs/reasoning-toggle.spec.js +++ b/crates/web/ui/e2e/specs/reasoning-toggle.spec.js @@ -197,8 +197,10 @@ test.describe("reasoning effort toggle", () => { const modelBtn = page.locator("#modelComboBtn"); await modelBtn.click(); - const items = page.locator("#modelDropdownList .model-dropdown-item"); - // Only the base model should appear, not the 3 reasoning variants + const items = page + .locator("#modelDropdownList .model-dropdown-item") + .filter({ has: page.locator(".model-item-provider") }); + // Only the base model should appear among the model entries, not the 3 reasoning variants. await expect(items).toHaveCount(1); await expect(items.first()).toContainText("Claude Opus 4.5"); diff --git a/crates/web/ui/src/components/SessionHeader.tsx b/crates/web/ui/src/components/SessionHeader.tsx index 3d90bc5d15..95413976a9 100644 --- a/crates/web/ui/src/components/SessionHeader.tsx +++ b/crates/web/ui/src/components/SessionHeader.tsx @@ -38,14 +38,6 @@ interface AgentOption { [key: string]: unknown; } -interface ExternalAgentInfo { - kind: string; - name: string; - installed: boolean; - isAcp?: boolean; - version?: string | null; -} - interface SelectOption { value: string; label: string; @@ -97,12 +89,6 @@ function isSshTargetNode(node: NodeInfo | null): boolean { return node?.platform === "ssh" || String(node?.nodeId || "").startsWith("ssh:"); } -function refreshModelComboAvailability(): void { - void import("../models").then(({ updateModelComboAvailability }) => { - updateModelComboAvailability(); - }); -} - function nodeOptionLabel(node: NodeInfo | null): string { if (!node) return "Local"; if (node.displayName) return node.displayName; @@ -162,11 +148,6 @@ export function SessionHeader({ const [agentOptionsLoaded, setAgentOptionsLoaded] = useState(initialAgentOptions.length > 0); const [nodeOptions, setNodeOptions] = useState([]); const [switchingNode, setSwitchingNode] = useState(false); - const [externalAgentOptions, setExternalAgentOptions] = useState([]); - const [switchingExternalAgent, setSwitchingExternalAgent] = useState(false); - const [hasLlmModels, setHasLlmModels] = useState(null); - const acpAutoBindAttemptedRef = useRef>(new Set()); - const acpAutoBindInFlightRef = useRef>(new Set()); const inputRef = useRef(null); const fullName = session ? session.label || session.key : currentKey; @@ -182,11 +163,6 @@ export function SessionHeader({ const showArchivedSessions = sessionStore.showArchivedSessions.value; const currentAgentId = session?.agent_id || defaultAgentId || "main"; const currentNodeId = session?.node_id || ""; - const currentExternalAgentKind = session?.external_agent_kind || ""; - const currentExternalAgent = currentExternalAgentKind - ? externalAgentOptions.find((agent) => agent.kind === currentExternalAgentKind) || null - : null; - const currentExternalAgentName = currentExternalAgent?.name || currentExternalAgentKind; useEffect(() => { let cancelled = false; @@ -206,29 +182,6 @@ export function SessionHeader({ }; }, [currentKey]); - useEffect(() => { - let cancelled = false; - setHasLlmModels(null); - sendRpc<{ id?: string }[]>("models.list", {}).then((res) => { - if (cancelled) return; - setHasLlmModels(Boolean(res?.ok && (res.payload || []).length > 0)); - }); - return () => { - cancelled = true; - }; - }, [currentKey]); - - useEffect(() => { - let cancelled = false; - sendRpc("external_agents.list", {}).then((res) => { - if (cancelled || !res?.ok) return; - setExternalAgentOptions(Array.isArray(res.payload) ? res.payload : []); - }); - return () => { - cancelled = true; - }; - }, [currentKey]); - // Fetch connected nodes and subscribe to presence updates. useEffect(() => { let cancelled = false; @@ -497,71 +450,6 @@ export function SessionHeader({ [currentKey, session, switchingNode], ); - const onExternalAgentChange = useCallback( - (nextKind: string) => { - if (switchingExternalAgent || nextKind === currentExternalAgentKind) return; - setSwitchingExternalAgent(true); - const request = nextKind - ? sendRpc("external_agents.bind", { sessionKey: currentKey, kind: nextKind }) - : sendRpc("external_agents.unbind", { sessionKey: currentKey }); - request - .then((res) => { - if (!res?.ok) { - showToast((res?.error as { message?: string })?.message || "Failed to update external agent", "error"); - return; - } - if (session) { - session.external_agent_kind = nextKind || null; - session.dataVersion.value++; - } - refreshModelComboAvailability(); - fetchSessions(); - }) - .finally(() => { - setSwitchingExternalAgent(false); - }); - }, - [currentExternalAgentKind, currentKey, session, switchingExternalAgent], - ); - - useEffect(() => { - if ( - isCron || - hasLlmModels !== false || - currentExternalAgentKind || - acpAutoBindAttemptedRef.current.has(currentKey) || - acpAutoBindInFlightRef.current.has(currentKey) - ) { - return; - } - const firstAcpAgent = externalAgentOptions.find((agent) => agent.installed && agent.isAcp); - if (!firstAcpAgent) return; - - let cancelled = false; - acpAutoBindInFlightRef.current.add(currentKey); - sendRpc("external_agents.bind", { sessionKey: currentKey, kind: firstAcpAgent.kind }) - .then((bindRes) => { - if (cancelled) return; - if (!bindRes?.ok) { - showToast((bindRes?.error as { message?: string })?.message || "Failed to select ACP agent", "error"); - return; - } - acpAutoBindAttemptedRef.current.add(currentKey); - if (session) { - session.external_agent_kind = firstAcpAgent.kind; - session.dataVersion.value++; - } - refreshModelComboAvailability(); - fetchSessions(); - }) - .finally(() => { - acpAutoBindInFlightRef.current.delete(currentKey); - }); - return () => { - cancelled = true; - }; - }, [currentExternalAgentKind, currentKey, externalAgentOptions, hasLlmModels, isCron, session]); - const agentSelectValue = currentAgentId; const hasCurrentAgentOption = agentOptions.some((agent) => agent.id === agentSelectValue); let agentSelectOptions: SelectOption[] = agentOptions.map((agent) => { @@ -585,28 +473,6 @@ export function SessionHeader({ const shouldShowAgentPicker = !isCron && agentOptionsLoaded && (agentOptions.length > 1 || !hasCurrentAgentOption); const shouldShowNodePicker = !isCron && (nodeOptions.length > 0 || Boolean(currentNodeId)); - const selectableExternalAgents = externalAgentOptions.filter( - (agent) => - (agent.isAcp || agent.kind === currentExternalAgentKind) && - (agent.installed || agent.kind === currentExternalAgentKind), - ); - const externalAgentSelectOptions: SelectOption[] = selectableExternalAgents.map((agent) => ({ - value: agent.kind, - label: `${agent.name}${agent.installed ? "" : " (unavailable)"}`, - })); - if (hasLlmModels !== false) { - externalAgentSelectOptions.unshift({ value: "", label: "Built-in LLM agent" }); - } else if (!currentExternalAgentKind) { - externalAgentSelectOptions.unshift({ value: "", label: "Select ACP agent" }); - } - const shouldShowExternalAgentPicker = !isCron && selectableExternalAgents.length > 0; - const externalAgentStatus = currentExternalAgentKind - ? currentExternalAgent?.installed === false - ? `${currentExternalAgentName} unavailable` - : session?.externalSessionId - ? `${currentExternalAgentName} session ${session.externalSessionId}` - : `${currentExternalAgentName} bound` - : ""; const hasCurrentNodeOption = currentNodeId === "" || nodeOptions.some((node) => node.nodeId === currentNodeId); let nodeSelectOptions: SelectOption[] = [ { value: "", label: "Local" }, @@ -704,25 +570,6 @@ export function SessionHeader({ disabled={switchingNode} /> )} - {showSelectors && shouldShowExternalAgentPicker && ( -
- - {externalAgentStatus && ( - - {externalAgentStatus} - - )} -
- )} {!nameOwnLine && showName && nameControl} {!nameOwnLine && renameCta} {showArchive && canArchive && ( diff --git a/crates/web/ui/src/models.ts b/crates/web/ui/src/models.ts index ff7caea94d..d55a138aa3 100644 --- a/crates/web/ui/src/models.ts +++ b/crates/web/ui/src/models.ts @@ -6,59 +6,165 @@ import { showModelNotice } from "./pages/ChatPage"; import * as S from "./state"; import { modelStore, REASONING_SEP } from "./stores/model-store"; import { sessionStore } from "./stores/session-store"; -import type { ModelInfo } from "./types"; +import type { ExternalAgentInfo, ModelInfo } from "./types"; +import { showToast } from "./ui"; + +let externalAgents: ExternalAgentInfo[] = []; +let externalAgentsLoaded = false; +let modelsLoaded = false; +let fetchModelsGeneration = 0; +const switchingBackendSessions = new Set(); +const acpAutoBindAttempted = new Set(); +const acpAutoBindInFlight = new Set(); +const acpAutoBindFailures = new Map(); +const acpAutoBindRetryTimers = new Map(); +const ACP_AUTO_BIND_RETRY_BASE_MS = 1_000; +const ACP_AUTO_BIND_RETRY_MAX_MS = 30_000; -interface ExternalAgentInfo { - kind: string; - name: string; - installed: boolean; - isAcp?: boolean; +function installedAcpAgents(): ExternalAgentInfo[] { + return externalAgents.filter((agent) => agent.installed && agent.isAcp); } -let installedExternalAgents: ExternalAgentInfo[] = []; - -function installedAcpAgents(): ExternalAgentInfo[] { - return installedExternalAgents.filter((agent) => agent.isAcp); +function selectableAcpAgents(): ExternalAgentInfo[] { + return sessionStore.activeSessionKey.value.startsWith("cron:") ? [] : installedAcpAgents(); } function activeExternalAgent(): ExternalAgentInfo | null { const kind = sessionStore.activeSession.value?.external_agent_kind || ""; if (!kind) return null; - return installedExternalAgents.find((agent) => agent.kind === kind) || null; + return externalAgents.find((agent) => agent.kind === kind) || null; } export function updateModelComboAvailability(): void { if (!(S.modelComboBtn && S.modelComboLabel)) return; - const acpAgent = activeExternalAgent(); - const disabled = Boolean(acpAgent?.isAcp); - (S.modelComboBtn as HTMLButtonElement).disabled = disabled; - S.modelComboBtn.setAttribute("aria-disabled", disabled ? "true" : "false"); - S.modelComboBtn.title = disabled ? `${acpAgent?.name || "ACP agent"} controls model selection` : ""; - if (disabled) { - closeModelDropdown(); - S.modelComboLabel.textContent = acpAgent?.name || "ACP agent"; - S.modelComboLabel.title = "Model selection is controlled by the selected ACP agent"; + const sessionKey = sessionStore.activeSessionKey.value; + const switchingBackend = switchingBackendSessions.has(sessionKey); + const externalKind = sessionStore.activeSession.value?.external_agent_kind || ""; + const externalAgent = activeExternalAgent(); + document + .getElementById("reasoningCombo") + ?.classList.toggle("hidden", Boolean(externalKind) || !modelStore.supportsReasoning.value); + (S.modelComboBtn as HTMLButtonElement).disabled = switchingBackend; + S.modelComboBtn.setAttribute("aria-disabled", switchingBackend ? "true" : "false"); + S.modelComboBtn.title = switchingBackend ? "Switching chat backend" : "Select model or ACP agent"; + if (externalKind) { + const label = externalAgent?.name || externalKind; + const unavailable = externalAgent?.installed === false; + S.modelComboLabel.textContent = unavailable ? `${label} (unavailable)` : label; + S.modelComboLabel.title = unavailable ? `${label} is unavailable` : `Using ${label}`; return; } const model = modelStore.selectedModel.value; if (model) updateModelComboLabel(model); else updateAcpOnlyModelComboLabel(); + maybeAutoBindAcp(); } -function refreshAcpAgents(): Promise { +function fetchAcpAgents(): Promise { return sendRpc("external_agents.list", {}) .then((res) => { - installedExternalAgents = res?.ok ? (res.payload || []).filter((agent) => agent.installed) : []; + if (!res?.ok) return null; + return res.payload || []; }) .catch(() => { - installedExternalAgents = []; + return null; }); } function updateAcpOnlyModelComboLabel(): void { - if (!(S.modelComboLabel && modelStore.models.value.length === 0 && installedAcpAgents().length > 0)) return; + if (!(S.modelComboLabel && modelStore.models.value.length === 0 && selectableAcpAgents().length > 0)) return; S.modelComboLabel.textContent = "ACP agent"; - S.modelComboLabel.title = "Using an ACP agent selected in the session header"; + S.modelComboLabel.title = "Select an ACP agent"; +} + +function setExternalAgentKind(sessionKey: string, kind: string | null): void { + const session = sessionStore.getByKey(sessionKey); + if (!session) return; + session.external_agent_kind = kind; + session.dataVersion.value++; +} + +function refreshSessionMetadata(): void { + void import("./sessions").then(({ fetchSessions }) => fetchSessions()); +} + +function clearAcpAutoBindRetry(sessionKey: string): void { + const timer = acpAutoBindRetryTimers.get(sessionKey); + if (timer !== undefined) window.clearTimeout(timer); + acpAutoBindRetryTimers.delete(sessionKey); + acpAutoBindFailures.delete(sessionKey); +} + +function scheduleAcpAutoBindRetry(sessionKey: string): void { + if (acpAutoBindRetryTimers.has(sessionKey)) return; + const failures = (acpAutoBindFailures.get(sessionKey) || 0) + 1; + acpAutoBindFailures.set(sessionKey, failures); + const delay = Math.min(ACP_AUTO_BIND_RETRY_BASE_MS * 2 ** Math.min(failures - 1, 5), ACP_AUTO_BIND_RETRY_MAX_MS); + const timer = window.setTimeout(() => { + acpAutoBindRetryTimers.delete(sessionKey); + if (sessionStore.activeSessionKey.value === sessionKey) updateModelComboAvailability(); + }, delay); + acpAutoBindRetryTimers.set(sessionKey, timer); +} + +function maybeAutoBindAcp(): void { + const session = sessionStore.activeSession.value; + const sessionKey = sessionStore.activeSessionKey.value; + if ( + !(session && sessionKey) || + sessionKey.startsWith("cron:") || + !modelsLoaded || + !externalAgentsLoaded || + modelStore.models.value.length > 0 || + session.external_agent_kind || + acpAutoBindAttempted.has(sessionKey) || + acpAutoBindInFlight.has(sessionKey) + ) { + return; + } + const agent = installedAcpAgents()[0]; + if (!agent) return; + acpAutoBindInFlight.add(sessionKey); + void bindAcpAgent(agent, false) + .then((bound) => { + if (bound) { + acpAutoBindAttempted.add(sessionKey); + clearAcpAutoBindRetry(sessionKey); + } + }) + .finally(() => { + acpAutoBindInFlight.delete(sessionKey); + if (!acpAutoBindAttempted.has(sessionKey)) scheduleAcpAutoBindRetry(sessionKey); + }); +} + +async function bindAcpAgent(agent: ExternalAgentInfo, notifyFailure = true): Promise { + const sessionKey = sessionStore.activeSessionKey.value; + if (!sessionKey || sessionKey.startsWith("cron:")) return false; + if (switchingBackendSessions.has(sessionKey)) return false; + if (sessionStore.activeSession.value?.external_agent_kind === agent.kind) { + closeModelDropdown(); + return true; + } + switchingBackendSessions.add(sessionKey); + updateModelComboAvailability(); + try { + const res = await sendRpc("external_agents.bind", { sessionKey, kind: agent.kind }); + if (!res?.ok) { + if (notifyFailure) showToast(res?.error?.message || "Failed to select ACP agent", "error"); + return false; + } + setExternalAgentKind(sessionKey, agent.kind); + refreshSessionMetadata(); + closeModelDropdown(); + return true; + } catch { + if (notifyFailure) showToast("Failed to select ACP agent", "error"); + return false; + } finally { + switchingBackendSessions.delete(sessionKey); + updateModelComboAvailability(); + } } function setSessionModel(sessionKey: string, modelId: string): void { @@ -89,7 +195,12 @@ function updateModelComboLabel(model: ModelInfo): void { } export function fetchModels(): Promise { - return Promise.all([modelStore.fetch(), refreshAcpAgents()]).then(() => { + const generation = ++fetchModelsGeneration; + return Promise.all([modelStore.fetch(), fetchAcpAgents()]).then(([didLoadModels, agents]) => { + if (generation !== fetchModelsGeneration) return; + modelsLoaded = didLoadModels; + externalAgentsLoaded = agents !== null; + if (agents !== null) externalAgents = agents; // Dual-write to state.js for backward compat S.setModels(modelStore.models.value); S.setSelectedModelId(modelStore.selectedModelId.value); @@ -107,18 +218,52 @@ export function fetchModels(): Promise { }); } -export function selectModel(m: ModelInfo): void { +function commitModelSelection(m: ModelInfo, sessionKey = S.activeSessionKey): void { modelStore.select(m.id); // Dual-write to state.js for backward compat S.setSelectedModelId(m.id); updateModelComboLabel(m); localStorage.setItem("moltis-model", m.id); - setSessionModel(S.activeSessionKey, m.id); + setSessionModel(sessionKey, m.id); closeModelDropdown(); // Show notice if model doesn't support tools showModelNotice(m); } +export function selectModel(m: ModelInfo): void { + const externalKind = sessionStore.activeSession.value?.external_agent_kind || ""; + if (!externalKind) { + commitModelSelection(m); + return; + } + const sessionKey = sessionStore.activeSessionKey.value; + if (!sessionKey) return; + if (switchingBackendSessions.has(sessionKey)) return; + switchingBackendSessions.add(sessionKey); + updateModelComboAvailability(); + void sendRpc("external_agents.unbind", { sessionKey }) + .then((res) => { + if (!res?.ok) { + showToast(res?.error?.message || "Failed to switch to the selected model", "error"); + return; + } + setExternalAgentKind(sessionKey, null); + refreshSessionMetadata(); + if (sessionStore.activeSessionKey.value === sessionKey) { + commitModelSelection(m, sessionKey); + } else { + setSessionModel(sessionKey, m.id); + } + }) + .catch(() => { + showToast("Failed to switch to the selected model", "error"); + }) + .finally(() => { + switchingBackendSessions.delete(sessionKey); + updateModelComboAvailability(); + }); +} + export function openModelDropdown(): void { if ((S.modelComboBtn as HTMLButtonElement | null)?.disabled) return; if (!S.modelDropdown) return; @@ -182,11 +327,44 @@ function buildModelItem(m: ModelInfo, currentId: string): HTMLDivElement { return el; } +function buildAcpItem(agent: ExternalAgentInfo, currentKind: string): HTMLDivElement { + const el = document.createElement("div"); + el.className = "model-dropdown-item"; + if (agent.kind === currentKind) el.classList.add("selected"); + el.title = `${agent.name} (${agent.kind})`; + + const label = document.createElement("span"); + label.className = "model-item-label"; + label.textContent = agent.name; + label.title = el.title; + el.appendChild(label); + + const meta = document.createElement("span"); + meta.className = "model-item-meta"; + const provider = document.createElement("span"); + provider.className = "model-item-provider"; + provider.textContent = "ACP agent"; + meta.appendChild(provider); + el.appendChild(meta); + el.addEventListener("click", () => void bindAcpAgent(agent)); + return el; +} + +function appendDivider(): void { + const divider = document.createElement("div"); + divider.className = "model-dropdown-divider"; + S.modelDropdownList?.appendChild(divider); +} + export function renderModelList(query: string): void { if (!S.modelDropdownList) return; S.modelDropdownList.textContent = ""; const q = query.toLowerCase(); const allModels = modelStore.models.value; + const acpAgents = selectableAcpAgents().filter((agent) => { + const name = agent.name.toLowerCase(); + return !q || name.includes(q) || agent.kind.toLowerCase().includes(q) || "acp agent".includes(q); + }); const filtered = allModels.filter((m) => { // Hide @reasoning-* virtual variants β€” the reasoning toggle handles these. if (m.id.indexOf(REASONING_SEP) !== -1) return false; @@ -194,33 +372,19 @@ export function renderModelList(query: string): void { const provider = (m.provider || "").toLowerCase(); return !q || label.indexOf(q) !== -1 || provider.indexOf(q) !== -1 || m.id.toLowerCase().indexOf(q) !== -1; }); - if (filtered.length === 0) { + if (filtered.length === 0 && acpAgents.length === 0) { const empty = document.createElement("div"); empty.className = "model-dropdown-empty"; - const acpAgents = installedAcpAgents(); - if (allModels.length === 0 && acpAgents.length > 0) { - empty.textContent = "No LLM models configured. ACP agents are selected from the session header."; - S.modelDropdownList.appendChild(empty); - acpAgents.forEach((agent) => { - const item = document.createElement("div"); - item.className = "model-dropdown-item model-dropdown-item-unsupported"; - const label = document.createElement("span"); - label.className = "model-item-label"; - label.textContent = agent.name; - item.appendChild(label); - const meta = document.createElement("span"); - meta.className = "model-item-meta"; - meta.textContent = "ACP agent"; - item.appendChild(meta); - S.modelDropdownList?.appendChild(item); - }); - return; - } empty.textContent = t("common:labels.noMatchingModels"); S.modelDropdownList.appendChild(empty); return; } - const currentId = modelStore.selectedModelId.value; + const currentKind = sessionStore.activeSession.value?.external_agent_kind || ""; + for (const agent of acpAgents) { + S.modelDropdownList.appendChild(buildAcpItem(agent, currentKind)); + } + if (acpAgents.length > 0 && filtered.length > 0) appendDivider(); + const currentId = currentKind ? "" : modelStore.selectedModelId.value; let lastPreferredIdx = -1; for (let i = filtered.length - 1; i >= 0; i--) { if (filtered[i].preferred) { @@ -232,9 +396,7 @@ export function renderModelList(query: string): void { S.modelDropdownList?.appendChild(buildModelItem(m, currentId)); if (idx === lastPreferredIdx && lastPreferredIdx < filtered.length - 1) { - const divider = document.createElement("div"); - divider.className = "model-dropdown-divider"; - S.modelDropdownList?.appendChild(divider); + appendDivider(); } }); } diff --git a/crates/web/ui/src/onboarding-view.tsx b/crates/web/ui/src/onboarding-view.tsx index 9285475d1a..1f6fa6d2bc 100644 --- a/crates/web/ui/src/onboarding-view.tsx +++ b/crates/web/ui/src/onboarding-view.tsx @@ -352,7 +352,7 @@ function SummaryStep({ onBack, onFinish }: { onBack: () => void; onFinish: () => ))} -
Available in each chat session's external-agent selector.
+
Available alongside models in each chat composer.
) : ( <>No ACP agents detected on PATH diff --git a/crates/web/ui/src/onboarding/types.ts b/crates/web/ui/src/onboarding/types.ts index 0d74a60d99..5252e210d9 100644 --- a/crates/web/ui/src/onboarding/types.ts +++ b/crates/web/ui/src/onboarding/types.ts @@ -1,5 +1,7 @@ // ── Shared types for onboarding sub-modules ────────────────── +export type { ExternalAgentInfo } from "../types"; + export interface ProviderInfo { name: string; displayName: string; @@ -14,14 +16,6 @@ export interface ProviderInfo { [key: string]: unknown; } -export interface ExternalAgentInfo { - kind: string; - name: string; - installed: boolean; - isAcp: boolean; - version?: string | null; -} - export interface ModelSelectorRow { id: string; displayName: string; diff --git a/crates/web/ui/src/pages/ChatPage.tsx b/crates/web/ui/src/pages/ChatPage.tsx index d59df5e5ff..8ce6d63863 100644 --- a/crates/web/ui/src/pages/ChatPage.tsx +++ b/crates/web/ui/src/pages/ChatPage.tsx @@ -19,7 +19,7 @@ import { import { SessionHeader } from "../components/SessionHeader"; import { formatTokens, sendRpc } from "../helpers"; import { initMediaDrop, teardownMediaDrop } from "../media-drop"; -import { bindModelComboEvents, modelDisplayLabel, modelTitle } from "../models"; +import { bindModelComboEvents, fetchModels, modelDisplayLabel, modelTitle } from "../models"; import { bindNodeComboEvents, fetchNodes, unbindNodeEvents } from "../nodes-selector"; import { bindProjectComboEvents } from "../project-combo"; import { fetchProjects } from "../projects"; @@ -731,8 +731,10 @@ function initializeChatControls(): void { S.setModelComboLabel(S.$("modelComboLabel")); S.setModelDropdown(S.$("modelDropdown")); S.setModelSearchInput(S.$("modelSearchInput")); + S.modelSearchInput?.setAttribute("placeholder", "Search models or ACP agents..."); S.setModelDropdownList(S.$("modelDropdownList")); bindModelComboEvents(); + void fetchModels(); bindReasoningToggle(); S.setNodeCombo(S.$("nodeCombo")); S.setNodeComboBtn(S.$("nodeComboBtn")); diff --git a/crates/web/ui/src/reasoning-toggle.ts b/crates/web/ui/src/reasoning-toggle.ts index cec7af0795..0ea0c6b0d7 100644 --- a/crates/web/ui/src/reasoning-toggle.ts +++ b/crates/web/ui/src/reasoning-toggle.ts @@ -9,6 +9,7 @@ import { effect } from "@preact/signals"; import { t } from "./i18n"; import { modelStore } from "./stores/model-store"; +import { sessionStore } from "./stores/session-store"; const EFFORT_VALUES: string[] = ["", "minimal", "low", "medium", "high", "xhigh"]; @@ -91,10 +92,15 @@ export function bindReasoningToggle(): void { // Reactively show/hide the combo based on model reasoning support disposeVisibility = effect(() => { - const show = modelStore.supportsReasoning.value; + const session = sessionStore.activeSession.value; + const sessionState = session + ? { externalAgentKind: session.external_agent_kind, version: session.dataVersion.value } + : null; + const supportsReasoning = modelStore.supportsReasoning.value; + const show = supportsReasoning && !sessionState?.externalAgentKind; reasoningCombo?.classList.toggle("hidden", !show); // Reset effort when switching to a non-reasoning model - if (!show && modelStore.reasoningEffort.value) { + if (!supportsReasoning && modelStore.reasoningEffort.value) { modelStore.setReasoningEffort(""); } if (reasoningComboLabel) { diff --git a/crates/web/ui/src/stores/model-store.ts b/crates/web/ui/src/stores/model-store.ts index 5c3643d03b..af51ff4107 100644 --- a/crates/web/ui/src/stores/model-store.ts +++ b/crates/web/ui/src/stores/model-store.ts @@ -13,6 +13,7 @@ export const REASONING_SEP = "@reasoning-"; export const models = signal([]); export const selectedModelId = signal(localStorage.getItem("moltis-model") || ""); export const reasoningEffort = signal(localStorage.getItem("moltis-reasoning-effort") || ""); +let modelListGeneration = 0; export const selectedModel = computed(() => { const id = selectedModelId.value; @@ -54,16 +55,19 @@ export function isReasoningVariant(modelId: string): boolean { /** Replace the full model list (e.g. after fetch or bootstrap). */ export function setAll(arr: ModelInfo[]): void { + modelListGeneration++; models.value = arr || []; } -/** Fetch models from the server via RPC. */ -export function fetch(): Promise { +/** Fetch models from the server via RPC. Returns whether the response succeeded. */ +export function fetch(): Promise { + const generation = ++modelListGeneration; return sendRpc("models.list", {}).then((r) => { const res = r as RpcResponse; - if (!res?.ok) return; - setAll(res.payload || []); - if (models.value.length === 0) return; + if (!res?.ok) return false; + if (generation !== modelListGeneration) return true; + models.value = res.payload || []; + if (models.value.length === 0) return true; let saved = localStorage.getItem("moltis-model") || ""; // If the saved model has a reasoning suffix, strip it and restore the effort const parsed = parseReasoningSuffix(saved); @@ -76,6 +80,7 @@ export function fetch(): Promise { const model = found || models.value[0]; select(model.id); if (!found) localStorage.setItem("moltis-model", model.id); + return true; }); } diff --git a/crates/web/ui/src/types/external-agent.ts b/crates/web/ui/src/types/external-agent.ts new file mode 100644 index 0000000000..2871291615 --- /dev/null +++ b/crates/web/ui/src/types/external-agent.ts @@ -0,0 +1,8 @@ +/** External chat agent discovered by the gateway. */ +export interface ExternalAgentInfo { + kind: string; + name: string; + installed: boolean; + isAcp: boolean; + version?: string | null; +} diff --git a/crates/web/ui/src/types/index.ts b/crates/web/ui/src/types/index.ts index a1cd4a9912..c6b707ac7a 100644 --- a/crates/web/ui/src/types/index.ts +++ b/crates/web/ui/src/types/index.ts @@ -11,6 +11,8 @@ export type { // ChannelType is both a type and a runtime const object, so use plain re-export. export { ChannelType } from "./channel"; +export type { ExternalAgentInfo } from "./external-agent"; + export type { ActiveHoursConfig, CronJob, diff --git a/crates/web/ui/src/types/rpc-methods.ts b/crates/web/ui/src/types/rpc-methods.ts index 47956db198..d543f7dd04 100644 --- a/crates/web/ui/src/types/rpc-methods.ts +++ b/crates/web/ui/src/types/rpc-methods.ts @@ -6,6 +6,7 @@ // typed use `unknown` as a placeholder -- callers can narrow with // `as` casts until we refine the type here. +import type { ExternalAgentInfo } from "./external-agent"; import type { ModelInfo } from "./model"; import type { SessionMeta } from "./session"; @@ -60,7 +61,7 @@ export interface RpcMethodMap { // ── Exec ──────────────────────────────────────────────────── "exec.approval.resolve": unknown; "external_agents.bind": unknown; - "external_agents.list": unknown; + "external_agents.list": ExternalAgentInfo[]; "external_agents.status": unknown; "external_agents.unbind": unknown; diff --git a/docs/src/external-agents.md b/docs/src/external-agents.md index 0e6d103f61..335d1ff84d 100644 --- a/docs/src/external-agents.md +++ b/docs/src/external-agents.md @@ -2,7 +2,7 @@ Moltis can bind a chat session to an external CLI coding agent. When a session is bound, `chat.send` persists the user turn in Moltis, sends the prompt and recent session context to the external process, streams the CLI output back to the web UI, and persists the assistant response. -ACP, the Agent Client Protocol, is a JSON-RPC protocol for connecting editor or coding agents to a host application. Moltis can run ACP-compatible command-line agents as external agents, route their permission prompts through Moltis approvals, and show them in the session header selector as `ACP: `. The canonical ACP agent catalog is https://agentclientprotocol.com/get-started/agents. +ACP, the Agent Client Protocol, is a JSON-RPC protocol for connecting editor or coding agents to a host application. Moltis can run ACP-compatible command-line agents as external agents, route their permission prompts through Moltis approvals, and show them alongside provider-backed models in the chat composer selector as `ACP: `. The canonical ACP agent catalog is https://agentclientprotocol.com/get-started/agents. Supported agent kinds: @@ -11,12 +11,12 @@ Supported agent kinds: | `claude-code` | `claude -p --output-format json` | Print mode with `session_id` capture; later turns add `--resume `. | | `codex` | `codex app-server` | Persistent app-server process; Moltis reuses the Codex `threadId` across turns. | | `acp` | `acp` | Persistent ACP JSON-RPC stdio session configured by `[external_agents.agents.acp]`. | -| `acp-copilot` | `copilot --acp` | Named ACP session shown as `ACP: Copilot` in the session header. | +| `acp-copilot` | `copilot --acp` | Named ACP session shown as `ACP: Copilot` in the chat model selector. | | `acp-codex` | `codex-acp` | Codex via Zed's ACP adapter, shown as `ACP: Codex`. | | `acp-claude` | `claude-agent-acp` | Claude Agent SDK via https://github.com/agentclientprotocol/claude-agent-acp, shown as `ACP: Claude`. | | `acp-pi` | `pi-acp` | Pi via the `pi-acp` adapter, shown as `ACP: Pi`. | -| `acp-opencode` | `opencode acp` | Named ACP session shown as `ACP: opencode` in the session header. | -| `acp-gemini` | `gemini --experimental-acp` | Named ACP session shown as `ACP: Gemini` in the session header. | +| `acp-opencode` | `opencode acp` | Named ACP session shown as `ACP: opencode` in the chat model selector. | +| `acp-gemini` | `gemini --experimental-acp` | Named ACP session shown as `ACP: Gemini` in the chat model selector. | | `acp-augment` | `auggie --acp` | Augment/Auggie ACP mode. | | `acp-kiro` | `kiro-cli acp` | Kiro CLI ACP mode. | | `acp-openclaw` | `openclaw acp` | OpenClaw ACP bridge. | @@ -45,7 +45,7 @@ External agent discovery is enabled by default. On startup, Moltis checks for th | `ACP: Stakpak` | `acp-stakpak` | `stakpak acp` | | `ACP: fast-agent` | `acp-fast-agent` | `fast-agent-acp` | -Installed ACP agents appear automatically in each chat session's external-agent selector. Missing commands are hidden from the selector, so a fresh install with no ACP agents available continues to show only the normal Moltis agent. +Installed ACP agents appear automatically alongside normal models in each chat session's composer selector. Missing commands are hidden, so a fresh install with no ACP agents available continues to show only provider-backed models. Default detection only checks whether the named command exists on Moltis' `$PATH`; it does not verify the binary publisher or installation source. Only install ACP agents from trusted sources, keep untrusted directories out of the service `$PATH`, and use explicit `binary = "/absolute/path/to/agent"` overrides when you want to pin the executable Moltis may launch after a user selects that agent for a session. @@ -164,7 +164,7 @@ binary = "codex" ## Select an ACP agent for a session -The session header in the web UI exposes an external-agent selector when agents are configured. ACP entries are labeled with the protocol and agent name, such as `ACP: Copilot`. Select `Moltis agent` to unbind and return the session to the normal provider-backed Moltis agent. +The model selector below the chat text field includes installed ACP agents, labeled with the protocol and agent name, such as `ACP: Copilot`. Selecting an ACP entry binds that chat session to the external agent. Select any normal model in the same menu to unbind the ACP agent and return the session to the provider-backed Moltis agent. Binding is per session. You can bind one chat session to `ACP: Copilot`, another to `ACP: Claude`, and leave other sessions on the normal Moltis agent. @@ -180,7 +180,7 @@ Moltis advertises ACP file-system and terminal capabilities to agents. File read - If an ACP agent does not appear in the selector, confirm `[external_agents] enabled = true`, restart Moltis, and verify the configured `binary` exists on `$PATH` or is an absolute path. - If an ACP entry appears as unavailable, run the configured command manually from the same shell or service environment that starts Moltis. -- If the wrong ACP agent is bound, use the session header selector and choose the desired `ACP: ` entry; choose `Moltis agent` to unbind. +- If the wrong ACP agent is bound, use the selector below the chat text field and choose the desired `ACP: ` entry; choose a normal model to unbind. - If the agent needs project-local context, set `working_dir` in that agent's config entry. Current limitations: From 0baf6a180eb06c0afc91b4cbe90f54cebcc6e372 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 12:22:42 -0700 Subject: [PATCH 15/23] fix(scripts): target local validation tests --- CLAUDE.md | 20 ++++-- CONTRIBUTING.md | 18 +++-- docs/src/local-validation.md | 31 +++++--- justfile | 12 ++++ scripts/local-validate.sh | 133 +++++++++++++++++++++++++++++++++-- 5 files changed, 190 insertions(+), 24 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 842839121b..8c8b04ead5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -187,8 +187,7 @@ CLI: `moltis auth reset-password`, `moltis auth reset-identity`. ## Testing ```bash -cargo test # All tests -cargo test # Specific test +cargo test # Specific changed/added test cargo test -- --nocapture # With stdout ``` @@ -199,10 +198,13 @@ Helpers in `e2e/helpers.js`. ```bash cd crates/web/ui -npx playwright test # All npx playwright test e2e/specs/chat-input.spec.js # Specific ``` +For local validation, run only Rust tests and Playwright specs changed or added +on the branch. Do not run the full Rust test suite or full E2E suite locally +unless explicitly requested; CI covers the full suites. + Rules: use `getByRole()`/`getByText({ exact: true })` selectors, shared helpers (`navigateAndWait`, `waitForWsConnected`, `watchPageErrors`), assert no JS errors, avoid `waitForTimeout()`. @@ -331,6 +333,12 @@ other AI/assistant session links) to commit messages or PR descriptions. Update ### Local Validation **Always** run `./scripts/local-validate.sh ` when a PR exists. +This runs broad validation for formatting, line limits, linting, builds, and +platform checks, but Rust tests and Playwright E2E are targeted to tests changed +or added on the branch. + +When explicitly asked to run the full local validation test suites, use +`just local-validate-full `. For incremental local edits before full validation: - TS/TSX changed: run `biome check --write` and `cd crates/web/ui && npm run build`. @@ -340,7 +348,8 @@ For incremental local edits before full validation: Exact commands (must match `local-validate.sh`): - Fmt: `cargo fmt --all -- --check` - Clippy: `just lint` (OS-aware: on macOS excludes CUDA features, on Linux uses `--all-features`) -- Tests: `just test` (OS-aware: on macOS uses nextest without CUDA features, on Linux uses `--all-features`) +- Tests: targeted changed/added Rust tests only, derived from the branch diff. Override with `LOCAL_VALIDATE_TEST_CMD` when needed. +- E2E: targeted changed/added Playwright specs only, derived from the branch diff. Override with `LOCAL_VALIDATE_E2E_CMD` when needed. - macOS app (Darwin hosts): `./scripts/build-swift-bridge.sh && ./scripts/generate-swift-project.sh && ./scripts/lint-swift.sh && xcodebuild -project apps/macos/Moltis.xcodeproj -scheme Moltis -configuration Release -destination "platform=macOS" -derivedDataPath apps/macos/.derivedData-local-validate CODE_SIGNING_ALLOWED=NO build` - iOS app (Darwin hosts): `cargo run -p moltis-schema-export -- apps/ios/GraphQL/Schema/schema.graphqls && ./scripts/generate-ios-graphql.sh && ./scripts/generate-ios-project.sh && xcodebuild -project apps/ios/Moltis.xcodeproj -scheme Moltis -configuration Debug -destination "generic/platform=iOS" CODE_SIGNING_ALLOWED=NO build` @@ -360,7 +369,8 @@ with exact commands), `## Manual QA`. Include concrete test steps. - [ ] Rust fmt passes (exact command above) - [ ] `just lint` passes (OS-aware clippy) - [ ] `just release-preflight` passes -- [ ] `just test` passes +- [ ] Changed/added Rust tests pass +- [ ] Changed/added Playwright specs pass for web UI changes - [ ] Conventional commit message - [ ] No debug code or temp files diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a3c7e80e9d..039ab68a03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -64,7 +64,7 @@ Core checks: ```bash just format-check just release-preflight -just test +cargo test ``` Changelog preview (for unreleased commits since the last tag): @@ -83,10 +83,10 @@ For web UI changes, run e2e tests: ```bash just ui-e2e-install -just ui-e2e +npx playwright test e2e/specs/.spec.js ``` -For CI-parity local validation (format, lint, test, e2e, lockfile, workflow security): +For local validation with broad non-test checks and targeted changed/added tests: ```bash ./scripts/local-validate.sh @@ -98,6 +98,12 @@ If you are working on an existing PR and have permissions to publish statuses: ./scripts/local-validate.sh ``` +To intentionally run full Rust and Playwright suites through local validation: + +```bash +just local-validate-full +``` + See also: - `docs/src/local-validation.md` @@ -107,6 +113,8 @@ See also: - Rust changes should include unit/integration coverage. - Web UI changes should include Playwright coverage in `crates/web/ui/e2e/specs/`. +- Local validation runs only changed or added Rust tests and Playwright specs; + CI covers broader suites. - Prefer real behavior tests over heavy mocking. - Keep tests deterministic and avoid timing-based flakiness. @@ -127,8 +135,8 @@ See also: - [ ] Tests added or updated for changed behavior - [ ] `just format-check` passes - [ ] `just release-preflight` passes -- [ ] `just test` passes -- [ ] `just ui-e2e` run for web UI changes +- [ ] Changed/added Rust tests pass +- [ ] Changed/added Playwright specs pass for web UI changes - [ ] Commit messages follow conventional commit style - [ ] Full session/context shared (or clear explanation if partial) - [ ] Shared session/logs are redacted (no API keys, private keys, tokens, passwords) diff --git a/docs/src/local-validation.md b/docs/src/local-validation.md index 5cda8c08eb..fc892da681 100644 --- a/docs/src/local-validation.md +++ b/docs/src/local-validation.md @@ -1,7 +1,8 @@ # Local Validation -Moltis provides a local validation script that runs the same checks as CI -(format, lint, test, e2e), plus a native macOS app build check on macOS hosts. +Moltis provides a local validation script that runs broad CI-style checks +(format, lint, build, line limits), plus targeted Rust and Playwright tests +for tests changed or added on the branch. ## Why this exists @@ -24,6 +25,13 @@ statuses to GitHub: ./scripts/local-validate.sh 63 ``` +To intentionally run the full Rust and Playwright suites locally, use: + +```bash +just local-validate-full +just local-validate-full 63 +``` + The script runs these checks: - `local/fmt` @@ -31,9 +39,9 @@ The script runs these checks: - `local/zizmor` - `local/lockfile` β€” verifies `Cargo.lock` is in sync (`cargo fetch --locked`) - `local/lint` -- `local/test` +- `local/test` β€” runs only changed or added Rust tests from the branch diff - `local/macos-app` β€” validates the native Swift macOS app build (`Darwin` only) -- `local/e2e` β€” runs gateway UI Playwright coverage +- `local/e2e` β€” runs only changed or added gateway UI Playwright specs - `local/e2e-ollama` β€” opt-in live Ollama/Qwen Playwright regression check In PR mode, the PR workflow verifies these contexts and surfaces them as @@ -55,20 +63,25 @@ checks in the PR. not already available. - `zizmor` is advisory in local runs and does not block lint/test execution. - Test output is suppressed unless tests fail. +- `local/test` derives changed Rust tests from the branch diff. Integration + tests under `crates//tests/*.rs` run as specific test targets; + changed in-source test modules run by package with a filename filter. Override + with `LOCAL_VALIDATE_TEST_CMD` when a branch needs broader local coverage. - `local/macos-app` runs only on macOS; on Linux it is marked skipped. - Override or disable macOS app validation with: `LOCAL_VALIDATE_MACOS_APP_CMD` and `LOCAL_VALIDATE_SKIP_MACOS_APP=1`. - `local/e2e` auto-runs `npm ci` only when `crates/web/ui/node_modules` - is missing, then runs `npm run e2e:install` and `npm run e2e`. Override with - `LOCAL_VALIDATE_E2E_CMD`. + is missing, then runs `npm run e2e:install` and changed Playwright specs from + `crates/web/ui/e2e/specs/`. Override with `LOCAL_VALIDATE_E2E_CMD`. - Enable the live Ollama/Qwen regression check with `LOCAL_VALIDATE_OLLAMA_QWEN_E2E=1`. It starts a local Ollama server on `MOLTIS_E2E_OLLAMA_QWEN_API_PORT` (default `11435`), pulls the configured Qwen model if missing, and runs the dedicated Playwright project. Override the command with `LOCAL_VALIDATE_OLLAMA_QWEN_E2E_CMD`. +- Coverage is opt-in because it runs Rust tests. Enable it with + `LOCAL_VALIDATE_COVERAGE=1` when needed. ## Merge and release safety -This local-first flow is for pull requests. Full CI still runs on GitHub -runners for non-PR events (for example push to `main`, scheduled runs, and -release paths). +This local-first flow is for pull requests. Full Rust and Playwright suites +still run on GitHub runners for CI/release paths where required. diff --git a/justfile b/justfile index 7122323bc4..48f907eee9 100644 --- a/justfile +++ b/justfile @@ -350,6 +350,18 @@ changelog-release version: ship commit_message='' pr_title='' pr_body='': ./scripts/ship-pr.sh {{ quote(commit_message) }} {{ quote(pr_title) }} {{ quote(pr_body) }} +# Run local validation with the full Rust and Playwright test suites. +local-validate-full pr_number='': + #!/usr/bin/env bash + set -euo pipefail + args=() + if [[ -n {{ quote(pr_number) }} ]]; then + args+=({{ quote(pr_number) }}) + fi + LOCAL_VALIDATE_TEST_CMD="just test" \ + LOCAL_VALIDATE_E2E_CMD="cd crates/web/ui && npm run e2e:install && npm run e2e" \ + ./scripts/local-validate.sh "${args[@]}" + # Run all tests (nightly to share build cache with clippy/lint, OS-aware). # On macOS: single nextest run using default features (includes Metal, not CUDA). # On Linux: --all-features (includes CUDA). diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index 2467f8905b..103d924b46 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -80,6 +80,7 @@ if [[ "$LOCAL_ONLY" -eq 0 ]]; then BASE_REPO="$(gh repo view --json nameWithOwner -q .nameWithOwner)" SHA="$(gh pr view "$PR_NUMBER" --repo "$BASE_REPO" --json headRefOid -q .headRefOid)" + BASE_REF_NAME="$(gh pr view "$PR_NUMBER" --repo "$BASE_REPO" --json baseRefName -q .baseRefName)" HEAD_OWNER="$(gh pr view "$PR_NUMBER" --repo "$BASE_REPO" --json headRepositoryOwner -q .headRepositoryOwner.login)" HEAD_REPO_NAME="$(gh pr view "$PR_NUMBER" --repo "$BASE_REPO" --json headRepository -q .headRepository.name)" @@ -101,6 +102,7 @@ EOF fi else SHA="$(git rev-parse HEAD)" + BASE_REF_NAME="${LOCAL_VALIDATE_BASE_REF:-main}" fi # Auto-sync Cargo.lock if stale (common after merging main). @@ -206,6 +208,119 @@ strip_all_features_flag() { printf '%s' "$cmd" } +changed_files() { + if [[ "$LOCAL_ONLY" -eq 0 ]]; then + gh pr diff "$PR_NUMBER" --repo "$BASE_REPO" --name-only + return + fi + + local base_ref="${LOCAL_VALIDATE_BASE_REF:-$BASE_REF_NAME}" + local base_commit="" + if git rev-parse --verify "origin/${base_ref}" >/dev/null 2>&1; then + base_commit="$(git merge-base "origin/${base_ref}" HEAD)" + elif git rev-parse --verify "$base_ref" >/dev/null 2>&1; then + base_commit="$(git merge-base "$base_ref" HEAD)" + elif git rev-parse --verify HEAD^ >/dev/null 2>&1; then + base_commit="HEAD^" + fi + + if [[ -n "$base_commit" ]]; then + git diff --name-only "$base_commit"...HEAD + else + git diff --name-only HEAD + fi +} + +package_name_for_path() { + local path="$1" + local dir + dir="$(dirname "$path")" + + while [[ "$dir" != "." && "$dir" != "/" ]]; do + if [[ -f "$dir/Cargo.toml" ]]; then + sed -nE 's/^name[[:space:]]*=[[:space:]]*"([^"]+)"/\1/p' "$dir/Cargo.toml" | head -n1 + return + fi + dir="$(dirname "$dir")" + done +} + +nextest_base_cmd_for_package() { + local package="$1" + if [[ "$(uname -s)" == "Darwin" ]]; then + case "$package" in + moltis-gateway|moltis-providers) + printf 'cargo +%s nextest run -p %s --features local-llm-metal' "$nightly_toolchain" "$package" + ;; + *) + printf 'cargo +%s nextest run -p %s' "$nightly_toolchain" "$package" + ;; + esac + else + printf 'cargo +%s nextest run -p %s --all-features' "$nightly_toolchain" "$package" + fi +} + +build_targeted_rust_test_cmd() { + local -a commands=() + local file + while IFS= read -r file; do + [[ -e "$file" ]] || continue + [[ "$file" == *.rs ]] || continue + + local package + package="$(package_name_for_path "$file")" + [[ -n "$package" ]] || continue + + local base_cmd + base_cmd="$(nextest_base_cmd_for_package "$package")" + if [[ "$file" == */tests/*.rs ]]; then + local test_name + test_name="$(basename "$file" .rs)" + commands+=("$base_cmd --test $test_name") + elif grep -Eq '#\[(tokio::)?test\]' "$file" 2>/dev/null; then + local filter_name + filter_name="$(basename "$file" .rs)" + commands+=("$base_cmd $filter_name") + fi + done < <(changed_files) + + if [[ "${#commands[@]}" -eq 0 ]]; then + printf '%s' 'echo "No changed Rust tests detected; skipping local/test."' + return + fi + + local joined="${commands[0]}" + local command + for command in "${commands[@]:1}"; do + joined="$joined && $command" + done + printf '%s' "$joined" +} + +build_targeted_e2e_cmd() { + local -a specs=() + local file + while IFS= read -r file; do + [[ -e "$file" ]] || continue + case "$file" in + crates/web/ui/e2e/specs/*.js|crates/web/ui/e2e/specs/*.ts) + specs+=("$file") + ;; + esac + done < <(changed_files) + + if [[ "${#specs[@]}" -eq 0 ]]; then + printf '%s' 'echo "No changed Playwright specs detected; skipping local/e2e."' + return + fi + + printf 'cd crates/web/ui && if [ ! -d node_modules ]; then npm ci; fi && npm run e2e:install && npx playwright test' + for file in "${specs[@]}"; do + printf ' %s' "${file#crates/web/ui/}" + done +} + if [[ "$(uname -s)" == "Darwin" ]] && ! command -v nvcc >/dev/null 2>&1; then if [[ -z "${LOCAL_VALIDATE_LINT_CMD:-}" ]]; then if command -v just >/dev/null 2>&1 && [[ -f justfile ]]; then @@ -233,6 +348,14 @@ if [[ "$(uname -s)" == "Darwin" ]] && ! command -v nvcc >/dev/null 2>&1; then echo "CI still covers the Linux/CUDA all-features path. Override with LOCAL_VALIDATE_* if you need a different split." >&2 fi +if [[ -z "${LOCAL_VALIDATE_TEST_CMD:-}" ]]; then + test_cmd="$(build_targeted_rust_test_cmd)" +fi + +if [[ -z "${LOCAL_VALIDATE_E2E_CMD:-}" ]]; then + e2e_cmd="$(build_targeted_e2e_cmd)" +fi + ensure_zizmor() { if command -v zizmor >/dev/null 2>&1; then return 0 @@ -616,14 +739,14 @@ else echo "Skipping Ollama Qwen live E2E (LOCAL_VALIDATE_OLLAMA_QWEN_E2E=0)." fi -# Coverage (optional β€” requires cargo-llvm-cov). -# Skipped silently when the tool is not installed. Disable explicitly with -# LOCAL_VALIDATE_SKIP_COVERAGE=1. -if [[ "${LOCAL_VALIDATE_SKIP_COVERAGE:-0}" != "1" ]] && cargo llvm-cov --version >/dev/null 2>&1; then +# Coverage is opt-in because llvm-cov runs the Rust test suite. +if [[ "${LOCAL_VALIDATE_COVERAGE:-0}" == "1" ]] && [[ "${LOCAL_VALIDATE_SKIP_COVERAGE:-0}" != "1" ]] && cargo llvm-cov --version >/dev/null 2>&1; then run_check "local/coverage" "$coverage_cmd" echo "Coverage report: target/llvm-cov/html/index.html" -elif [[ "${LOCAL_VALIDATE_SKIP_COVERAGE:-0}" != "1" ]]; then +elif [[ "${LOCAL_VALIDATE_COVERAGE:-0}" == "1" ]] && [[ "${LOCAL_VALIDATE_SKIP_COVERAGE:-0}" != "1" ]]; then echo "Skipping coverage (cargo-llvm-cov not installed). Install with: cargo install cargo-llvm-cov" +else + echo "Skipping coverage (set LOCAL_VALIDATE_COVERAGE=1 to run)." fi # Collect local/zizmor result at the end and fail if it found issues. From 4180b54cbdb19786cdbcf7c329cc7e2870f4ea3f Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 17:05:10 -0700 Subject: [PATCH 16/23] fix(slack): harden callback and delivery lifecycle Slack callbacks and channel acknowledgments could be reordered, stranded, or finalized against a newer turn because admission, deduplication, abort cleanup, and reaction ownership used different identities and lifetimes. Native streaming also used an obsolete API shape and could silently lose final content. Use run-addressed terminal ownership, bounded ordered callback workers, retry-window deduplication, official native stream payloads, and run-scoped delivery cleanup. Keep callback verification behind exact public routes with pre-auth throttling, and document the required Slack scopes and events. --- crates/chat/src/agent_loop.rs | 279 ++++++- crates/chat/src/channel_acks.rs | 8 +- crates/chat/src/channels.rs | 598 ++++++--------- crates/chat/src/channels/tests.rs | 218 ++++++ crates/chat/src/run_with_tools.rs | 259 ++++--- .../src/run_with_tools/run_scope_tests.rs | 69 ++ crates/chat/src/run_with_tools/tests.rs | 87 +++ crates/chat/src/runtime.rs | 12 +- crates/chat/src/service/chat_impl.rs | 218 ++++-- .../chat/src/service/chat_impl/queue_drain.rs | 659 ++++++++++++---- crates/chat/src/service/chat_impl/send.rs | 281 ++++--- crates/chat/src/service/mod.rs | 4 +- crates/chat/src/service/types.rs | 715 ++++++++++++++++-- crates/chat/src/streaming.rs | 56 +- crates/chat/src/types.rs | 4 + crates/gateway/src/channel_reactions.rs | 370 ++++++--- crates/gateway/src/channel_webhook_dedup.rs | 182 ++++- .../gateway/src/channel_webhook_middleware.rs | 144 +++- .../gateway/src/channel_webhook_rate_limit.rs | 69 +- crates/gateway/src/chat.rs | 17 +- crates/httpd/src/auth_middleware.rs | 166 +++- crates/httpd/src/request_throttle.rs | 125 ++- crates/httpd/src/server/gateway.rs | 242 +++--- crates/httpd/src/server/mod.rs | 2 + crates/httpd/src/server/slack_callbacks.rs | 678 +++++++++++++++++ crates/slack/src/blocks.rs | 23 +- crates/slack/src/callback_worker.rs | 232 ++++++ crates/slack/src/channel_webhook_verifier.rs | 113 ++- crates/slack/src/lib.rs | 3 + crates/slack/src/native_stream.rs | 535 +++++++++++++ crates/slack/src/outbound.rs | 400 +++++----- crates/slack/src/plugin.rs | 2 +- crates/slack/src/socket.rs | 592 ++------------- crates/slack/src/socket/callbacks.rs | 291 +++++++ crates/slack/src/socket/tests.rs | 245 ++++++ crates/slack/src/socket_reconnect.rs | 37 + crates/slack/src/state.rs | 287 ++++++- crates/slack/src/webhook.rs | 336 ++++++-- .../web/ui/e2e/specs/channels-slack.spec.js | 51 ++ .../ui/src/onboarding/steps/ChannelStep.tsx | 45 +- .../pages/channels/modals/AddSlackModal.tsx | 38 +- docs/src/slack.md | 47 +- 42 files changed, 6627 insertions(+), 2112 deletions(-) create mode 100644 crates/chat/src/channels/tests.rs create mode 100644 crates/chat/src/run_with_tools/run_scope_tests.rs create mode 100644 crates/chat/src/run_with_tools/tests.rs create mode 100644 crates/httpd/src/server/slack_callbacks.rs create mode 100644 crates/slack/src/callback_worker.rs create mode 100644 crates/slack/src/native_stream.rs create mode 100644 crates/slack/src/socket/callbacks.rs create mode 100644 crates/slack/src/socket/tests.rs create mode 100644 crates/slack/src/socket_reconnect.rs diff --git a/crates/chat/src/agent_loop.rs b/crates/chat/src/agent_loop.rs index 0f62e43a2b..6edf0b6021 100644 --- a/crates/chat/src/agent_loop.rs +++ b/crates/chat/src/agent_loop.rs @@ -1,6 +1,10 @@ //! Agent loop support: model flagging, shell commands, channel streaming, and compaction. -use std::{collections::HashSet, sync::Arc, time::Instant}; +use std::{ + collections::HashSet, + sync::Arc, + time::{Duration, Instant}, +}; use { moltis_config::schema::ToolMode, @@ -20,7 +24,7 @@ use crate::{ compaction_run, error, models::DisabledModelsStore, runtime::ChatRuntime, - service::{build_tool_call_assistant_message, persist_tool_history_pair}, + service::{build_tool_call_assistant_message, commit_terminal_run, persist_tool_history_pair}, types::*, }; @@ -125,6 +129,7 @@ pub(crate) fn ordered_runner_event_callback() -> ( } const CHANNEL_STREAM_BUFFER_SIZE: usize = 64; +const FINAL_CHANNEL_STREAM_TIMEOUT: Duration = Duration::from_secs(30); #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub(crate) struct ChannelReplyTargetKey { @@ -152,6 +157,35 @@ struct ChannelStreamWorker { receives_progress_deltas: bool, } +struct AbortOnDropTask(Option>); + +impl AbortOnDropTask { + fn new(task: tokio::task::JoinHandle<()>) -> Self { + Self(Some(task)) + } + + fn abort(&self) { + if let Some(task) = &self.0 { + task.abort(); + } + } + + async fn join(mut self) -> Result<(), tokio::task::JoinError> { + let Some(task) = self.0.as_mut() else { + return Ok(()); + }; + let result = task.await; + self.0.take(); + result + } +} + +impl Drop for AbortOnDropTask { + fn drop(&mut self) { + self.abort(); + } +} + /// Fan out model deltas to channel stream workers (Telegram/Discord edit-in-place). /// /// Workers are started eagerly so channel typing indicators remain active @@ -161,7 +195,7 @@ pub(crate) struct ChannelStreamDispatcher { outbound: Arc, targets: Vec, workers: Vec, - tasks: Vec>, + tasks: Vec, completed: Arc>>, started: bool, sent_final_delta: bool, @@ -234,26 +268,27 @@ impl ChannelStreamDispatcher { sender: tx, receives_progress_deltas, }); - self.tasks.push(tokio::spawn(async move { - match outbound - .send_stream(&account_id, &to, reply_to.as_deref(), rx) - .await - { - Ok(()) => { - if streams_final_replies { - completed.lock().await.insert(key_for_insert); - } - }, - Err(e) => { - warn!( - account_id = account_for_log, - chat_id = chat_for_log, - thread_id = thread_for_log.as_deref().unwrap_or("-"), - "channel stream outbound failed: {e}" - ); - }, - } - })); + self.tasks + .push(AbortOnDropTask::new(tokio::spawn(async move { + match outbound + .send_stream(&account_id, &to, reply_to.as_deref(), rx) + .await + { + Ok(()) => { + if streams_final_replies { + completed.lock().await.insert(key_for_insert); + } + }, + Err(e) => { + warn!( + account_id = account_for_log, + chat_id = chat_for_log, + thread_id = thread_for_log.as_deref().unwrap_or("-"), + "channel stream outbound failed: {e}" + ); + }, + } + }))); } } @@ -292,6 +327,18 @@ impl ChannelStreamDispatcher { } pub(crate) async fn finish(&mut self) { + if tokio::time::timeout(FINAL_CHANNEL_STREAM_TIMEOUT, self.finish_inner()) + .await + .is_err() + { + warn!("timed out finishing channel stream workers"); + self.abort_workers(); + self.workers.clear(); + self.join_workers().await; + } + } + + async fn finish_inner(&mut self) { self.send_terminal(moltis_channels::StreamEvent::Done).await; self.join_workers().await; } @@ -311,12 +358,18 @@ impl ChannelStreamDispatcher { async fn join_workers(&mut self) { let tasks = std::mem::take(&mut self.tasks); for task in tasks { - if let Err(e) = task.await { + if let Err(e) = task.join().await { warn!(error = %e, "channel stream worker task join failed"); } } } + fn abort_workers(&self) { + for task in &self.tasks { + task.abort(); + } + } + pub(crate) async fn completed_target_keys(&self) -> HashSet { if !self.sent_final_delta { return HashSet::new(); @@ -325,12 +378,33 @@ impl ChannelStreamDispatcher { } } +impl Drop for ChannelStreamDispatcher { + fn drop(&mut self) { + // Abort receivers before dropping senders. Some channel implementations + // treat sender closure like Done and would otherwise publish a late final. + self.abort_workers(); + } +} + +pub(crate) async fn commit_terminal_and_finish_channel_stream( + terminal_runs: &Arc>>, + run_id: &str, + dispatcher: Option<&mut ChannelStreamDispatcher>, +) -> HashSet { + commit_terminal_run(terminal_runs, run_id).await; + let Some(dispatcher) = dispatcher else { + return HashSet::new(); + }; + dispatcher.finish().await; + dispatcher.completed_target_keys().await +} + pub(crate) async fn run_explicit_shell_command( state: &Arc, run_id: &str, + terminal_runs: &Arc>>, tool_registry: &Arc>, session_store: &Arc, - terminal_runs: &Arc>>, session_key: &str, command: &str, user_message_index: usize, @@ -342,7 +416,7 @@ pub(crate) async fn run_explicit_shell_command( let tool_call_id = format!("sh_{}", uuid::Uuid::new_v4().simple()); let tool_args = serde_json::json!({ "command": command }); - send_tool_status_to_channels(state, session_key, "exec", &tool_args).await; + send_tool_status_to_channels(state, run_id, session_key, "exec", &tool_args).await; broadcast( state, @@ -493,10 +567,15 @@ pub(crate) async fn run_explicit_shell_command( }, } + // A channel can accept the reply immediately, so claim terminal ownership + // before final delivery. This prevents a concurrent abort from emitting a + // contradictory aborted terminal after the user already received output. + commit_terminal_run(terminal_runs, run_id).await; if !final_text.trim().is_empty() { let streamed_target_keys = HashSet::new(); deliver_channel_replies( state, + run_id, session_key, &final_text, ReplyMedium::Text, @@ -527,10 +606,8 @@ pub(crate) async fn run_explicit_shell_command( ); #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload = serde_json::to_value(&final_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload, BroadcastOpts::default()).await; - build_assistant_turn_output( + let mut output = build_assistant_turn_output( final_text, UsageSnapshot::new( moltis_agents::model::Usage::default(), @@ -540,7 +617,9 @@ pub(crate) async fn run_explicit_shell_command( None, None, None, - ) + ); + output.final_broadcast = Some(payload); + output } /// Resolve the effective tool mode for a provider. @@ -606,6 +685,89 @@ mod tests { events: Arc>>, } + struct DropNotice(Option>); + + impl Drop for DropNotice { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + + struct ClosureFinalizingOutbound { + started: Mutex>>, + dropped: Mutex>>, + finalized_on_close: std::sync::atomic::AtomicBool, + } + + #[async_trait] + impl moltis_channels::ChannelStreamOutbound for ClosureFinalizingOutbound { + async fn send_stream( + &self, + _account_id: &str, + _to: &str, + _reply_to: Option<&str>, + mut stream: moltis_channels::StreamReceiver, + ) -> moltis_channels::Result<()> { + let dropped = self.dropped.lock().await.take(); + let _notice = DropNotice(dropped); + if let Some(started) = self.started.lock().await.take() { + let _ = started.send(()); + } + while let Some(event) = stream.recv().await { + if matches!( + event, + moltis_channels::StreamEvent::Done | moltis_channels::StreamEvent::Error(_) + ) { + return Ok(()); + } + } + self.finalized_on_close + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + async fn is_stream_enabled(&self, _account_id: &str) -> bool { + true + } + } + + struct TerminalCheckingOutbound { + terminal_runs: Arc>>, + final_observed_after_terminal: std::sync::atomic::AtomicBool, + } + + #[async_trait] + impl moltis_channels::ChannelStreamOutbound for TerminalCheckingOutbound { + async fn send_stream( + &self, + _account_id: &str, + _to: &str, + _reply_to: Option<&str>, + mut stream: moltis_channels::StreamReceiver, + ) -> moltis_channels::Result<()> { + while let Some(event) = stream.recv().await { + if matches!(event, moltis_channels::StreamEvent::Done) { + self.final_observed_after_terminal.store( + self.terminal_runs.read().await.contains("run-1"), + std::sync::atomic::Ordering::SeqCst, + ); + break; + } + } + Ok(()) + } + + async fn is_stream_enabled(&self, _account_id: &str) -> bool { + true + } + + async fn streams_final_replies(&self, _account_id: &str) -> bool { + true + } + } + #[async_trait] impl moltis_channels::ChannelStreamOutbound for RecordingStreamOutbound { async fn send_stream( @@ -651,7 +813,9 @@ mod tests { } } - fn dispatcher_with(outbound: Arc) -> ChannelStreamDispatcher { + fn dispatcher_with( + outbound: Arc, + ) -> ChannelStreamDispatcher { ChannelStreamDispatcher { outbound, targets: vec![channel_target()], @@ -712,4 +876,57 @@ mod tests { assert_eq!(event_kinds(&events), vec!["progress", "delta", "done"]); assert_eq!(dispatcher.completed_target_keys().await.len(), 1); } + + #[tokio::test] + async fn dropping_dispatcher_aborts_worker_before_sender_closure_can_finalize() { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let outbound = Arc::new(ClosureFinalizingOutbound { + started: Mutex::new(Some(started_tx)), + dropped: Mutex::new(Some(dropped_tx)), + finalized_on_close: std::sync::atomic::AtomicBool::new(false), + }); + let mut dispatcher = dispatcher_with(outbound.clone()); + dispatcher.send_delta("final").await; + assert!(started_rx.await.is_ok()); + + drop(dispatcher); + + assert!( + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .is_ok() + ); + assert!( + !outbound + .finalized_on_close + .load(std::sync::atomic::Ordering::SeqCst) + ); + } + + #[tokio::test] + async fn terminal_is_committed_before_channel_stream_final() { + let terminal_runs = Arc::new(RwLock::new(HashSet::new())); + let outbound = Arc::new(TerminalCheckingOutbound { + terminal_runs: Arc::clone(&terminal_runs), + final_observed_after_terminal: std::sync::atomic::AtomicBool::new(false), + }); + let mut dispatcher = dispatcher_with(outbound.clone()); + dispatcher.send_delta("final").await; + + let completed = commit_terminal_and_finish_channel_stream( + &terminal_runs, + "run-1", + Some(&mut dispatcher), + ) + .await; + + assert!(terminal_runs.read().await.contains("run-1")); + assert!( + outbound + .final_observed_after_terminal + .load(std::sync::atomic::Ordering::SeqCst) + ); + assert_eq!(completed.len(), 1); + } } diff --git a/crates/chat/src/channel_acks.rs b/crates/chat/src/channel_acks.rs index b32bc2d11d..f390a034c3 100644 --- a/crates/chat/src/channel_acks.rs +++ b/crates/chat/src/channel_acks.rs @@ -44,10 +44,10 @@ pub(crate) fn merged_ack_keys<'a>(params: impl Iterator) -> Ve /// Mark the channel acknowledgment as failed because the reply never reached /// the user. Terminal protection means this wins over the run's later Success. -pub(crate) async fn note_delivery_failed(state: &Arc, session_key: &str) { +pub(crate) async fn note_delivery_failed(state: &Arc, activity_id: &str) { state .note_channel_activity( - session_key, + activity_id, moltis_channels::ChannelActivity::Finished(moltis_channels::ChannelAckOutcome::Failure), ) .await; @@ -56,7 +56,7 @@ pub(crate) async fn note_delivery_failed(state: &Arc, session_k /// Finalize the acknowledgment for a completed run. pub(crate) async fn note_turn_finished( state: &Arc, - session_key: &str, + activity_id: &str, succeeded: bool, ) { let outcome = if succeeded { @@ -66,7 +66,7 @@ pub(crate) async fn note_turn_finished( }; state .note_channel_activity( - session_key, + activity_id, moltis_channels::ChannelActivity::Finished(outcome), ) .await; diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index b0470d3091..9ce2ef4150 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -5,19 +5,19 @@ use std::{collections::HashSet, sync::Arc, time::Duration}; use { serde::{Deserialize, Serialize}, serde_json::Value, + tokio::task::JoinSet, tracing::{debug, info, warn}, }; use moltis_sessions::store::SessionStore; use crate::{ - agent_loop::ChannelReplyTargetKey, compaction_run, error, runtime::ChatRuntime, types::*, + agent_loop::ChannelReplyTargetKey, channel_acks::note_delivery_failed, compaction_run, error, + runtime::ChatRuntime, types::*, }; -/// Build the SPA URL for a push notification click-through. -/// -/// Must match the frontend `sessionPath()` in `router.ts`: -/// `/chats/${key.replace(/:/g, "/")}`. +const FINAL_CHANNEL_IO_TIMEOUT: Duration = Duration::from_secs(30); + #[cfg(any(feature = "push-notifications", test))] pub(crate) fn push_notification_url(session_key: &str) -> String { format!("/chats/{}", session_key.replace(':', "/")) @@ -29,7 +29,6 @@ pub(crate) async fn send_chat_push_notification( session_key: &str, text: &str, ) { - // Create a short summary of the response (first 100 chars) let summary = if text.len() > 100 { format!("{}…", truncate_at_char_boundary(text, 100)) } else { @@ -52,12 +51,39 @@ pub(crate) async fn send_chat_push_notification( } } -/// Drain any pending channel reply targets for a session and send the -/// response text back to each originating channel via outbound. -/// Each delivery runs in its own spawned task so slow network calls -/// don't block each other or the chat pipeline. pub(crate) async fn deliver_channel_replies( state: &Arc, + activity_id: &str, + session_key: &str, + text: &str, + desired_reply_medium: ReplyMedium, + streamed_target_keys: &HashSet, +) { + if tokio::time::timeout( + FINAL_CHANNEL_IO_TIMEOUT, + deliver_channel_replies_inner( + state, + activity_id, + session_key, + text, + desired_reply_medium, + streamed_target_keys, + ), + ) + .await + .is_err() + { + warn!( + activity_id, + session_key, "timed out delivering final channel reply" + ); + note_delivery_failed(state, activity_id).await; + } +} + +async fn deliver_channel_replies_inner( + state: &Arc, + activity_id: &str, session_key: &str, text: &str, desired_reply_medium: ReplyMedium, @@ -66,8 +92,6 @@ pub(crate) async fn deliver_channel_replies( let drained_targets = state.drain_channel_replies(session_key).await; let mut targets = Vec::with_capacity(drained_targets.len()); let mut streamed_targets = Vec::new(); - // When the reply medium is voice we must still deliver TTS audio even if - // the text was already streamed β€” skip the stream dedupe entirely. if desired_reply_medium != ReplyMedium::Voice && !streamed_target_keys.is_empty() { for target in drained_targets { let key = ChannelReplyTargetKey::from(&target); @@ -125,7 +149,7 @@ pub(crate) async fn deliver_channel_replies( "channel reply delivery skipped: outbound unavailable" ); } - crate::channel_acks::note_delivery_failed(state, session_key).await; + note_delivery_failed(state, activity_id).await; return; }, }; @@ -154,6 +178,7 @@ pub(crate) async fn deliver_channel_replies( deliver_channel_replies_to_targets( outbound, targets, + activity_id, session_key, text, Arc::clone(state), @@ -164,8 +189,6 @@ pub(crate) async fn deliver_channel_replies( .await; } -/// Format buffered status log entries into a Telegram expandable blockquote HTML. -/// Returns an empty string if there are no entries. fn format_logbook_html(entries: &[String]) -> String { if entries.is_empty() { return String::new(); @@ -193,12 +216,12 @@ async fn send_channel_logbook_follow_up_to_targets( } let html = logbook_html.to_string(); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let html = html.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { if let Err(e) = outbound .send_html(&target.account_id, &to, &html, None) .await @@ -210,11 +233,11 @@ async fn send_channel_logbook_follow_up_to_targets( "failed to send logbook follow-up: {e}" ); } - })); + }); } - for task in tasks { - if let Err(e) = task.await { + while let Some(result) = tasks.join_next().await { + if let Err(e) = result { warn!(error = %e, "channel logbook follow-up task join failed"); } } @@ -246,16 +269,6 @@ fn format_channel_error_message(error_obj: &Value) -> String { format!("⚠️ {title}: {detail}") } -/// Format a user-facing notice announcing that a session was compacted. -/// -/// Shown verbatim to channel users (Telegram, Discord, WhatsApp, etc.) and -/// kept short so small mobile clients don't wrap the whole thing. -/// -/// When `include_settings_hint` is false, the "Change chat.compaction.mode…" -/// footer is omitted so users who have set -/// `chat.compaction.show_settings_hint = false` don't see the repetitive -/// hint on every compaction. Mode + token lines are always included. -/// The LLM retry path never sees this text regardless. fn format_channel_compaction_notice( outcome: &compaction_run::CompactionOutcome, include_settings_hint: bool, @@ -297,14 +310,6 @@ fn format_channel_compaction_notice( } } -/// Send a silent "session compacted" notice to pending channel targets -/// without draining them. -/// -/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, -/// not drained, so the in-flight agent run can still deliver its final -/// reply to them afterward. Uses `send_text_silent` so the channel -/// integration doesn't count it toward user-visible interactive replies -/// (no TTS, no delivery receipts beyond the channel's own). pub(crate) async fn notify_channels_of_compaction( state: &Arc, session_key: &str, @@ -321,12 +326,12 @@ pub(crate) async fn notify_channels_of_compaction( }; let message = format_channel_compaction_notice(outcome, include_settings_hint); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let message = message.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let reply_to = target.message_id.as_deref(); if let Err(e) = outbound .send_text_silent(&target.account_id, &to, &message, reply_to) @@ -339,18 +344,16 @@ pub(crate) async fn notify_channels_of_compaction( "failed to send compaction notice to channel: {e}" ); } - })); + }); } - for task in tasks { - if let Err(e) = task.await { + while let Some(result) = tasks.join_next().await { + if let Err(e) = result { warn!(error = %e, "channel compaction notice task join failed"); } } } -/// Send a short retry status update to pending channel targets without draining -/// them. The final reply (or terminal error) will still use the same targets. pub(crate) async fn send_retry_status_to_channels( state: &Arc, session_key: &str, @@ -368,12 +371,12 @@ pub(crate) async fn send_retry_status_to_channels( }; let message = format_channel_retry_message(error_obj, retry_after); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let message = message.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let reply_to = target.message_id.as_deref(); if let Err(e) = outbound .send_text_silent(&target.account_id, &to, &message, reply_to) @@ -386,11 +389,11 @@ pub(crate) async fn send_retry_status_to_channels( "failed to send retry status to channel: {e}" ); } - })); + }); } - for task in tasks { - if let Err(e) = task.await { + while let Some(result) = tasks.join_next().await { + if let Err(e) = result { warn!(error = %e, "channel retry status task join failed"); } } @@ -401,6 +404,22 @@ pub(crate) async fn deliver_channel_error( state: &Arc, session_key: &str, error_obj: &Value, +) { + if tokio::time::timeout( + FINAL_CHANNEL_IO_TIMEOUT, + deliver_channel_error_inner(state, session_key, error_obj), + ) + .await + .is_err() + { + warn!(session_key, "timed out delivering terminal channel error"); + } +} + +async fn deliver_channel_error_inner( + state: &Arc, + session_key: &str, + error_obj: &Value, ) { let targets = state.drain_channel_replies(session_key).await; let status_log = state.drain_channel_status_log(session_key).await; @@ -415,13 +434,13 @@ pub(crate) async fn deliver_channel_error( let error_text = format_channel_error_message(error_obj); let logbook_html = format_logbook_html(&status_log); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let error_text = error_text.clone(); let logbook_html = logbook_html.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let reply_to = target.message_id.as_deref(); let send_result = if logbook_html.is_empty() { outbound @@ -446,11 +465,11 @@ pub(crate) async fn deliver_channel_error( "failed to send channel error reply: {e}" ); } - })); + }); } - for task in tasks { - if let Err(e) = task.await { + while let Some(result) = tasks.join_next().await { + if let Err(e) = result { warn!(error = %e, "channel error task join failed"); } } @@ -459,6 +478,7 @@ pub(crate) async fn deliver_channel_error( async fn deliver_channel_replies_to_targets( outbound: Arc, targets: Vec, + activity_id: &str, session_key: &str, text: &str, state: Arc, @@ -467,21 +487,21 @@ async fn deliver_channel_replies_to_targets( streamed_target_keys: &HashSet, ) { let session_key = session_key.to_string(); + let activity_id = activity_id.to_string(); let text = text.to_string(); let logbook_html = format_logbook_html(&status_log); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let state = Arc::clone(&state); let session_key = session_key.clone(); + let activity_id = activity_id.clone(); let text = text.clone(); let logbook_html = logbook_html.clone(); - // Text was already delivered via edit-in-place streaming β€” skip text - // caption/follow-up and only send the TTS voice audio. let text_already_streamed = streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let tts_payload = match desired_reply_medium { ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, ReplyMedium::Text => None, @@ -493,7 +513,6 @@ async fn deliver_channel_replies_to_targets( let transcript = std::mem::take(&mut payload.text); if text_already_streamed { - // Text was already streamed β€” send voice audio only. if let Err(e) = outbound .send_media(&target.account_id, &to, &payload, reply_to) .await @@ -504,8 +523,8 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel voice reply: {e}" ); + note_delivery_failed(&state, &activity_id).await; } - // Send logbook as a follow-up if present. if !logbook_html.is_empty() && let Err(e) = outbound .send_html(&target.account_id, &to, &logbook_html, None) @@ -519,9 +538,6 @@ async fn deliver_channel_replies_to_targets( ); } } else { - // Check if transcript fits as Telegram caption (when feature enabled). - // When telegram feature is disabled, this evaluates to false and we - // send voice + follow-up text. #[cfg(feature = "telegram")] let fits_in_caption = transcript.len() <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; @@ -529,7 +545,6 @@ async fn deliver_channel_replies_to_targets( let fits_in_caption = false; if fits_in_caption { - // Short transcript fits as a caption on the voice message. payload.text = transcript; if let Err(e) = outbound .send_media(&target.account_id, &to, &payload, reply_to) @@ -541,8 +556,8 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel voice reply: {e}" ); + note_delivery_failed(&state, &activity_id).await; } - // Send logbook as a follow-up if present. if !logbook_html.is_empty() && let Err(e) = outbound .send_html(&target.account_id, &to, &logbook_html, None) @@ -556,8 +571,6 @@ async fn deliver_channel_replies_to_targets( ); } } else { - // Transcript too long for a caption β€” send voice - // without caption, then the full text as a follow-up. if let Err(e) = outbound .send_media(&target.account_id, &to, &payload, reply_to) .await @@ -568,6 +581,7 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel voice reply: {e}" ); + note_delivery_failed(&state, &activity_id).await; } let text_result = if logbook_html.is_empty() { outbound @@ -591,50 +605,24 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send transcript follow-up: {e}" ); + note_delivery_failed(&state, &activity_id).await; } } } }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed β€” - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - }, None => { - let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - crate::channel_acks::note_delivery_failed(&state, &session_key).await; + if !deliver_text_fallback( + outbound.as_ref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + text_already_streamed, + ) + .await + { + note_delivery_failed(&state, &activity_id).await; } }, }, @@ -650,64 +638,84 @@ async fn deliver_channel_replies_to_targets( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send channel voice reply: {e}" ); - } - }, - None if text_already_streamed => { - // TTS disabled/failed but text was already streamed β€” - // only send logbook follow-up if present. - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); + note_delivery_failed(&state, &activity_id).await; } }, None => { - let result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &text, reply_to) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &text, - &logbook_html, - reply_to, - ) - .await - }; - if let Err(e) = result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" - ); - crate::channel_acks::note_delivery_failed(&state, &session_key).await; + if !deliver_text_fallback( + outbound.as_ref(), + &target, + &to, + &text, + &logbook_html, + reply_to, + text_already_streamed, + ) + .await + { + note_delivery_failed(&state, &activity_id).await; } }, }, } - })); + }); } - for task in tasks { - if let Err(e) = task.await { + while let Some(result) = tasks.join_next().await { + if let Err(e) = result { warn!(error = %e, "channel reply task join failed"); // A panicked task may have sent nothing β€” do not claim success. - crate::channel_acks::note_delivery_failed(&state, &session_key).await; + note_delivery_failed(&state, &activity_id).await; } } } +async fn deliver_text_fallback( + outbound: &dyn moltis_channels::ChannelOutbound, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + text: &str, + logbook_html: &str, + reply_to: Option<&str>, + text_already_streamed: bool, +) -> bool { + if text_already_streamed { + if !logbook_html.is_empty() + && let Err(e) = outbound + .send_html(&target.account_id, to, logbook_html, None) + .await + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } + return true; + } + + let result = if logbook_html.is_empty() { + outbound + .send_text(&target.account_id, to, text, reply_to) + .await + } else { + outbound + .send_text_with_suffix(&target.account_id, to, text, logbook_html, reply_to) + .await + }; + if let Err(e) = result { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel reply: {e}" + ); + return false; + } + true +} + #[derive(Debug, Deserialize)] struct TtsStatusResponse { enabled: bool, @@ -733,11 +741,6 @@ struct TtsConvertResponse { mime_type: Option, } -/// Generate TTS audio bytes for a web UI response. -/// -/// Uses the session-level TTS override if configured, otherwise the global TTS -/// config. Returns raw audio bytes (OGG format) on success, `None` if TTS is -/// disabled or generation fails. pub(crate) async fn generate_tts_audio( state: &Arc, session_key: &str, @@ -756,7 +759,6 @@ pub(crate) async fn generate_tts_audio( return Err(error::Error::message("TTS is disabled or not configured")); } - // Layer 2: strip markdown/URLs the LLM may have included despite the prompt. let text = moltis_voice::tts::sanitize_text_for_tts(text); let text = text.trim(); if text.is_empty() { @@ -847,6 +849,7 @@ async fn build_tts_payload( /// response is delivered, instead of being sent as separate messages. pub(crate) async fn send_tool_status_to_channels( state: &Arc, + activity_id: &str, session_key: &str, tool_name: &str, arguments: &Value, @@ -859,7 +862,7 @@ pub(crate) async fn send_tool_status_to_channels( // Drive the acknowledgment reaction into its "tool" phase (πŸ› οΈ/🌐/πŸ’»/…). state .note_channel_activity( - session_key, + activity_id, moltis_channels::ChannelActivity::Tool(tool_name.to_string()), ) .await; @@ -1056,10 +1059,9 @@ fn truncate_url(url: &str) -> String { } } -/// Send a screenshot to all pending channel targets for a session. -/// Uses `peek_channel_replies` so targets remain for the final text response. pub(crate) async fn send_screenshot_to_channels( state: &Arc, + activity_id: &str, session_key: &str, screenshot_data: &str, caption: Option<&str>, @@ -1071,10 +1073,15 @@ pub(crate) async fn send_screenshot_to_channels( let outbound = match state.channel_outbound() { Some(o) => o, - None => return, + None => { + note_delivery_failed(state, activity_id).await; + return; + }, }; - dispatch_screenshot_to_targets(outbound, targets, screenshot_data, caption).await; + if !dispatch_screenshot_to_targets(outbound, targets, screenshot_data, caption).await { + note_delivery_failed(state, activity_id).await; + } } fn build_screenshot_reply_payload( @@ -1108,55 +1115,57 @@ async fn dispatch_screenshot_to_targets( targets: Vec, screenshot_data: &str, caption: Option<&str>, -) { +) -> bool { let payload = build_screenshot_reply_payload(screenshot_data, caption); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let payload = payload.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { + let reply_to = target.message_id.as_deref(); + if let Err(e) = outbound + .send_media(&target.account_id, &to, &payload, reply_to) + .await { - let reply_to = target.message_id.as_deref(); - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send screenshot to channel: {e}" - ); - // Notify the user of the error - let error_msg = format!("⚠️ Failed to send screenshot: {e}"); - let _ = outbound - .send_text(&target.account_id, &to, &error_msg, reply_to) - .await; - } else { - debug!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "sent screenshot to channel" - ); - } + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + "failed to send screenshot to channel: {e}" + ); + let error_msg = format!("⚠️ Failed to send screenshot: {e}"); + let _ = outbound + .send_text(&target.account_id, &to, &error_msg, reply_to) + .await; + false + } else { + debug!( + account_id = target.account_id, + chat_id = target.chat_id, + "sent screenshot to channel" + ); + true } - })); + }); } - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel reply task join failed"); + let mut delivered = true; + while let Some(result) = tasks.join_next().await { + match result { + Ok(sent) => delivered &= sent, + Err(e) => { + warn!(error = %e, "channel reply task join failed"); + delivered = false; + }, } } + delivered } -/// Send a document payload to all pending channel targets for a session. -/// Uses `peek_channel_replies` so targets remain for the final text response. pub(crate) async fn dispatch_document_to_channels( state: &Arc, + activity_id: &str, session_key: &str, payload: moltis_common::types::ReplyPayload, ) { @@ -1167,15 +1176,18 @@ pub(crate) async fn dispatch_document_to_channels( let outbound = match state.channel_outbound() { Some(o) => o, - None => return, + None => { + note_delivery_failed(state, activity_id).await; + return; + }, }; - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let payload = payload.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let reply_to = target.message_id.as_deref(); if let Err(e) = outbound .send_media(&target.account_id, &to, &payload, reply_to) @@ -1191,6 +1203,7 @@ pub(crate) async fn dispatch_document_to_channels( let _ = outbound .send_text(&target.account_id, &to, &error_msg, reply_to) .await; + false } else { debug!( account_id = target.account_id, @@ -1198,15 +1211,24 @@ pub(crate) async fn dispatch_document_to_channels( thread_id = target.thread_id.as_deref().unwrap_or("-"), "sent document to channel" ); + true } - })); + }); } - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel document task join failed"); + let mut delivered = true; + while let Some(result) = tasks.join_next().await { + match result { + Ok(sent) => delivered &= sent, + Err(e) => { + warn!(error = %e, "channel document task join failed"); + delivered = false; + }, } } + if !delivered { + note_delivery_failed(state, activity_id).await; + } } /// Build a `ReplyPayload` from a data URI (legacy path). @@ -1293,6 +1315,7 @@ pub(crate) async fn document_payload_from_ref( /// Uses `peek_channel_replies` so targets remain for the final text response. pub(crate) async fn send_location_to_channels( state: &Arc, + activity_id: &str, session_key: &str, latitude: f64, longitude: f64, @@ -1305,17 +1328,20 @@ pub(crate) async fn send_location_to_channels( let outbound = match state.channel_outbound() { Some(o) => o, - None => return, + None => { + note_delivery_failed(state, activity_id).await; + return; + }, }; let title_owned = title.map(String::from); - let mut tasks = Vec::with_capacity(targets.len()); + let mut tasks = JoinSet::new(); for target in targets { let outbound = Arc::clone(&outbound); let title_ref = title_owned.clone(); let to = target.outbound_to().into_owned(); - tasks.push(tokio::spawn(async move { + tasks.spawn(async move { let reply_to = target.message_id.as_deref(); if let Err(e) = outbound .send_location( @@ -1334,6 +1360,7 @@ pub(crate) async fn send_location_to_channels( thread_id = target.thread_id.as_deref().unwrap_or("-"), "failed to send location to channel: {e}" ); + false } else { debug!( account_id = target.account_id, @@ -1341,160 +1368,25 @@ pub(crate) async fn send_location_to_channels( thread_id = target.thread_id.as_deref().unwrap_or("-"), "sent location pin to channel" ); + true } - })); + }); } - for task in tasks { - if let Err(e) = task.await { - warn!(error = %e, "channel location task join failed"); + let mut delivered = true; + while let Some(result) = tasks.join_next().await { + match result { + Ok(sent) => delivered &= sent, + Err(e) => { + warn!(error = %e, "channel location task join failed"); + delivered = false; + }, } } + if !delivered { + note_delivery_failed(state, activity_id).await; + } } #[cfg(test)] -mod tests { - use { - super::*, - async_trait::async_trait, - moltis_channels::{ChannelReplyTarget, ChannelType}, - moltis_common::types::ReplyPayload, - std::sync::{Arc, Mutex}, - }; - - #[derive(Debug, Clone)] - struct SentMedia { - account_id: String, - to: String, - payload: ReplyPayload, - reply_to: Option, - } - - #[derive(Default)] - struct RecordingOutbound { - sent_media: Mutex>, - } - - #[async_trait] - impl moltis_channels::ChannelOutbound for RecordingOutbound { - async fn send_text( - &self, - _account_id: &str, - _to: &str, - _text: &str, - _reply_to: Option<&str>, - ) -> moltis_channels::Result<()> { - Ok(()) - } - - async fn send_media( - &self, - account_id: &str, - to: &str, - payload: &ReplyPayload, - reply_to: Option<&str>, - ) -> moltis_channels::Result<()> { - self.sent_media - .lock() - .unwrap_or_else(|e| e.into_inner()) - .push(SentMedia { - account_id: account_id.to_string(), - to: to.to_string(), - payload: payload.clone(), - reply_to: reply_to.map(ToString::to_string), - }); - Ok(()) - } - } - - #[test] - fn push_notification_url_uses_chats_prefix_and_replaces_colons() { - // Must match frontend sessionPath(): `/chats/${key.replace(/:/g, "/")}` - assert_eq!(push_notification_url("session:42"), "/chats/session/42"); - } - - #[test] - fn push_notification_url_handles_nested_session_keys() { - assert_eq!( - push_notification_url("telegram:bot123:chat456"), - "/chats/telegram/bot123/chat456" - ); - } - - #[test] - fn push_notification_url_handles_key_without_colons() { - assert_eq!(push_notification_url("main"), "/chats/main"); - } - - #[tokio::test] - async fn generated_image_payload_dispatches_to_telegram_as_media() { - let outbound = Arc::new(RecordingOutbound::default()); - let targets = vec![ChannelReplyTarget { - ack_message_id: None, - channel_type: ChannelType::Telegram, - account_id: "telegram-main".into(), - chat_id: "-100123".into(), - message_id: Some("42".into()), - thread_id: Some("7".into()), - }]; - - dispatch_screenshot_to_targets( - outbound.clone(), - targets, - "data:image/png;base64,cG5n", - Some("Generated image: fox"), - ) - .await; - - let sent = outbound - .sent_media - .lock() - .unwrap_or_else(|e| e.into_inner()); - assert_eq!(sent.len(), 1); - assert_eq!(sent[0].account_id, "telegram-main"); - assert_eq!(sent[0].to, "-100123:7"); - assert_eq!(sent[0].reply_to.as_deref(), Some("42")); - assert_eq!(sent[0].payload.text, "Generated image: fox"); - let Some(media) = sent[0].payload.media.as_ref() else { - panic!("media payload"); - }; - assert_eq!(media.mime_type, "image/png"); - assert_eq!(media.url, "data:image/png;base64,cG5n"); - } - - #[tokio::test] - async fn generated_image_payload_dispatches_to_matrix_as_media() { - let outbound = Arc::new(RecordingOutbound::default()); - let targets = vec![ChannelReplyTarget { - ack_message_id: None, - channel_type: ChannelType::Matrix, - account_id: "matrix-main".into(), - chat_id: "!room:example.org".into(), - message_id: Some("$event".into()), - thread_id: None, - }]; - - dispatch_screenshot_to_targets( - outbound.clone(), - targets, - "data:image/webp;base64,d2VicA==", - Some("Generated image: logo"), - ) - .await; - - let sent = outbound - .sent_media - .lock() - .unwrap_or_else(|e| e.into_inner()); - assert_eq!(sent.len(), 1); - assert_eq!(sent[0].account_id, "matrix-main"); - assert_eq!(sent[0].to, "!room:example.org"); - assert_eq!(sent[0].reply_to.as_deref(), Some("$event")); - assert_eq!(sent[0].payload.text, "Generated image: logo"); - let Some(media) = sent[0].payload.media.as_ref() else { - panic!("media payload"); - }; - assert_eq!(media.mime_type, "image/webp"); - assert_eq!(media.url, "data:image/webp;base64,d2VicA=="); - } -} +mod tests; diff --git a/crates/chat/src/channels/tests.rs b/crates/chat/src/channels/tests.rs new file mode 100644 index 0000000000..b33e0a12b8 --- /dev/null +++ b/crates/chat/src/channels/tests.rs @@ -0,0 +1,218 @@ +use { + super::*, + async_trait::async_trait, + moltis_channels::{ChannelReplyTarget, ChannelType}, + moltis_common::types::ReplyPayload, + std::sync::{Arc, Mutex}, +}; + +#[derive(Debug, Clone)] +struct SentMedia { + account_id: String, + to: String, + payload: ReplyPayload, + reply_to: Option, +} + +#[derive(Debug, Clone)] +struct SentText { + account_id: String, + to: String, + text: String, + reply_to: Option, +} + +#[derive(Default)] +struct RecordingOutbound { + sent_media: Mutex>, + sent_text: Mutex>, + fail_media: bool, + fail_text: bool, +} + +#[async_trait] +impl moltis_channels::ChannelOutbound for RecordingOutbound { + async fn send_text( + &self, + account_id: &str, + to: &str, + text: &str, + reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + if self.fail_text { + return Err(moltis_channels::Error::unavailable("test failure")); + } + self.sent_text + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(SentText { + account_id: account_id.to_string(), + to: to.to_string(), + text: text.to_string(), + reply_to: reply_to.map(ToString::to_string), + }); + Ok(()) + } + + async fn send_media( + &self, + account_id: &str, + to: &str, + payload: &ReplyPayload, + reply_to: Option<&str>, + ) -> moltis_channels::Result<()> { + if self.fail_media { + return Err(moltis_channels::Error::unavailable("test failure")); + } + self.sent_media + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(SentMedia { + account_id: account_id.to_string(), + to: to.to_string(), + payload: payload.clone(), + reply_to: reply_to.map(ToString::to_string), + }); + Ok(()) + } +} + +fn telegram_target() -> ChannelReplyTarget { + ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Telegram, + account_id: "telegram-main".into(), + chat_id: "-100123".into(), + message_id: Some("42".into()), + thread_id: Some("7".into()), + } +} + +#[test] +fn push_notification_url_uses_chats_prefix_and_replaces_colons() { + assert_eq!(push_notification_url("session:42"), "/chats/session/42"); +} + +#[tokio::test] +async fn unavailable_tts_uses_successful_text_as_final_delivery() { + let outbound = RecordingOutbound::default(); + let target = telegram_target(); + + assert!( + deliver_text_fallback( + &outbound, + &target, + "-100123:7", + "fallback transcript", + "", + Some("42"), + false, + ) + .await + ); + + let sent = outbound.sent_text.lock().unwrap_or_else(|e| e.into_inner()); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].account_id, "telegram-main"); + assert_eq!(sent[0].to, "-100123:7"); + assert_eq!(sent[0].text, "fallback transcript"); + assert_eq!(sent[0].reply_to.as_deref(), Some("42")); +} + +#[tokio::test] +async fn failed_text_fallback_is_a_final_delivery_failure() { + let outbound = RecordingOutbound { + fail_text: true, + ..Default::default() + }; + let target = telegram_target(); + + assert!(!deliver_text_fallback(&outbound, &target, "-100123:7", "text", "", None, false).await); +} + +#[tokio::test] +async fn generated_image_payload_dispatches_to_telegram_as_media() { + let outbound = Arc::new(RecordingOutbound::default()); + let targets = vec![telegram_target()]; + + assert!( + dispatch_screenshot_to_targets( + outbound.clone(), + targets, + "data:image/png;base64,cG5n", + Some("Generated image: fox"), + ) + .await + ); + + { + let sent = outbound + .sent_media + .lock() + .unwrap_or_else(|e| e.into_inner()); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].account_id, "telegram-main"); + assert_eq!(sent[0].to, "-100123:7"); + assert_eq!(sent[0].reply_to.as_deref(), Some("42")); + assert_eq!(sent[0].payload.text, "Generated image: fox"); + let Some(media) = sent[0].payload.media.as_ref() else { + panic!("media payload"); + }; + assert_eq!(media.mime_type, "image/png"); + assert_eq!(media.url, "data:image/png;base64,cG5n"); + } + + let failing = Arc::new(RecordingOutbound { + fail_media: true, + ..Default::default() + }); + assert!( + !dispatch_screenshot_to_targets( + failing, + vec![ChannelReplyTarget { + message_id: None, + thread_id: None, + ..telegram_target() + }], + "data:image/png;base64,cG5n", + None, + ) + .await + ); +} + +#[tokio::test] +async fn generated_image_payload_dispatches_to_matrix_as_media() { + let outbound = Arc::new(RecordingOutbound::default()); + let targets = vec![ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Matrix, + account_id: "matrix-main".into(), + chat_id: "!room:example.org".into(), + message_id: Some("$event".into()), + thread_id: None, + }]; + + dispatch_screenshot_to_targets( + outbound.clone(), + targets, + "data:image/webp;base64,d2VicA==", + Some("Generated image: logo"), + ) + .await; + + let sent = outbound + .sent_media + .lock() + .unwrap_or_else(|e| e.into_inner()); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].account_id, "matrix-main"); + assert_eq!(sent[0].to, "!room:example.org"); + assert_eq!(sent[0].reply_to.as_deref(), Some("$event")); + assert_eq!(sent[0].payload.text, "Generated image: logo"); + let Some(media) = sent[0].payload.media.as_ref() else { + panic!("media payload"); + }; + assert_eq!(media.mime_type, "image/webp"); + assert_eq!(media.url, "data:image/webp;base64,d2VicA=="); +} diff --git a/crates/chat/src/run_with_tools.rs b/crates/chat/src/run_with_tools.rs index 22be1d4b31..c724345d9f 100644 --- a/crates/chat/src/run_with_tools.rs +++ b/crates/chat/src/run_with_tools.rs @@ -9,7 +9,10 @@ use std::{ use { serde_json::Value, - tokio::sync::{Mutex, RwLock}, + tokio::{ + sync::{Mutex, RwLock}, + task::{JoinHandle, JoinSet}, + }, tracing::{info, warn}, }; @@ -33,7 +36,8 @@ use { use crate::{ ActiveToolCall, LiveChatService, agent_loop::{ - ChannelStreamDispatcher, clear_unsupported_model, compact_session, mark_unsupported_model, + ChannelStreamDispatcher, clear_unsupported_model, + commit_terminal_and_finish_channel_stream, compact_session, mark_unsupported_model, ordered_runner_event_callback, }, channels::{ @@ -51,13 +55,30 @@ use crate::{ prompt_build_limits_from_config, }, runtime::ChatRuntime, - service::{ActiveAssistantDraft, build_tool_call_assistant_message, persist_tool_history_pair}, + service::{ + ActiveAssistantDraft, EventForwarder, build_tool_call_assistant_message, + persist_tool_history_pair, + }, types::*, }; #[cfg(feature = "push-notifications")] use crate::channels::send_chat_push_notification; +struct AbortTask(JoinHandle<()>); + +impl AbortTask { + fn new(task: JoinHandle<()>) -> Self { + Self(task) + } +} + +impl Drop for AbortTask { + fn drop(&mut self) { + self.0.abort(); + } +} + pub(crate) async fn run_with_tools( persona: PromptPersona, state: &Arc, @@ -85,7 +106,7 @@ pub(crate) async fn run_with_tools( active_thinking_text: Option>>>, active_tool_calls: Option>>>>, active_partial_assistant: Option>>>, - active_event_forwarders: &Arc>>>, + active_event_forwarders: &Arc>>, terminal_runs: &Arc>>, sender_name: Option, tool_controls: Option, @@ -281,10 +302,11 @@ pub(crate) async fn run_with_tools( .await .map(|dispatcher| Arc::new(Mutex::new(dispatcher))); let channel_stream_for_events = channel_stream_dispatcher.as_ref().map(Arc::clone); - let event_forwarder = tokio::spawn(async move { + let event_forwarder_task = tokio::spawn(async move { // Track tool call arguments from ToolCallStart so they can be persisted in ToolCallEnd. let mut tool_args_map: HashMap = HashMap::new(); let mut tool_metadata_map: HashMap> = HashMap::new(); + let mut delivery_tasks = JoinSet::new(); // Track reasoning text that should be persisted with the first tool call after thinking. let mut tool_reasoning_map: HashMap = HashMap::new(); let mut latest_reasoning = String::new(); @@ -341,11 +363,13 @@ pub(crate) async fn run_with_tools( // Send tool status to channels (Telegram, etc.) let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); + let activity_id = run_id.clone(); let name_clone = name.clone(); let args_clone = arguments.clone(); - tokio::spawn(async move { + delivery_tasks.spawn(async move { send_tool_status_to_channels( &state_clone, + &activity_id, &sk_clone, &name_clone, &args_clone, @@ -511,9 +535,11 @@ pub(crate) async fn run_with_tools( if let Some((lat, lon, label)) = location_to_send { let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); - tokio::spawn(async move { + let activity_id = run_id.clone(); + delivery_tasks.spawn(async move { send_location_to_channels( &state_clone, + &activity_id, &sk_clone, lat, lon, @@ -527,9 +553,11 @@ pub(crate) async fn run_with_tools( if let Some(screenshot_data) = screenshot_to_send { let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); - tokio::spawn(async move { + let activity_id = run_id.clone(); + delivery_tasks.spawn(async move { send_screenshot_to_channels( &state_clone, + &activity_id, &sk_clone, &screenshot_data, image_caption.as_deref(), @@ -543,10 +571,11 @@ pub(crate) async fn run_with_tools( // New path: read from media dir at upload time. let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); + let activity_id = run_id.clone(); let store_clone = store.clone(); let mime = document_ref_mime .unwrap_or_else(|| "application/octet-stream".to_string()); - tokio::spawn(async move { + delivery_tasks.spawn(async move { if let Some(payload) = document_payload_from_ref( store_clone.as_ref(), &sk_clone, @@ -557,21 +586,39 @@ pub(crate) async fn run_with_tools( ) .await { - dispatch_document_to_channels(&state_clone, &sk_clone, payload) - .await; + dispatch_document_to_channels( + &state_clone, + &activity_id, + &sk_clone, + payload, + ) + .await; + } else { + crate::channel_acks::note_delivery_failed( + &state_clone, + &activity_id, + ) + .await; } }); } else if let Some(document_data) = document_to_send { // Legacy fallback: data URI. let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); + let activity_id = run_id.clone(); let payload = document_payload_from_data_uri( &document_data, document_filename.as_deref(), document_caption.as_deref(), ); - tokio::spawn(async move { - dispatch_document_to_channels(&state_clone, &sk_clone, payload).await; + delivery_tasks.spawn(async move { + dispatch_document_to_channels( + &state_clone, + &activity_id, + &sk_clone, + payload, + ) + .await; }); } @@ -591,7 +638,7 @@ pub(crate) async fn run_with_tools( let store_media = Arc::clone(store); let sk_media = sk.clone(); let tool_call_id = id.clone(); - let persisted_result = result.as_ref().map(|res| { + let persisted_result = if let Some(res) = result.as_ref() { let mut r = res.clone(); // Try to decode and persist the screenshot to the media // directory. Extract base64 into an owned Vec first to @@ -607,18 +654,17 @@ pub(crate) async fn run_with_tools( }); if let Some(bytes) = decoded_screenshot { let filename = format!("{tool_call_id}.png"); - let store_ref = Arc::clone(&store_media); - let sk_ref = sk_media.clone(); - tokio::spawn(async move { - if let Err(e) = - store_ref.save_media(&sk_ref, &filename, &bytes).await - { + match store_media.save_media(&sk_media, &filename, &bytes).await { + Ok(_) => { + let sanitized = SessionStore::key_to_filename(&sk_media); + r["screenshot"] = Value::String(format!( + "media/{sanitized}/{tool_call_id}.png" + )); + }, + Err(e) => { warn!("failed to save screenshot media: {e}"); - } - }); - let sanitized = SessionStore::key_to_filename(&sk_media); - r["screenshot"] = - Value::String(format!("media/{sanitized}/{tool_call_id}.png")); + }, + } } // If screenshot is still a data URI (decode failed), strip it. let strip_screenshot = r @@ -653,8 +699,10 @@ pub(crate) async fn run_with_tools( r[*field] = Value::String(truncated); } } - r - }); + Some(r) + } else { + None + }; let tracked_reasoning = tool_reasoning_map.remove(&id); let tracked_metadata = tool_metadata_map.remove(&id); let assistant_tool_call_msg = build_tool_call_assistant_message( @@ -788,7 +836,7 @@ pub(crate) async fn run_with_tools( let state_clone = Arc::clone(&state); let sk_clone = sk.clone(); let error_clone = error_obj.clone(); - tokio::spawn(async move { + delivery_tasks.spawn(async move { send_retry_status_to_channels( &state_clone, &sk_clone, @@ -858,12 +906,16 @@ pub(crate) async fn run_with_tools( }; broadcast(&state, "chat", payload, BroadcastOpts::default()).await; } + if !join_channel_delivery_tasks(&mut delivery_tasks).await { + crate::channel_acks::note_delivery_failed(&state_for_events, &run_id_for_events).await; + } latest_reasoning }); + let event_forwarder = EventForwarder::new(event_forwarder_task, session_key.to_string()); active_event_forwarders .write() .await - .insert(session_key.to_string(), event_forwarder); + .insert(run_id.to_string(), event_forwarder); // Convert persisted JSON history to typed ChatMessages for the LLM provider. let chat_history = values_to_chat_messages_with_tool_result_limit( @@ -911,7 +963,7 @@ pub(crate) async fn run_with_tools( let steer_inbox_writer = steer_inbox.clone(); let steer_state = state.clone(); let steer_session_key = session_key.to_string(); - let steer_task = tokio::spawn(async move { + let steer_task = AbortTask::new(tokio::spawn(async move { // Drain any stale steering text left over from a previous run. let _ = steer_state.take_steer_text(&steer_session_key).await; loop { @@ -920,7 +972,7 @@ pub(crate) async fn run_with_tools( steer_inbox_writer.lock().await.extend(texts); } } - }); + })); let provider_ref = provider.clone(); let first_agent_future = run_agent_loop_streaming_with_limits( @@ -1075,25 +1127,17 @@ pub(crate) async fn run_with_tools( }, other => other, }; - steer_task.abort(); + drop(steer_task); // Ensure all runner events (including deltas) are broadcast in order before // emitting terminal final/error frames. drop(on_event); let reasoning_text = - LiveChatService::wait_for_event_forwarder(active_event_forwarders, session_key).await; + LiveChatService::wait_for_event_forwarder(active_event_forwarders, run_id).await; let reasoning = { let trimmed = reasoning_text.trim(); (!trimmed.is_empty()).then(|| trimmed.to_string()) }; - let streamed_target_keys = if let Some(ref dispatcher) = channel_stream_dispatcher { - let mut dispatcher = dispatcher.lock().await; - dispatcher.finish().await; - dispatcher.completed_target_keys().await - } else { - HashSet::new() - }; - match result { Ok(result) => { clear_unsupported_model(state, model_store, model_id).await; @@ -1129,17 +1173,27 @@ pub(crate) async fn run_with_tools( "The provider returned an empty response (possible network error). Please try again.", Some(provider_name), ); - deliver_channel_error(state, session_key, &error_obj).await; let error_payload = ChatErrorBroadcast { run_id: run_id.to_string(), session_key: session_key.to_string(), state: "error", - error: error_obj, + error: error_obj.clone(), seq: client_seq, }; #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); + if let Some(ref dispatcher) = channel_stream_dispatcher { + let mut dispatcher = dispatcher.lock().await; + commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + Some(&mut dispatcher), + ) + .await; + } else { + commit_terminal_and_finish_channel_stream(terminal_runs, run_id, None).await; + } + deliver_channel_error(state, session_key, &error_obj).await; broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; return None; } @@ -1203,8 +1257,20 @@ pub(crate) async fn run_with_tools( ); #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&final_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + + // From this point abort must lose: Done or a fallback reply can be + // accepted by a channel immediately and cannot be rolled back. + let streamed_target_keys = if let Some(ref dispatcher) = channel_stream_dispatcher { + let mut dispatcher = dispatcher.lock().await; + commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + Some(&mut dispatcher), + ) + .await + } else { + commit_terminal_and_finish_channel_stream(terminal_runs, run_id, None).await + }; if !is_silent { // Send push notification when chat response completes @@ -1215,6 +1281,7 @@ pub(crate) async fn run_with_tools( } deliver_channel_replies( state, + run_id, session_key, &display_text, desired_reply_medium, @@ -1222,14 +1289,16 @@ pub(crate) async fn run_with_tools( ) .await; } - Some(build_assistant_turn_output( + let mut output = build_assistant_turn_output( display_text, UsageSnapshot::new(usage, Some(request_usage)), run_started.elapsed().as_millis() as u64, audio_path, reasoning, llm_api_response, - )) + ); + output.final_broadcast = Some(payload_val); + Some(output) }, Err(e) => { let error_str = e.to_string(); @@ -1237,23 +1306,44 @@ pub(crate) async fn run_with_tools( state.set_run_error(run_id, error_str.clone()).await; let error_obj = parse_chat_error(&error_str, Some(provider_name)); mark_unsupported_model(state, model_store, model_id, provider_name, &error_obj).await; - deliver_channel_error(state, session_key, &error_obj).await; let error_payload = ChatErrorBroadcast { run_id: run_id.to_string(), session_key: session_key.to_string(), state: "error", - error: error_obj, + error: error_obj.clone(), seq: client_seq, }; #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); + if let Some(ref dispatcher) = channel_stream_dispatcher { + let mut dispatcher = dispatcher.lock().await; + commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + Some(&mut dispatcher), + ) + .await; + } else { + commit_terminal_and_finish_channel_stream(terminal_runs, run_id, None).await; + } + deliver_channel_error(state, session_key, &error_obj).await; broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; None }, } } +async fn join_channel_delivery_tasks(tasks: &mut JoinSet<()>) -> bool { + let mut completed = true; + while let Some(result) = tasks.join_next().await { + if let Err(error) = result { + warn!(%error, "run-scoped channel delivery task failed"); + completed = false; + } + } + completed +} + async fn await_with_agent_timeout( timeout_secs: u64, started: Instant, @@ -1324,66 +1414,9 @@ fn escape_xml(s: &str) -> String { } #[cfg(test)] -mod tests { - use super::*; - - fn mock_result(path: &str, text: &str) -> moltis_memory::search::SearchResult { - moltis_memory::search::SearchResult { - chunk_id: "c1".into(), - path: path.into(), - source: "test".into(), - start_line: 1, - end_line: 1, - score: 0.9, - text: text.into(), - } - } - - #[test] - fn test_format_recalled_context_empty() { - assert_eq!(format_recalled_context(&[]), ""); - } - - #[test] - fn test_format_recalled_context_basic() { - let results = vec![mock_result("memory/2026.md", "User prefers Rust.")]; - let ctx = format_recalled_context(&results); - assert!(ctx.contains("")); - assert!(ctx.contains("")); - assert!(ctx.contains("[memory/2026.md]")); - assert!(ctx.contains("User prefers Rust.")); - } +#[path = "run_with_tools/run_scope_tests.rs"] +mod run_scope_tests; - #[test] - fn test_format_recalled_context_escapes_xml() { - let results = vec![mock_result( - "memory/test.md", - "ignore previous", - )]; - let ctx = format_recalled_context(&results); - assert!( - !ctx.contains(""), - "XML metacharacters must be escaped: {ctx}" - ); - assert!(ctx.contains("</recalled_context>")); - } - - #[test] - fn test_format_recalled_context_truncates_long_text() { - let long_text = "x".repeat(500); - let results = vec![mock_result("m.md", &long_text)]; - let ctx = format_recalled_context(&results); - // Should contain truncation marker. - assert!(ctx.contains('…')); - // Should not contain the full 500-char string. - assert!(!ctx.contains(&long_text)); - } - - #[test] - fn test_format_recalled_context_replaces_newlines() { - let results = vec![mock_result("m.md", "line1\nline2\nline3")]; - let ctx = format_recalled_context(&results); - assert!(!ctx.contains('\n') || !ctx.contains("line1\nline2")); - assert!(ctx.contains("line1 line2 line3")); - } -} +#[cfg(test)] +#[path = "run_with_tools/tests.rs"] +mod tests; diff --git a/crates/chat/src/run_with_tools/run_scope_tests.rs b/crates/chat/src/run_with_tools/run_scope_tests.rs new file mode 100644 index 0000000000..c78e227ad9 --- /dev/null +++ b/crates/chat/src/run_with_tools/run_scope_tests.rs @@ -0,0 +1,69 @@ +use {super::*, std::task::Poll}; + +#[tokio::test] +async fn run_scoped_delivery_join_waits_for_completion() { + let (release, released) = tokio::sync::oneshot::channel(); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let task_events = Arc::clone(&events); + let mut tasks = JoinSet::new(); + tasks.spawn(async move { + let _ = released.await; + task_events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("status"); + }); + let joined = join_channel_delivery_tasks(&mut tasks); + tokio::pin!(joined); + std::future::poll_fn(|cx| { + assert!(matches!(joined.as_mut().poll(cx), Poll::Pending)); + Poll::Ready(()) + }) + .await; + + assert!(release.send(()).is_ok()); + assert!(joined.await); + events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("final"); + assert_eq!(*events.lock().unwrap_or_else(|error| error.into_inner()), [ + "status", "final" + ]); +} + +#[tokio::test] +async fn dropping_run_scoped_delivery_tasks_cancels_late_status_side_effect() { + struct DropNotice(Option>); + impl Drop for DropNotice { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + + let leaked = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let leaked_by_task = Arc::clone(&leaked); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let mut tasks = JoinSet::new(); + tasks.spawn(async move { + let _notice = DropNotice(Some(dropped_tx)); + let _ = started_tx.send(()); + let _ = release_rx.await; + leaked_by_task.store(true, std::sync::atomic::Ordering::SeqCst); + }); + assert!(started_rx.await.is_ok()); + + drop(tasks); + + assert!( + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .is_ok() + ); + assert!(release_tx.send(()).is_err()); + assert!(!leaked.load(std::sync::atomic::Ordering::SeqCst)); +} diff --git a/crates/chat/src/run_with_tools/tests.rs b/crates/chat/src/run_with_tools/tests.rs new file mode 100644 index 0000000000..5cbbae6ff9 --- /dev/null +++ b/crates/chat/src/run_with_tools/tests.rs @@ -0,0 +1,87 @@ +use super::*; + +fn mock_result(path: &str, text: &str) -> moltis_memory::search::SearchResult { + moltis_memory::search::SearchResult { + chunk_id: "c1".into(), + path: path.into(), + source: "test".into(), + start_line: 1, + end_line: 1, + score: 0.9, + text: text.into(), + } +} + +#[tokio::test] +async fn steering_task_is_aborted_when_guard_is_dropped() { + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + struct DropNotice(Option>); + impl Drop for DropNotice { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + + let guard = AbortTask::new(tokio::spawn(async move { + let _notice = DropNotice(Some(dropped_tx)); + let _ = started_tx.send(()); + std::future::pending::<()>().await; + })); + assert!(started_rx.await.is_ok()); + drop(guard); + + assert!( + tokio::time::timeout(Duration::from_secs(1), dropped_rx) + .await + .is_ok() + ); +} + +#[test] +fn test_format_recalled_context_empty() { + assert_eq!(format_recalled_context(&[]), ""); +} + +#[test] +fn test_format_recalled_context_basic() { + let results = vec![mock_result("memory/2026.md", "User prefers Rust.")]; + let ctx = format_recalled_context(&results); + assert!(ctx.contains("")); + assert!(ctx.contains("")); + assert!(ctx.contains("[memory/2026.md]")); + assert!(ctx.contains("User prefers Rust.")); +} + +#[test] +fn test_format_recalled_context_escapes_xml() { + let results = vec![mock_result( + "memory/test.md", + "ignore previous", + )]; + let ctx = format_recalled_context(&results); + assert!( + !ctx.contains(""), + "XML metacharacters must be escaped: {ctx}" + ); + assert!(ctx.contains("</recalled_context>")); +} + +#[test] +fn test_format_recalled_context_truncates_long_text() { + let long_text = "x".repeat(500); + let results = vec![mock_result("m.md", &long_text)]; + let ctx = format_recalled_context(&results); + assert!(ctx.contains('…')); + assert!(!ctx.contains(&long_text)); +} + +#[test] +fn test_format_recalled_context_replaces_newlines() { + let results = vec![mock_result("m.md", "line1\nline2\nline3")]; + let ctx = format_recalled_context(&results); + assert!(!ctx.contains('\n') || !ctx.contains("line1\nline2")); + assert!(ctx.contains("line1 line2 line3")); +} diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index f9d54a7ca6..4d05480f7a 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -139,12 +139,18 @@ pub trait ChatRuntime: Send + Sync { /// Default no-op: runtimes without channel reactions (tests, headless) /// ignore it. Emitted from the agent loop (Thinking/Tool) and from the run /// completion (Finished). - async fn note_channel_activity(&self, _session_key: &str, _activity: ChannelActivity) {} + async fn note_channel_activity(&self, _activity_id: &str, _activity: ChannelActivity) {} /// Bind the given acknowledgment keys to this session's now-executing run, /// so activity routes to exactly the inbound message(s) this run handles. /// Called once the queue decision is known and the run actually starts. - async fn activate_channel_acks(&self, _session_key: &str, _ack_keys: Vec) {} + async fn activate_channel_acks( + &self, + _activity_id: &str, + _session_key: &str, + _ack_keys: Vec, + ) { + } /// Finalize acknowledgment keys directly, for paths where no run executes /// (rejected hook, early error) and nothing else will signal a terminal. @@ -154,7 +160,7 @@ pub trait ChatRuntime: Send + Sync { /// terminal. Turn-addressed, so it cannot resolve a turn that starts later. async fn finalize_active_channel_acks( &self, - _session_key: &str, + _activity_id: &str, _outcome: moltis_channels::ChannelAckOutcome, ) { } diff --git a/crates/chat/src/service/chat_impl.rs b/crates/chat/src/service/chat_impl.rs index 64b3886899..18a5ec39a9 100644 --- a/crates/chat/src/service/chat_impl.rs +++ b/crates/chat/src/service/chat_impl.rs @@ -47,12 +47,15 @@ use crate::{ prompt_build_limits_from_config, resolve_prompt_agent_id, resolve_prompt_mode_context, }, run_with_tools::run_with_tools, - service::build_persisted_assistant_message, + service::{ + build_persisted_assistant_message, + types::{cancel_queued_messages, commit_successful_turn}, + }, streaming::run_streaming, types::*, }; -use super::*; +use super::{types::RunHandleDisposition, *}; #[async_trait] impl ChatService for LiveChatService { @@ -227,7 +230,7 @@ impl ChatService for LiveChatService { let user_content = UserContent::text(&text); let active_event_forwarders = Arc::new(RwLock::new(HashMap::new())); let terminal_runs = Arc::new(RwLock::new(HashSet::new())); - let result = if stream_only { + let mut result = if stream_only { run_streaming( persona, &state, @@ -288,26 +291,44 @@ impl ChatService for LiveChatService { .await }; - // Persist assistant response (even empty ones β€” needed for LLM history coherence). - if !ephemeral && let Some(ref assistant_output) = result { - let assistant_msg = build_persisted_assistant_message( - assistant_output.clone(), - Some(model_id.clone()), - Some(provider_name.clone()), - None, - Some(run_id.clone()), - ); - if let Err(e) = self - .session_store - .append(&session_key, &assistant_msg.to_value()) - .await - { - warn!("send_sync: failed to persist assistant message: {e}"); - } - // Update metadata message count. - if let Ok(count) = self.session_store.count(&session_key).await { - self.session_metadata.touch(&session_key, count).await; - } + let final_payload = result + .as_mut() + .and_then(|assistant_output| assistant_output.final_broadcast.take()); + + if let Some(ref assistant_output) = result { + let assistant_msg = (!ephemeral).then(|| { + build_persisted_assistant_message( + assistant_output.clone(), + Some(model_id.clone()), + Some(provider_name.clone()), + None, + Some(run_id.clone()), + ) + }); + commit_successful_turn( + &terminal_runs, + &run_id, + async { + if let Some(assistant_msg) = assistant_msg { + if let Err(e) = self + .session_store + .append(&session_key, &assistant_msg.to_value()) + .await + { + warn!("send_sync: failed to persist assistant message: {e}"); + } + if let Ok(count) = self.session_store.count(&session_key).await { + self.session_metadata.touch(&session_key, count).await; + } + } + }, + async { + if let Some(payload) = final_payload { + broadcast(&state, "chat", payload, BroadcastOpts::default()).await; + } + }, + ) + .await; } match result { @@ -356,71 +377,122 @@ impl ChatService for LiveChatService { let resolved_session_key = Self::resolve_session_key_for_run(&self.active_runs_by_session, run_id, session_key) .await; + let Some(resolved_session_key) = resolved_session_key else { + return Ok(serde_json::json!({ + "aborted": false, + "runId": run_id, + "sessionKey": session_key, + })); + }; + let resolved_run_id = match run_id { + Some(id) => id.to_string(), + None => { + let Some(id) = self + .active_runs_by_session + .read() + .await + .get(&resolved_session_key) + .cloned() + else { + return Ok(serde_json::json!({ + "aborted": false, + "runId": null, + "sessionKey": resolved_session_key, + })); + }; + id + }, + }; - // Take ownership of this turn's acknowledgments *before* killing the - // run. Aborting drops the task and with it the session permit, so a new - // turn can activate immediately; finalizing afterwards could resolve - // that newer turn instead of the one being aborted. - if let Some(key) = resolved_session_key.as_deref() { - self.state - .finalize_active_channel_acks(key, moltis_channels::ChannelAckOutcome::Cancelled) - .await; - } - - let (resolved_run_id, aborted) = Self::abort_run_handle( + let session_sem = self.session_semaphore(&resolved_session_key).await; + let (_, claim) = Self::claim_run_for_abort( &self.active_runs, &self.active_runs_by_session, &self.terminal_runs, - run_id, - session_key, + session_sem, + Some(&resolved_run_id), + Some(&resolved_session_key), ) .await; + let disposition = claim.disposition; + let aborted = disposition == RunHandleDisposition::Aborted; + let stale = disposition == RunHandleDisposition::Stale; info!( requested_run_id = ?run_id, session_key = ?session_key, - resolved_run_id = ?resolved_run_id, + resolved_run_id = %resolved_run_id, aborted, + stale, "chat.abort" ); - if aborted && let Some(key) = resolved_session_key.as_deref() { - let _ = Self::wait_for_event_forwarder(&self.active_event_forwarders, key).await; - let partial = self.persist_partial_assistant_on_abort(key).await; - self.active_thinking_text.write().await.remove(key); - self.active_tool_calls.write().await.remove(key); - self.active_reply_medium.write().await.remove(key); - let mut payload = serde_json::json!({ - "state": "aborted", + if disposition == RunHandleDisposition::Unavailable { + return Ok(serde_json::json!({ + "aborted": false, "runId": resolved_run_id, - "sessionKey": key, - }); - if let Some((partial_message, message_index)) = partial { - payload["partialMessage"] = partial_message; - if let Some(index) = message_index { - payload["messageIndex"] = serde_json::json!(index); - } - } - broadcast(&self.state, "chat", payload, BroadcastOpts::default()).await; + "sessionKey": resolved_session_key, + })); } - // The aborted future was killed mid-flight, so it never ran its own - // drain step. Without this, anything queued behind it waits forever. - if aborted && let Some(key) = resolved_session_key.clone() { - let queue = Arc::clone(&self.message_queue); - let state = Arc::clone(&self.state); - let mode = self.config.chat.message_queue_mode; - let session_sem = self.session_semaphore(&key).await; - tokio::spawn(async move { - // Wait for the aborted task to release the session permit, - // otherwise the replay would simply re-queue itself. - let permit = session_sem.acquire_owned().await; - drop(permit); - queue_drain::drain_and_replay(&queue, &key, mode, &state).await; - }); + // The forwarder owns run-scoped channel delivery tasks. Cancel it before + // waiting for the run permit so abort cleanup cannot be held hostage by + // channel I/O. + Self::cancel_event_forwarder(&self.active_event_forwarders, &resolved_run_id).await; + + let cleanup_permit = claim + .cleanup + .ok_or_else(|| ServiceError::message("abort cleanup reservation missing"))? + .acquire() + .await + .map_err(|_| ServiceError::message("session semaphore closed during abort"))?; + + self.state + .finalize_active_channel_acks( + &resolved_run_id, + moltis_channels::ChannelAckOutcome::Cancelled, + ) + .await; + let partial = self + .persist_partial_assistant_on_abort(&resolved_session_key, &resolved_run_id) + .await; + self.active_thinking_text + .write() + .await + .remove(&resolved_session_key); + self.active_tool_calls + .write() + .await + .remove(&resolved_session_key); + self.active_reply_medium + .write() + .await + .remove(&resolved_session_key); + self.terminal_runs.write().await.remove(&resolved_run_id); + let mut payload = serde_json::json!({ + "state": "aborted", + "runId": resolved_run_id, + "sessionKey": resolved_session_key, + }); + if let Some((partial_message, message_index)) = partial { + payload["partialMessage"] = partial_message; + if let Some(index) = message_index { + payload["messageIndex"] = serde_json::json!(index); + } } + broadcast(&self.state, "chat", payload, BroadcastOpts::default()).await; + + drop(cleanup_permit); + queue_drain::drain_and_replay( + &self.message_queue, + &resolved_session_key, + self.config.chat.message_queue_mode, + &self.state, + ) + .await; Ok(serde_json::json!({ "aborted": aborted, + "cleanedStale": stale, "runId": resolved_run_id, "sessionKey": resolved_session_key, })) @@ -432,12 +504,10 @@ impl ChatService for LiveChatService { .and_then(|v| v.as_str()) .ok_or_else(|| "missing 'sessionKey'".to_string())?; - let removed = self - .message_queue - .write() - .await - .remove(session_key) - .unwrap_or_default(); + let removed: Vec = { + let mut queues = self.message_queue.write().await; + cancel_queued_messages(&mut queues, session_key) + }; let count = removed.len(); info!(session = %session_key, count, "cancel_queued: cleared message queue"); diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs index 7f3a91724f..1e6f65a59c 100644 --- a/crates/chat/src/service/chat_impl/queue_drain.rs +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -1,7 +1,4 @@ //! Replay of messages that queued behind an active run. -//! -//! Shared by both run paths (model turns and explicit `/sh` commands), which -//! previously carried identical copies of this logic. use std::{collections::HashMap, sync::Arc}; @@ -13,184 +10,189 @@ use { use moltis_config::MessageQueueMode; -use crate::{runtime::ChatRuntime, service::types::QueuedMessage}; +use crate::{ + runtime::ChatRuntime, + service::types::{QueuedMessage, SessionMessageQueue}, +}; + +type MessageQueues = Arc>>; -/// Drain this session's queue and replay it according to `mode`. -/// -/// `Followup` replays the oldest message and puts the rest back, so the -/// replayed run drains them in turn. `Collect` merges every queued message into -/// one turn β€” which therefore owns all of their acknowledgment identities, so a -/// reaction is resolved on each of the combined messages rather than only the -/// last. +/// Drain in FIFO order. A replay that starts a run delegates the remaining +/// drain to that run; a replay with no run settles its ack and continues here. pub(super) async fn drain_and_replay( - message_queue: &Arc>>>, + message_queue: &MessageQueues, session_key: &str, mode: MessageQueueMode, state: &Arc, ) { - let queued = message_queue - .write() - .await - .remove(session_key) - .unwrap_or_default(); - if queued.is_empty() { - return; - } let chat = state.chat_service().await; - match mode { - MessageQueueMode::Followup => { - let mut iter = queued.into_iter(); - let Some(first) = iter.next() else { - return; - }; - // Put remaining messages back so the replayed run's own drain loop - // picks them up after it completes. - let rest: Vec = iter.collect(); - if !rest.is_empty() { - message_queue - .write() - .await - .entry(session_key.to_string()) - .or_default() - .extend(rest); - } - info!(session = %session_key, "replaying queued message (followup)"); - let keys = crate::channel_acks::ack_keys_from_params(&first.params); - let mut replay_params = first.params; - replay_params["_queued_replay"] = Value::Bool(true); - let result = chat.send(replay_params).await; - settle_replay(state, session_key, keys, &result).await; - }, - MessageQueueMode::Collect => { - // Merge *all* content, not just text. `send` prefers `content` over - // `text` when both are present, so a multimodal message anywhere in - // the batch would otherwise discard every other message's words - // while their acknowledgments still reported success. - let multimodal = queued.iter().any(|m| m.params.get("content").is_some()); - let combined: Vec<&str> = queued - .iter() - .filter_map(|m| m.params.get("text").and_then(|v| v.as_str())) - .collect(); - if multimodal { - let blocks = merge_content_blocks(&queued); - if blocks.is_empty() { - abandon(state, &queued).await; - return; - } - let Some(last) = queued.last() else { - abandon(state, &queued).await; - return; + loop { + let queued = take_next(message_queue, session_key, mode).await; + if queued.is_empty() { + finish_draining(message_queue, session_key).await; + return; + } + + let (replay_params, keys) = match mode { + MessageQueueMode::Followup => { + let Some(first) = queued.into_iter().next() else { + continue; }; - info!( - session = %session_key, - count = queued.len(), - "replaying collected messages (multimodal)" - ); - let mut merged = last.params.clone(); - merged["content"] = Value::Array(blocks); - // `text` would be ignored anyway; drop it so the two cannot - // disagree about what was actually sent. - if let Some(obj) = merged.as_object_mut() { - obj.remove("text"); - } - merged["_queued_replay"] = Value::Bool(true); - merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( - crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) - ); - let keys = crate::channel_acks::ack_keys_from_params(&merged); - let result = chat.send(merged).await; - settle_replay(state, session_key, keys, &result).await; - return; - } - if combined.is_empty() { - // Nothing replayable (e.g. every queued message was non-text), - // so no reply will ever come. Resolve their acknowledgments - // instead of leaving markers to expire. - abandon(state, &queued).await; - return; - } - info!( - session = %session_key, - count = combined.len(), - "replaying collected messages" - ); - // Use the last queued message as the base params, override text. - let Some(last) = queued.last() else { - abandon(state, &queued).await; + info!(session = %session_key, "replaying queued message (followup)"); + let keys = crate::channel_acks::ack_keys_from_params(&first.params); + let mut params = first.params; + params["_queued_replay"] = Value::Bool(true); + (Some(params), keys) + }, + MessageQueueMode::Collect => prepare_collected(state, session_key, queued).await, + }; + let Some(replay_params) = replay_params else { + continue; + }; + + let result = chat.send(replay_params).await; + settle_replay(state, session_key, keys, &result).await; + match replay_disposition(&result) { + ReplayDisposition::Terminal => continue, + ReplayDisposition::Started | ReplayDisposition::Queued => { + finish_draining(message_queue, session_key).await; return; - }; - let mut merged = last.params.clone(); - merged["text"] = Value::String(combined.join("\n\n")); - merged["_queued_replay"] = Value::Bool(true); - // One run answers all of them, so it owns every acknowledgment. - merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( - crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) - ); - let keys = crate::channel_acks::ack_keys_from_params(&merged); - let result = chat.send(merged).await; - settle_replay(state, session_key, keys, &result).await; - }, + }, + } } } -/// Flatten every queued message into one ordered list of content blocks. -/// -/// Text-only messages contribute a single text block; multimodal messages -/// contribute their blocks verbatim, so images survive the merge. -fn merge_content_blocks(queued: &[QueuedMessage]) -> Vec { - let mut blocks = Vec::new(); - for m in queued { - match m.params.get("content").and_then(Value::as_array) { - Some(content) => blocks.extend(content.iter().cloned()), - None => { - if let Some(text) = m.params.get("text").and_then(Value::as_str) - && !text.is_empty() - { - blocks.push(serde_json::json!({ "type": "text", "text": text })); - } - }, +async fn take_next( + message_queue: &MessageQueues, + session_key: &str, + mode: MessageQueueMode, +) -> Vec { + let mut queues = message_queue.write().await; + let Some(queue) = queues.get_mut(session_key) else { + return Vec::new(); + }; + queue.draining = true; + match mode { + MessageQueueMode::Followup => queue.messages.pop_front().into_iter().collect(), + MessageQueueMode::Collect => queue.messages.drain(..).collect(), + } +} + +async fn finish_draining(message_queue: &MessageQueues, session_key: &str) { + let mut queues = message_queue.write().await; + let remove = if let Some(queue) = queues.get_mut(session_key) { + queue.draining = false; + queue.messages.is_empty() + } else { + false + }; + if remove { + queues.remove(session_key); + } +} + +async fn prepare_collected( + state: &Arc, + session_key: &str, + queued: Vec, +) -> (Option, Vec) { + let multimodal = queued.iter().any(|m| m.params.get("content").is_some()); + if multimodal { + let blocks = merge_content_blocks(&queued); + if blocks.is_empty() { + abandon(state, &queued).await; + return (None, Vec::new()); + } + let Some(last) = queued.last() else { + abandon(state, &queued).await; + return (None, Vec::new()); + }; + info!(session = %session_key, count = queued.len(), "replaying collected messages (multimodal)"); + let mut merged = last.params.clone(); + merged["content"] = Value::Array(blocks); + if let Some(obj) = merged.as_object_mut() { + obj.remove("text"); + obj.remove("message"); } + merged["_queued_replay"] = Value::Bool(true); + merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( + crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) + ); + let keys = crate::channel_acks::ack_keys_from_params(&merged); + return (Some(merged), keys); + } + let Some(mut merged) = merge_collected_text(&queued) else { + abandon(state, &queued).await; + return (None, Vec::new()); + }; + info!(session = %session_key, count = queued.len(), "replaying collected messages"); + merged["_queued_replay"] = Value::Bool(true); + merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( + crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) + ); + let keys = crate::channel_acks::ack_keys_from_params(&merged); + (Some(merged), keys) +} + +fn message_text(params: &Value) -> Option<&str> { + params + .get("text") + .or_else(|| params.get("message")) + .and_then(Value::as_str) +} + +fn merge_collected_text(queued: &[QueuedMessage]) -> Option { + let combined: Vec<&str> = queued + .iter() + .filter_map(|message| message_text(&message.params)) + .collect(); + if combined.is_empty() { + return None; + } + let mut merged = queued.last()?.params.clone(); + merged["text"] = Value::String(combined.join("\n\n")); + if let Some(object) = merged.as_object_mut() { + object.remove("message"); + } + Some(merged) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReplayDisposition { + Started, + Queued, + Terminal, +} + +fn replay_disposition(result: &moltis_service_traits::ServiceResult) -> ReplayDisposition { + let Ok(payload) = result else { + return ReplayDisposition::Terminal; + }; + if payload.get("queued").and_then(Value::as_bool) == Some(true) { + ReplayDisposition::Queued + } else if payload.get("runId").and_then(Value::as_str).is_some() { + ReplayDisposition::Started + } else { + ReplayDisposition::Terminal } - blocks } -/// Resolve acknowledgments for a replay that never became a run. -/// -/// A replay can fail outright, or return a terminal payload (a rejected hook, -/// an error) with no run behind it. Either way nothing else will finalize the -/// reaction, so it is settled here. A replay that queues again keeps its -/// acknowledgment: it will be claimed when it finally runs. async fn settle_replay( state: &Arc, session_key: &str, keys: Vec, result: &moltis_service_traits::ServiceResult, ) { - let failed = match result { - Err(e) => { - warn!(session = %session_key, error = %e, "failed to replay queued message"); - true - }, - Ok(payload) => { - if payload.get("queued").and_then(Value::as_bool) == Some(true) { - false - } else { - payload.get("rejected").and_then(Value::as_bool) == Some(true) - || matches!( - payload.get("state").and_then(Value::as_str), - Some("rejected" | "error" | "blocked") - ) - } - }, - }; - if failed && !keys.is_empty() { + if let Err(error) = result { + warn!(session = %session_key, error = %error, "failed to replay queued message"); + } + if replay_disposition(result) == ReplayDisposition::Terminal && !keys.is_empty() { state .finalize_channel_acks(keys, moltis_channels::ChannelAckOutcome::Failure) .await; } } -/// Resolve the acknowledgments of queued messages that will never be answered. async fn abandon(state: &Arc, queued: &[QueuedMessage]) { let keys = crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)); if !keys.is_empty() { @@ -200,13 +202,325 @@ async fn abandon(state: &Arc, queued: &[QueuedMessage]) { } } +fn merge_content_blocks(queued: &[QueuedMessage]) -> Vec { + let mut blocks = Vec::new(); + for message in queued { + match message.params.get("content").and_then(Value::as_array) { + Some(content) => blocks.extend(content.iter().cloned()), + None => { + if let Some(text) = message_text(&message.params) + && !text.is_empty() + { + blocks.push(serde_json::json!({ "type": "text", "text": text })); + } + }, + } + } + blocks +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { - use super::*; + use { + super::*, + async_trait::async_trait, + moltis_service_traits::{ChatService, ServiceResult}, + std::{collections::HashSet, sync::Mutex}, + tokio::sync::{OwnedSemaphorePermit, Semaphore}, + }; + + struct ReplayChat { + semaphore: Arc, + sent: Mutex>, + permits: Mutex>, + } + + #[async_trait] + impl ChatService for ReplayChat { + async fn send(&self, params: Value) -> ServiceResult { + let permit = Arc::clone(&self.semaphore) + .try_acquire_owned() + .map_err(|_| "session permit unavailable during replay")?; + self.sent + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(params); + self.permits + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push(permit); + Ok(serde_json::json!({"runId": "replayed-run"})) + } + + async fn abort(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn cancel_queued(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn history(&self, _params: Value) -> ServiceResult { + Ok(serde_json::json!([])) + } + + async fn inject(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } - fn msg(v: Value) -> QueuedMessage { - QueuedMessage { params: v } + async fn clear(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn compact(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn context(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn raw_prompt(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + + async fn full_context(&self, _params: Value) -> ServiceResult { + Ok(Value::Null) + } + } + + struct ReplayRuntime { + chat: Arc, + tts: moltis_service_traits::NoopTtsService, + project: moltis_service_traits::NoopProjectService, + mcp: moltis_service_traits::NoopMcpService, + } + + #[async_trait] + impl ChatRuntime for ReplayRuntime { + async fn broadcast(&self, _topic: &str, _payload: Value) {} + + async fn push_channel_reply( + &self, + _session_key: &str, + _target: moltis_channels::ChannelReplyTarget, + ) { + } + + async fn drain_channel_replies( + &self, + _session_key: &str, + ) -> Vec { + Vec::new() + } + + async fn peek_channel_replies( + &self, + _session_key: &str, + ) -> Vec { + Vec::new() + } + + async fn push_channel_status_log(&self, _session_key: &str, _message: String) {} + + async fn drain_channel_status_log(&self, _session_key: &str) -> Vec { + Vec::new() + } + + async fn set_run_error(&self, _run_id: &str, _error: String) {} + + async fn active_session_key(&self, _conn_id: &str) -> Option { + None + } + + async fn active_project_id(&self, _conn_id: &str) -> Option { + None + } + + fn hostname(&self) -> &str { + "test" + } + + fn sandbox_router(&self) -> Option<&Arc> { + None + } + + fn memory_manager(&self) -> Option<&moltis_memory::runtime::DynMemoryRuntime> { + None + } + + async fn cached_location(&self) -> Option { + None + } + + async fn tts_overrides( + &self, + _session_key: &str, + _channel_key: &str, + ) -> ( + Option, + Option, + ) { + (None, None) + } + + fn channel_outbound(&self) -> Option> { + None + } + + fn channel_stream_outbound( + &self, + ) -> Option> { + None + } + + fn tts_service(&self) -> &dyn moltis_service_traits::TtsService { + &self.tts + } + + fn project_service(&self) -> &dyn moltis_service_traits::ProjectService { + &self.project + } + + fn mcp_service(&self) -> &dyn moltis_service_traits::McpService { + &self.mcp + } + + async fn chat_service(&self) -> Arc { + self.chat.clone() + } + + async fn last_run_error(&self, _run_id: &str) -> Option { + None + } + + async fn send_push_notification( + &self, + _title: &str, + _body: &str, + _url: Option<&str>, + _session_key: Option<&str>, + ) -> crate::error::Result { + Ok(0) + } + + async fn ensure_local_model_cached(&self, _model_id: &str) -> crate::error::Result { + Ok(false) + } + + async fn connected_nodes(&self) -> Vec { + Vec::new() + } + } + + fn msg(value: Value) -> QueuedMessage { + QueuedMessage { params: value } + } + + #[test] + fn terminal_results_require_same_drainer_to_continue() { + assert_eq!( + replay_disposition(&Err("failed".into())), + ReplayDisposition::Terminal + ); + assert_eq!( + replay_disposition(&Ok(serde_json::json!({ "rejected": true }))), + ReplayDisposition::Terminal + ); + assert_eq!( + replay_disposition(&Ok(serde_json::json!({ "runId": "next" }))), + ReplayDisposition::Started + ); + } + + #[tokio::test] + async fn followup_take_preserves_fifo_after_terminal_replay() { + let queues = Arc::new(RwLock::new(HashMap::from([( + "s".to_string(), + SessionMessageQueue { + messages: [ + msg(serde_json::json!({"text": "one"})), + msg(serde_json::json!({"text": "two"})), + ] + .into_iter() + .collect(), + draining: false, + }, + )]))); + + let first = take_next(&queues, "s", MessageQueueMode::Followup).await; + let terminal_result = Err("no run spawned".into()); + let second = if replay_disposition(&terminal_result) == ReplayDisposition::Terminal { + take_next(&queues, "s", MessageQueueMode::Followup).await + } else { + Vec::new() + }; + assert_eq!(first[0].params["text"], "one"); + assert_eq!(second[0].params["text"], "two"); + } + + #[tokio::test] + async fn unavailable_abort_does_not_strand_replay_when_run_completes() { + let semaphore = Arc::new(Semaphore::new(1)); + let active_permit = Arc::clone(&semaphore).acquire_owned().await.unwrap(); + let task = tokio::spawn(std::future::pending::<()>()); + let active_runs = Arc::new(RwLock::new(HashMap::from([( + "run-1".to_string(), + task.abort_handle(), + )]))); + let by_session = Arc::new(RwLock::new(HashMap::from([( + "s".to_string(), + "run-1".to_string(), + )]))); + let terminal = Arc::new(RwLock::new(HashSet::from(["run-1".to_string()]))); + let (_, claim) = crate::service::LiveChatService::claim_run_for_abort( + &active_runs, + &by_session, + &terminal, + Arc::clone(&semaphore), + Some("run-1"), + Some("s"), + ) + .await; + assert_eq!( + claim.disposition, + crate::service::types::RunHandleDisposition::Unavailable + ); + assert!(claim.cleanup.is_none()); + + let queues = Arc::new(RwLock::new(HashMap::from([( + "s".to_string(), + SessionMessageQueue { + messages: [msg(serde_json::json!({"text": "queued"}))] + .into_iter() + .collect(), + draining: false, + }, + )]))); + let chat = Arc::new(ReplayChat { + semaphore, + sent: Mutex::new(Vec::new()), + permits: Mutex::new(Vec::new()), + }); + let runtime: Arc = Arc::new(ReplayRuntime { + chat: Arc::clone(&chat), + tts: moltis_service_traits::NoopTtsService, + project: moltis_service_traits::NoopProjectService, + mcp: moltis_service_traits::NoopMcpService, + }); + + // Completion releases the permit after abort has lost the terminal race, + // then follows the production queue-drain path. + drop(active_permit); + drain_and_replay(&queues, "s", MessageQueueMode::Followup, &runtime).await; + + assert_eq!( + chat.sent.lock().unwrap_or_else(|error| error.into_inner())[0]["text"], + "queued" + ); + assert!(!queues.read().await.contains_key("s")); + task.abort(); } #[test] @@ -220,7 +534,6 @@ mod tests { msg(serde_json::json!({ "text": "third" })), ]; let blocks = merge_content_blocks(&queued); - // Nothing is dropped: three texts plus the image. assert_eq!(blocks.len(), 4); assert_eq!(blocks[0]["text"], "first"); assert_eq!(blocks[1]["text"], "second"); @@ -229,13 +542,35 @@ mod tests { } #[test] - fn skips_empty_text_messages() { + fn collect_normalizes_text_and_message_alias_without_dropping_content() { let queued = vec![ - msg(serde_json::json!({ "text": "" })), - msg(serde_json::json!({ "text": "kept" })), + msg(serde_json::json!({ "text": "first", "source": 1 })), + msg(serde_json::json!({ "message": "second", "source": 2 })), + msg(serde_json::json!({ "text": "third", "message": "ignored alias", "source": 3 })), ]; + + let Some(merged) = merge_collected_text(&queued) else { + panic!("expected collected text"); + }; + assert_eq!(merged["text"], "first\n\nsecond\n\nthird"); + assert!(merged.get("message").is_none()); + assert_eq!(merged["source"], 3); + } + + #[test] + fn multimodal_collect_preserves_message_alias_as_text_block() { + let queued = vec![ + msg(serde_json::json!({ "message": "aliased" })), + msg(serde_json::json!({ "content": [ + { "type": "image_url", "image_url": { "url": "data:image/png;base64,AAA" } } + ]})), + ]; + let blocks = merge_content_blocks(&queued); - assert_eq!(blocks.len(), 1); - assert_eq!(blocks[0]["text"], "kept"); + assert_eq!( + blocks[0], + serde_json::json!({ "type": "text", "text": "aliased" }) + ); + assert_eq!(blocks[1]["type"], "image_url"); } } diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index 23cea3a3a0..23eaf109d7 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -30,7 +30,13 @@ use crate::{ types::*, }; -use {super::*, crate::service::build_persisted_assistant_message}; +use { + super::*, + crate::service::{ + build_persisted_assistant_message, + types::{TurnAdmission, commit_successful_turn, commit_terminal_run}, + }, +}; use super::queue_drain; @@ -40,6 +46,28 @@ use { }; impl LiveChatService { + async fn finish_unstarted_turn( + &self, + activity_id: &str, + session_key: &str, + permit: OwnedSemaphorePermit, + queued_replay: bool, + ) { + self.state + .finalize_active_channel_acks(activity_id, moltis_channels::ChannelAckOutcome::Failure) + .await; + drop(permit); + if !queued_replay { + queue_drain::drain_and_replay( + &self.message_queue, + session_key, + self.config.chat.message_queue_mode, + &self.state, + ) + .await; + } + } + #[tracing::instrument(skip(self, params), fields(session_id))] pub(super) async fn send_impl(&self, mut params: Value) -> ServiceResult { // Support both text-only and multimodal content. @@ -129,6 +157,8 @@ impl LiveChatService { // Carried through queueing/replay so reactions follow the message. let ack_keys = crate::channel_acks::ack_keys_from_params(¶ms); + // This identity owns all activity and cleanup for the admitted turn. + let run_id = uuid::Uuid::new_v4().to_string(); // Track client-side sequence number for ordering diagnostics. // Note: seq resets to 1 on page reload, so a drop from a high value @@ -192,9 +222,11 @@ impl LiveChatService { // session, queue immediately instead of letting a follow-up request // contend with the active run's locks. let message_queue_mode = self.config.chat.message_queue_mode; - let session_sem = self.session_semaphore(&session_key).await; - let permit: OwnedSemaphorePermit = match session_sem.clone().try_acquire_owned() { - Ok(p) => { + let permit: OwnedSemaphorePermit = match self + .admit_turn(&session_key, params.clone(), queued_replay) + .await + { + TurnAdmission::Acquired(p) => { info!( session = %session_key, client_seq = ?client_seq, @@ -203,20 +235,12 @@ impl LiveChatService { ); // This call owns the session and will execute: claim its acks. self.state - .activate_channel_acks(&session_key, ack_keys.clone()) + .activate_channel_acks(&run_id, &session_key, ack_keys.clone()) .await; p }, - Err(_) => { + TurnAdmission::Queued(position) => { let queue_mode = message_queue_mode; - let position = { - let mut q = self.message_queue.write().await; - let entry = q.entry(session_key.clone()).or_default(); - entry.push(QueuedMessage { - params: params.clone(), - }); - entry.len() - }; info!( session = %session_key, mode = ?queue_mode, @@ -252,7 +276,6 @@ impl LiveChatService { if let Some(shell_command) = explicit_shell_command { // Generate run_id early so we can link the user message to this run. - let run_id = uuid::Uuid::new_v4().to_string(); let run_id_clone = run_id.clone(); let channel_meta = params.get("channel").cloned(); let user_audio = user_audio_path_from_params(¶ms, &session_key); @@ -392,7 +415,11 @@ impl LiveChatService { .map(String::from); let conn_id_for_tool = conn_id.clone(); + let (start_run, run_registered) = tokio::sync::oneshot::channel(); let handle = tokio::spawn(async move { + if run_registered.await.is_err() { + return; + } let permit = permit; // hold permit until command run completes if let Some(target) = deferred_channel_target { state.push_channel_reply(&session_key_clone, target).await; @@ -405,9 +432,9 @@ impl LiveChatService { let assistant_output = run_explicit_shell_command( &state, &run_id_clone, + &terminal_runs, &tool_registry, &session_store, - &terminal_runs, &session_key_clone, &shell_command, user_message_index, @@ -417,6 +444,8 @@ impl LiveChatService { ) .await; + let mut assistant_output = assistant_output; + let final_payload = assistant_output.final_broadcast.take(); let assistant_msg = build_persisted_assistant_message( assistant_output, None, @@ -424,19 +453,30 @@ impl LiveChatService { client_seq, Some(run_id_clone.clone()), ); - if let Err(e) = session_store - .append(&session_key_clone, &assistant_msg.to_value()) - .await - { - warn!("failed to persist /sh assistant message: {e}"); - } - if let Ok(count) = session_store.count(&session_key_clone).await { - session_metadata.touch(&session_key_clone, count).await; - } + commit_successful_turn( + &terminal_runs, + &run_id_clone, + async { + if let Err(e) = session_store + .append(&session_key_clone, &assistant_msg.to_value()) + .await + { + warn!("failed to persist /sh assistant message: {e}"); + } + if let Ok(count) = session_store.count(&session_key_clone).await { + session_metadata.touch(&session_key_clone, count).await; + } - // Explicit /sh runs finalize their own acknowledgment: they - // never reach the model path that emits the terminal below. - crate::channel_acks::note_turn_finished(&state, &session_key_clone, true).await; + // Explicit /sh runs never reach the model completion path. + crate::channel_acks::note_turn_finished(&state, &run_id_clone, true).await; + }, + async { + if let Some(payload) = final_payload { + broadcast(&state, "chat", payload, BroadcastOpts::default()).await; + } + }, + ) + .await; active_runs.write().await.remove(&run_id_clone); let mut runs_by_session = active_runs_by_session.write().await; @@ -468,14 +508,15 @@ impl LiveChatService { .await; }); - self.active_runs - .write() - .await - .insert(run_id.clone(), handle.abort_handle()); - self.active_runs_by_session - .write() - .await - .insert(session_key.clone(), run_id.clone()); + Self::register_run_handle( + &self.active_runs, + &self.active_runs_by_session, + &run_id, + &session_key, + handle.abort_handle(), + ) + .await; + let _ = start_run.send(()); info!( run_id = %run_id, @@ -498,47 +539,57 @@ impl LiveChatService { }; let model_id = explicit_model.or(session_model.as_deref()); - let provider: Arc = { + let provider_result: Result, String> = { let reg = self.providers.read().await; - let primary = if let Some(id) = model_id { + let primary_result = if let Some(id) = model_id { reg.get(id).ok_or_else(|| { let available: Vec<_> = reg.list_models().iter().map(|m| m.id.clone()).collect(); format!("model '{}' not found. available: {:?}", id, available) - })? + }) } else if !stream_only { reg.first_with_tools() - .ok_or_else(|| "no LLM providers configured".to_string())? + .ok_or_else(|| "no LLM providers configured".to_string()) } else { reg.first() - .ok_or_else(|| "no LLM providers configured".to_string())? + .ok_or_else(|| "no LLM providers configured".to_string()) }; - // When exact_model is set and the user explicitly selected a model, - // skip failover β€” use the chosen model or fail. - let user_selected = model_id.is_some(); - let skip_failover = !self.failover_config.enabled - || (self.failover_config.exact_model && user_selected); - - if skip_failover { - primary - } else { - let fallbacks = if self.failover_config.fallback_models.is_empty() { - // Auto-build: same model on other providers first, then same - // provider's other models, then everything else. - reg.fallback_providers_for(primary.id(), primary.name()) - } else { - reg.providers_for_models(&self.failover_config.fallback_models) - }; - if fallbacks.is_empty() { - primary - } else { - let mut chain = vec![primary]; - chain.extend(fallbacks); - Arc::new(moltis_agents::provider_chain::ProviderChain::new(chain)) - } + match primary_result { + Err(error) => Err(error), + Ok(primary) => { + let user_selected = model_id.is_some(); + let skip_failover = !self.failover_config.enabled + || (self.failover_config.exact_model && user_selected); + if skip_failover { + Ok(primary) + } else { + let fallbacks = if self.failover_config.fallback_models.is_empty() { + reg.fallback_providers_for(primary.id(), primary.name()) + } else { + reg.providers_for_models(&self.failover_config.fallback_models) + }; + if fallbacks.is_empty() { + Ok(primary) + } else { + let mut chain = vec![primary]; + chain.extend(fallbacks); + Ok(Arc::new(moltis_agents::provider_chain::ProviderChain::new( + chain, + ))) + } + } + }, } }; + let provider = match provider_result { + Ok(provider) => provider, + Err(error) => { + self.finish_unstarted_turn(&run_id, &session_key, permit, queued_replay) + .await; + return Err(error.into()); + }, + }; info!( session = %session_key, provider = provider.name(), @@ -562,6 +613,8 @@ impl LiveChatService { "checking local model cache" ); if let Err(e) = self.state.ensure_local_model_cached(&model_to_check).await { + self.finish_unstarted_turn(&run_id, &session_key, permit, queued_replay) + .await; return Err(format!("Failed to prepare local model: {}", e).into()); } // Pre-load the model into RAM (broadcasts lifecycle events so the @@ -576,9 +629,6 @@ impl LiveChatService { .resolve_turn_context(&session_key, conn_id.as_deref()) .await; - // Generate run_id early so we can link the user message to its agent run. - let run_id = uuid::Uuid::new_v4().to_string(); - // Load conversation history (the current user message is NOT yet // persisted β€” run_streaming / run_agent_loop add it themselves). let mut history = self @@ -750,6 +800,9 @@ impl LiveChatService { ) .await; + self.finish_unstarted_turn(&run_id, &session_key, permit, queued_replay) + .await; + return Ok(serde_json::json!({ "ok": false, "rejected": true, @@ -1096,7 +1149,11 @@ impl LiveChatService { let terminal_runs = Arc::clone(&self.terminal_runs); let deferred_channel_target = deferred_channel_target.clone(); + let (start_run, run_registered) = tokio::sync::oneshot::channel(); let handle = tokio::spawn(async move { + if run_registered.await.is_err() { + return; + } let permit = permit; // hold permit until agent run completes let ctx_ref = project_context.as_deref(); if let Some(target) = deferred_channel_target { @@ -1197,11 +1254,17 @@ impl LiveChatService { }; let assistant_text = if outer_agent_timeout_secs > 0 { - match tokio::time::timeout(Duration::from_secs(outer_agent_timeout_secs), agent_fut) - .await - { - Ok(result) => result, - Err(_) => { + tokio::pin!(agent_fut); + let deadline = tokio::time::sleep(Duration::from_secs(outer_agent_timeout_secs)); + tokio::pin!(deadline); + tokio::select! { + result = &mut agent_fut => result, + () = &mut deadline => { + // A committed run is only completing bounded final I/O; + // do not replace an accepted channel final with timeout. + if terminal_runs.read().await.contains(&run_id_clone) { + agent_fut.await + } else { warn!( run_id = %run_id_clone, session = %session_key_clone, @@ -1216,8 +1279,8 @@ impl LiveChatService { "detail": detail, }); state.set_run_error(&run_id_clone, detail.clone()).await; + commit_terminal_run(&terminal_runs, &run_id_clone).await; deliver_channel_error(&state, &session_key_clone, &error_obj).await; - terminal_runs.write().await.insert(run_id_clone.clone()); broadcast( &state, "chat", @@ -1231,23 +1294,18 @@ impl LiveChatService { ) .await; None - }, + } + } } } else { agent_fut.await }; - // Finalize channel ack reactions (aborted runs emit Cancelled on the - // abort path and never reach here). - crate::channel_acks::note_turn_finished( - &state, - &session_key_clone, - assistant_text.is_some(), - ) - .await; - - // Persist assistant response (even empty ones β€” needed for LLM history coherence). - if let Some(assistant_output) = assistant_text { + // Channel delivery is complete when a successful output returns. + // Claim terminal ownership before persistence so abort cannot turn + // a committed assistant message into an aborted run. + if let Some(mut assistant_output) = assistant_text { + let final_payload = assistant_output.final_broadcast.take(); let assistant_msg = build_persisted_assistant_message( assistant_output, Some(model_id.clone()), @@ -1255,12 +1313,26 @@ impl LiveChatService { client_seq, Some(run_id_clone.clone()), ); - if let Err(e) = session_store - .append(&session_key_clone, &assistant_msg.to_value()) - .await - { - warn!("failed to persist assistant message: {e}"); - } + commit_successful_turn( + &terminal_runs, + &run_id_clone, + async { + if let Err(e) = session_store + .append(&session_key_clone, &assistant_msg.to_value()) + .await + { + warn!("failed to persist assistant message: {e}"); + } + crate::channel_acks::note_turn_finished(&state, &run_id_clone, true).await; + }, + async { + if let Some(payload) = final_payload { + broadcast(&state, "chat", payload, BroadcastOpts::default()).await; + } + }, + ) + .await; + // Update metadata counts. if let Ok(count) = session_store.count(&session_key_clone).await { session_metadata.touch(&session_key_clone, count).await; @@ -1334,6 +1406,8 @@ impl LiveChatService { } } } + } else { + crate::channel_acks::note_turn_finished(&state, &run_id_clone, false).await; } // ── Auto-title generation ────────────────────────────── @@ -1349,11 +1423,9 @@ impl LiveChatService { state.trigger_auto_title(&session_key_clone).await; } - let _ = LiveChatService::wait_for_event_forwarder( - &active_event_forwarders, - &session_key_clone, - ) - .await; + let _ = + LiveChatService::wait_for_event_forwarder(&active_event_forwarders, &run_id_clone) + .await; active_runs.write().await.remove(&run_id_clone); let mut runs_by_session = active_runs_by_session.write().await; @@ -1388,14 +1460,15 @@ impl LiveChatService { .await; }); - self.active_runs - .write() - .await - .insert(run_id.clone(), handle.abort_handle()); - self.active_runs_by_session - .write() - .await - .insert(session_key.clone(), run_id.clone()); + Self::register_run_handle( + &self.active_runs, + &self.active_runs_by_session, + &run_id, + &session_key, + handle.abort_handle(), + ) + .await; + let _ = start_run.send(()); info!( run_id = %run_id, diff --git a/crates/chat/src/service/mod.rs b/crates/chat/src/service/mod.rs index eaaa23e9dc..089f456b7a 100644 --- a/crates/chat/src/service/mod.rs +++ b/crates/chat/src/service/mod.rs @@ -5,7 +5,7 @@ mod types; use types::QueuedMessage; pub(crate) use types::{ - ActiveAssistantDraft, build_persisted_assistant_message, build_tool_call_assistant_message, - persist_tool_history_pair, + ActiveAssistantDraft, EventForwarder, build_persisted_assistant_message, + build_tool_call_assistant_message, commit_terminal_run, persist_tool_history_pair, }; pub use types::{ActiveToolCall, LiveChatService}; diff --git a/crates/chat/src/service/types.rs b/crates/chat/src/service/types.rs index c9431ffa17..00b980e8f2 100644 --- a/crates/chat/src/service/types.rs +++ b/crates/chat/src/service/types.rs @@ -1,17 +1,25 @@ //! `LiveChatService` struct, constructors, and helper methods. use std::{ - collections::{HashMap, HashSet}, + collections::{HashMap, HashSet, VecDeque}, + future::Future, path::{Path, PathBuf}, + pin::Pin, sync::Arc, + task::Poll, + time::Duration, }; use { + futures::{ + FutureExt, + future::{BoxFuture, Shared}, + }, serde::Serialize, serde_json::Value, tokio::{ - sync::{RwLock, Semaphore}, - task::AbortHandle, + sync::{AcquireError, OwnedSemaphorePermit, RwLock, Semaphore}, + task::{AbortHandle, JoinHandle}, }, tracing::warn, }; @@ -36,6 +44,107 @@ pub(in crate::service) struct QueuedMessage { pub(in crate::service) params: Value, } +/// Per-session FIFO plus an in-progress replay reservation. +#[derive(Debug, Default)] +pub(in crate::service) struct SessionMessageQueue { + pub(in crate::service) messages: VecDeque, + pub(in crate::service) draining: bool, +} + +pub(in crate::service) enum TurnAdmission { + Acquired(OwnedSemaphorePermit), + Queued(usize), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::service) enum RunHandleDisposition { + Aborted, + Stale, + Unavailable, +} + +type CleanupPermitFuture = + Pin> + Send>>; + +pub(in crate::service) enum CleanupPermitReservation { + Acquired(Result), + Waiting(CleanupPermitFuture), +} + +impl CleanupPermitReservation { + pub(in crate::service) async fn acquire(self) -> Result { + match self { + Self::Acquired(result) => result, + Self::Waiting(future) => future.await, + } + } +} + +pub(in crate::service) struct AbortRunClaim { + pub(in crate::service) disposition: RunHandleDisposition, + pub(in crate::service) cleanup: Option, +} + +pub(crate) async fn commit_terminal_run( + terminal_runs: &Arc>>, + run_id: &str, +) { + terminal_runs.write().await.insert(run_id.to_string()); +} + +pub(in crate::service) async fn commit_successful_turn( + terminal_runs: &Arc>>, + run_id: &str, + persistence: P, + final_broadcast: B, +) where + P: Future, + B: Future, +{ + commit_terminal_run(terminal_runs, run_id).await; + persistence.await; + final_broadcast.await; +} + +#[derive(Clone)] +pub(crate) struct EventForwarder { + completion: Shared>, + abort_handle: AbortHandle, +} + +impl EventForwarder { + pub(crate) fn new(task: JoinHandle, session_key: String) -> Self { + let abort_handle = task.abort_handle(); + let completion = async move { + match task.await { + Ok(reasoning) => reasoning, + Err(error) => { + warn!( + session = %session_key, + error = %error, + "runner event forwarder task failed" + ); + String::new() + }, + } + } + .boxed() + .shared(); + Self { + completion, + abort_handle, + } + } + + async fn wait(&self) -> String { + self.completion.clone().await + } + + fn abort(&self) { + self.abort_handle.abort(); + } +} + /// A tool call currently executing within an active agent run. #[derive(Debug, Clone, Serialize)] pub struct ActiveToolCall { @@ -228,8 +337,7 @@ pub struct LiveChatService { pub(in crate::service) state: Arc, pub(in crate::service) active_runs: Arc>>, pub(in crate::service) active_runs_by_session: Arc>>, - pub(in crate::service) active_event_forwarders: - Arc>>>, + pub(in crate::service) active_event_forwarders: Arc>>, pub(in crate::service) terminal_runs: Arc>>, pub(in crate::service) tool_registry: Arc>, pub(in crate::service) session_store: Arc, @@ -239,7 +347,7 @@ pub struct LiveChatService { /// Per-session semaphore ensuring only one agent run executes per session at a time. pub(in crate::service) session_locks: Arc>>>, /// Per-session message queue for messages arriving during an active run. - pub(in crate::service) message_queue: Arc>>>, + pub(in crate::service) message_queue: Arc>>, /// Per-session last-seen client sequence number for ordering diagnostics. pub(in crate::service) last_client_seq: Arc>>, /// Per-session accumulated thinking text for active runs, so it can be @@ -359,45 +467,100 @@ impl LiveChatService { ) } - pub(in crate::service) async fn abort_run_handle( + /// Arbitrate the permit and FIFO under one queue lock. Ordinary arrivals + /// cannot pass queued work, while the drainer's replay can consume its + /// reservation. If that replay cannot acquire, it returns to the front. + pub(in crate::service) async fn admit_turn( + &self, + session_key: &str, + params: Value, + queued_replay: bool, + ) -> TurnAdmission { + let session_sem = self.session_semaphore(session_key).await; + admit_queued_turn( + &self.message_queue, + session_sem, + session_key, + params, + queued_replay, + ) + .await + } + + pub(in crate::service) async fn claim_run_for_abort( active_runs: &Arc>>, active_runs_by_session: &Arc>>, terminal_runs: &Arc>>, + session_sem: Arc, run_id: Option<&str>, session_key: Option<&str>, - ) -> (Option, bool) { - let resolved_run_id = if let Some(id) = run_id { - Some(id.to_string()) - } else if let Some(key) = session_key { - active_runs_by_session.read().await.get(key).cloned() - } else { - None + ) -> (Option, AbortRunClaim) { + let terminal = terminal_runs.read().await; + let mut by_session = active_runs_by_session.write().await; + let target = match (run_id, session_key) { + (Some(id), Some(key)) if by_session.get(key).is_some_and(|active| active == id) => { + Some((id.to_string(), key.to_string())) + }, + (Some(_), Some(_)) => None, + (Some(id), None) => by_session + .iter() + .find_map(|(key, active)| (active == id).then(|| (id.to_string(), key.clone()))), + (None, Some(key)) => by_session.get(key).cloned().map(|id| (id, key.to_string())), + (None, None) => None, }; - - let Some(target_run_id) = resolved_run_id.clone() else { - return (None, false); + let Some((target_run_id, target_session_key)) = target else { + return (None, AbortRunClaim { + disposition: RunHandleDisposition::Unavailable, + cleanup: None, + }); }; - if terminal_runs.read().await.contains(&target_run_id) { - return (resolved_run_id, false); + let mut runs = active_runs.write().await; + if terminal.contains(&target_run_id) { + return (Some(target_run_id), AbortRunClaim { + disposition: RunHandleDisposition::Unavailable, + cleanup: None, + }); } - let aborted = if let Some(handle) = active_runs.write().await.remove(&target_run_id) { - handle.abort(); - true - } else { - false + let disposition = match runs.get(&target_run_id) { + Some(handle) if handle.is_finished() => RunHandleDisposition::Stale, + None => RunHandleDisposition::Stale, + Some(_) => RunHandleDisposition::Aborted, }; - let mut by_session = active_runs_by_session.write().await; - if let Some(key) = session_key - && by_session.get(key).is_some_and(|id| id == &target_run_id) + // Establish ownership before registering the fair waiter. Holding the + // terminal and run-map locks makes abort versus successful terminal + // transition a single-winner race; an unavailable request never + // reserves the session permit. + let mut cleanup_future: CleanupPermitFuture = Box::pin(session_sem.acquire_owned()); + let mut immediate = None; + std::future::poll_fn(|cx| { + if let Poll::Ready(result) = cleanup_future.as_mut().poll(cx) { + immediate = Some(result); + } + Poll::Ready(()) + }) + .await; + let cleanup = match immediate { + Some(result) => CleanupPermitReservation::Acquired(result), + None => CleanupPermitReservation::Waiting(cleanup_future), + }; + + if disposition == RunHandleDisposition::Aborted + && let Some(handle) = runs.get(&target_run_id) { - by_session.remove(key); + handle.abort(); + } + runs.remove(&target_run_id); + if by_session.get(&target_session_key) == Some(&target_run_id) { + by_session.remove(&target_session_key); } - by_session.retain(|_, id| id != &target_run_id); - (resolved_run_id, aborted) + (Some(target_run_id), AbortRunClaim { + disposition, + cleanup: Some(cleanup), + }) } pub(in crate::service) async fn resolve_session_key_for_run( @@ -405,48 +568,81 @@ impl LiveChatService { run_id: Option<&str>, session_key: Option<&str>, ) -> Option { - if let Some(key) = session_key { - return Some(key.to_string()); + let by_session = active_runs_by_session.read().await; + match (run_id, session_key) { + (Some(id), Some(key)) => by_session + .get(key) + .is_some_and(|active| active == id) + .then(|| key.to_string()), + (Some(id), None) => by_session + .iter() + .find_map(|(key, active)| (active == id).then(|| key.clone())), + (None, Some(key)) => by_session.contains_key(key).then(|| key.to_string()), + (None, None) => None, } - let target_run_id = run_id?; - active_runs_by_session - .read() - .await - .iter() - .find_map(|(key, active_run_id)| (active_run_id == target_run_id).then(|| key.clone())) } pub(crate) async fn wait_for_event_forwarder( - active_event_forwarders: &Arc>>>, - session_key: &str, + active_event_forwarders: &Arc>>, + run_id: &str, ) -> String { - let handle = active_event_forwarders.write().await.remove(session_key); - let Some(handle) = handle else { + let completion = active_event_forwarders.read().await.get(run_id).cloned(); + let Some(completion) = completion else { return String::new(); }; - match handle.await { - Ok(reasoning) => reasoning, - Err(e) => { - warn!( - session = %session_key, - error = %e, - "runner event forwarder task failed" - ); - String::new() - }, + let reasoning = completion.wait().await; + active_event_forwarders.write().await.remove(run_id); + reasoning + } + + pub(in crate::service) async fn cancel_event_forwarder( + active_event_forwarders: &Arc>>, + run_id: &str, + ) { + let forwarder = active_event_forwarders.write().await.remove(run_id); + let Some(forwarder) = forwarder else { + return; + }; + forwarder.abort(); + if tokio::time::timeout(Duration::from_secs(1), forwarder.wait()) + .await + .is_err() + { + warn!(run_id, "timed out waiting for cancelled event forwarder"); } } + pub(in crate::service) async fn register_run_handle( + active_runs: &Arc>>, + active_runs_by_session: &Arc>>, + run_id: &str, + session_key: &str, + abort_handle: AbortHandle, + ) { + // Keep the same lock order as claim_run_for_abort so registration is atomic + // from an abort caller's perspective. + let mut by_session = active_runs_by_session.write().await; + let mut runs = active_runs.write().await; + by_session.insert(session_key.to_string(), run_id.to_string()); + runs.insert(run_id.to_string(), abort_handle); + } + pub(in crate::service) async fn persist_partial_assistant_on_abort( &self, session_key: &str, + run_id: &str, ) -> Option<(Value, Option)> { - let partial = self - .active_partial_assistant - .write() - .await - .remove(session_key)?; + let partial = { + let mut partials = self.active_partial_assistant.write().await; + if !partials + .get(session_key) + .is_some_and(|partial| partial.run_id == run_id) + { + return None; + } + partials.remove(session_key)? + }; if !partial.has_visible_content() { return None; } @@ -628,6 +824,50 @@ impl LiveChatService { } } +async fn admit_queued_turn( + queues: &Arc>>, + session_sem: Arc, + session_key: &str, + params: Value, + queued_replay: bool, +) -> TurnAdmission { + let mut queues = queues.write().await; + let queue = queues.entry(session_key.to_string()).or_default(); + let reserved_replay = queued_replay && queue.draining; + + if !reserved_replay && (queue.draining || !queue.messages.is_empty()) { + queue.messages.push_back(QueuedMessage { params }); + return TurnAdmission::Queued(queue.messages.len()); + } + + match session_sem.try_acquire_owned() { + Ok(permit) => TurnAdmission::Acquired(permit), + Err(_) => { + let message = QueuedMessage { params }; + if reserved_replay { + queue.messages.push_front(message); + } else { + queue.messages.push_back(message); + } + TurnAdmission::Queued(queue.messages.len()) + }, + } +} + +pub(in crate::service) fn cancel_queued_messages( + queues: &mut HashMap, + session_key: &str, +) -> Vec { + match queues.get_mut(session_key) { + Some(queue) if queue.draining => queue.messages.drain(..).collect(), + Some(_) => queues + .remove(session_key) + .map(|queue| queue.messages.into_iter().collect()) + .unwrap_or_default(), + None => Vec::new(), + } +} + pub(in crate::service) fn merge_context_sections( project_context: Option, command_context: Option, @@ -645,11 +885,21 @@ pub(in crate::service) fn merge_context_sections( mod tests { use { super::{ - ActiveAssistantDraft, build_persisted_assistant_message, - build_tool_call_assistant_message, merge_context_sections, + ActiveAssistantDraft, EventForwarder, LiveChatService, QueuedMessage, + RunHandleDisposition, SessionMessageQueue, TurnAdmission, admit_queued_turn, + build_persisted_assistant_message, build_tool_call_assistant_message, + cancel_queued_messages, commit_successful_turn, merge_context_sections, }, crate::types::AssistantTurnOutput, moltis_sessions::PersistedMessage, + serde_json::json, + std::{ + collections::{HashMap, HashSet}, + future::Future, + sync::Arc, + task::Poll, + }, + tokio::sync::{RwLock, Semaphore}, }; #[test] @@ -701,6 +951,352 @@ mod tests { assert_eq!(merge_context_sections(None, None), None); } + #[tokio::test] + async fn replay_reservation_prevents_new_arrival_overtaking_fifo() { + let queues = Arc::new(RwLock::new(HashMap::from([( + "s".to_string(), + SessionMessageQueue { + messages: [QueuedMessage { + params: json!({"text": "second"}), + }] + .into_iter() + .collect(), + draining: true, + }, + )]))); + let semaphore = Arc::new(Semaphore::new(1)); + let replay = admit_queued_turn( + &queues, + Arc::clone(&semaphore), + "s", + json!({"text": "first"}), + true, + ) + .await; + let TurnAdmission::Acquired(permit) = replay else { + panic!("reserved replay should acquire"); + }; + + let arrival = + admit_queued_turn(&queues, semaphore, "s", json!({"text": "third"}), false).await; + assert!(matches!(arrival, TurnAdmission::Queued(2))); + let queue = queues.read().await; + let texts: Vec<_> = queue["s"] + .messages + .iter() + .filter_map(|message| message.params["text"].as_str()) + .collect(); + assert_eq!(texts, ["second", "third"]); + drop(permit); + } + + #[tokio::test] + async fn abort_mismatch_preserves_active_run_state() { + let task = tokio::spawn(std::future::pending::<()>()); + let active_runs = Arc::new(RwLock::new(HashMap::from([( + "run-1".to_string(), + task.abort_handle(), + )]))); + let by_session = Arc::new(RwLock::new(HashMap::from([( + "session-1".to_string(), + "run-1".to_string(), + )]))); + let terminal = Arc::new(RwLock::new(HashSet::new())); + + let semaphore = Arc::new(Semaphore::new(1)); + let (resolved, claim) = LiveChatService::claim_run_for_abort( + &active_runs, + &by_session, + &terminal, + Arc::clone(&semaphore), + Some("run-2"), + Some("session-1"), + ) + .await; + assert_eq!(resolved, None); + assert_eq!(claim.disposition, RunHandleDisposition::Unavailable); + assert!(claim.cleanup.is_none()); + assert!(active_runs.read().await.contains_key("run-1")); + assert_eq!( + by_session.read().await.get("session-1").map(String::as_str), + Some("run-1") + ); + + let (_, claim) = LiveChatService::claim_run_for_abort( + &active_runs, + &by_session, + &terminal, + semaphore, + Some("run-1"), + Some("session-1"), + ) + .await; + assert_eq!(claim.disposition, RunHandleDisposition::Aborted); + drop(claim.cleanup.unwrap().acquire().await.unwrap()); + assert!(active_runs.read().await.is_empty()); + assert!(by_session.read().await.is_empty()); + assert!(task.await.is_err()); + } + + #[tokio::test] + async fn panicked_terminal_run_handle_is_removed_as_stale() { + let task = tokio::spawn(async { panic!("simulated run panic") }); + let handle = task.abort_handle(); + assert!(task.await.is_err()); + let active_runs = Arc::new(RwLock::new(HashMap::from([("run-1".to_string(), handle)]))); + let by_session = Arc::new(RwLock::new(HashMap::from([( + "session-1".to_string(), + "run-1".to_string(), + )]))); + let terminal = Arc::new(RwLock::new(HashSet::new())); + + let (resolved, claim) = LiveChatService::claim_run_for_abort( + &active_runs, + &by_session, + &terminal, + Arc::new(Semaphore::new(1)), + Some("run-1"), + Some("session-1"), + ) + .await; + + assert_eq!(resolved.as_deref(), Some("run-1")); + assert_eq!(claim.disposition, RunHandleDisposition::Stale); + drop(claim.cleanup.unwrap().acquire().await.unwrap()); + assert!(active_runs.read().await.is_empty()); + assert!(by_session.read().await.is_empty()); + } + + #[tokio::test] + async fn shared_success_commit_transitions_then_persists_before_final() { + let terminal = Arc::new(RwLock::new(HashSet::new())); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let persist_events = Arc::clone(&events); + let final_events = Arc::clone(&events); + let terminal_at_persistence = Arc::clone(&terminal); + let terminal_at_broadcast = Arc::clone(&terminal); + + commit_successful_turn( + &terminal, + "run-1", + async move { + assert!(terminal_at_persistence.read().await.contains("run-1")); + persist_events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("persist"); + }, + async move { + assert!(terminal_at_broadcast.read().await.contains("run-1")); + final_events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("final"); + }, + ) + .await; + + assert_eq!(*events.lock().unwrap_or_else(|error| error.into_inner()), [ + "persist", "final" + ]); + } + + #[tokio::test] + async fn abort_during_shared_delivery_prevents_persist_and_final() { + let semaphore = Arc::new(Semaphore::new(1)); + let permit = Arc::clone(&semaphore).acquire_owned().await.unwrap(); + let terminal = Arc::new(RwLock::new(HashSet::new())); + let active_runs = Arc::new(RwLock::new(HashMap::new())); + let by_session = Arc::new(RwLock::new(HashMap::new())); + let events = Arc::new(std::sync::Mutex::new(Vec::new())); + let task_events = Arc::clone(&events); + let task_terminal = Arc::clone(&terminal); + let (delivery_started, started) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _permit = permit; + let _ = delivery_started.send(()); + std::future::pending::<()>().await; + commit_successful_turn( + &task_terminal, + "run-1", + async { + task_events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("persist"); + }, + async { + task_events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .push("final"); + }, + ) + .await; + }); + LiveChatService::register_run_handle( + &active_runs, + &by_session, + "run-1", + "session-1", + task.abort_handle(), + ) + .await; + assert!(started.await.is_ok()); + + let (_, claim) = LiveChatService::claim_run_for_abort( + &active_runs, + &by_session, + &terminal, + semaphore, + Some("run-1"), + Some("session-1"), + ) + .await; + assert_eq!(claim.disposition, RunHandleDisposition::Aborted); + drop(claim.cleanup.unwrap().acquire().await.unwrap()); + assert!(task.await.is_err()); + assert!( + events + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty() + ); + assert!(terminal.read().await.is_empty()); + } + + #[tokio::test] + async fn concurrent_forwarder_waiters_share_run_scoped_completion() { + let (release, released) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _ = released.await; + "reasoning".to_string() + }); + let completion = EventForwarder::new(task, "session-1".to_string()); + let forwarders = Arc::new(RwLock::new(HashMap::from([( + "run-1".to_string(), + completion, + )]))); + let first = LiveChatService::wait_for_event_forwarder(&forwarders, "run-1"); + let second = LiveChatService::wait_for_event_forwarder(&forwarders, "run-1"); + tokio::pin!(first, second); + std::future::poll_fn(|cx| { + assert!(matches!(first.as_mut().poll(cx), Poll::Pending)); + assert!(matches!(second.as_mut().poll(cx), Poll::Pending)); + Poll::Ready(()) + }) + .await; + + release.send(()).unwrap(); + let (first_result, second_result) = tokio::join!(&mut first, &mut second); + assert_eq!(first_result, "reasoning"); + assert_eq!(second_result, "reasoning"); + assert!(forwarders.read().await.is_empty()); + } + + #[tokio::test] + async fn cancelling_forwarder_aborts_pending_delivery_work() { + struct DropNotice(Option>); + impl Drop for DropNotice { + fn drop(&mut self) { + if let Some(sender) = self.0.take() { + let _ = sender.send(()); + } + } + } + + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + let _notice = DropNotice(Some(dropped_tx)); + let _ = started_tx.send(()); + std::future::pending::().await + }); + let forwarders = Arc::new(RwLock::new(HashMap::from([( + "run-1".to_string(), + EventForwarder::new(task, "session-1".to_string()), + )]))); + assert!(started_rx.await.is_ok()); + + LiveChatService::cancel_event_forwarder(&forwarders, "run-1").await; + + assert!(forwarders.read().await.is_empty()); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), dropped_rx) + .await + .is_ok() + ); + } + + #[tokio::test] + async fn run_cannot_finish_before_abort_handle_registration() { + let active_runs = Arc::new(RwLock::new(HashMap::new())); + let by_session = Arc::new(RwLock::new(HashMap::new())); + let active_runs_for_task = Arc::clone(&active_runs); + let by_session_for_task = Arc::clone(&by_session); + let (start_run, run_registered) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + if run_registered.await.is_err() { + return; + } + active_runs_for_task.write().await.remove("run-1"); + by_session_for_task.write().await.remove("session-1"); + }); + LiveChatService::register_run_handle( + &active_runs, + &by_session, + "run-1", + "session-1", + task.abort_handle(), + ) + .await; + assert!(active_runs.read().await.contains_key("run-1")); + assert_eq!( + by_session.read().await.get("session-1").map(String::as_str), + Some("run-1") + ); + + assert!(start_run.send(()).is_ok()); + assert!(task.await.is_ok()); + assert!(active_runs.read().await.is_empty()); + assert!(by_session.read().await.is_empty()); + } + + #[tokio::test] + async fn registered_abort_cleanup_waiter_blocks_try_acquire_race() { + let semaphore = Arc::new(Semaphore::new(1)); + let active_permit = Arc::clone(&semaphore).acquire_owned().await.unwrap(); + let cleanup_waiter = Arc::clone(&semaphore).acquire_owned(); + tokio::pin!(cleanup_waiter); + std::future::poll_fn(|cx| { + assert!(matches!(cleanup_waiter.as_mut().poll(cx), Poll::Pending)); + Poll::Ready(()) + }) + .await; + + drop(active_permit); + assert!(Arc::clone(&semaphore).try_acquire_owned().is_err()); + let cleanup_permit = cleanup_waiter.await.unwrap(); + drop(cleanup_permit); + assert!(semaphore.try_acquire_owned().is_ok()); + } + + #[test] + fn cancellation_keeps_in_progress_replay_reserved() { + let mut queues = HashMap::from([("s".to_string(), SessionMessageQueue { + messages: [QueuedMessage { + params: json!({"text": "cancel"}), + }] + .into_iter() + .collect(), + draining: true, + })]); + let removed = cancel_queued_messages(&mut queues, "s"); + assert_eq!(removed.len(), 1); + assert!(queues["s"].draining); + assert!(queues["s"].messages.is_empty()); + } + #[test] fn tool_call_assistant_message_omits_cache_usage_fields() { let message = build_tool_call_assistant_message( @@ -748,6 +1344,7 @@ mod tests { audio_path: None, reasoning: Some("thinking".to_string()), llm_api_response: None, + final_broadcast: None, }, Some("gpt-4.1".to_string()), Some("openai".to_string()), diff --git a/crates/chat/src/streaming.rs b/crates/chat/src/streaming.rs index 631954706e..ca3dfd52c3 100644 --- a/crates/chat/src/streaming.rs +++ b/crates/chat/src/streaming.rs @@ -26,7 +26,10 @@ use { }; use crate::{ - agent_loop::{ChannelStreamDispatcher, clear_unsupported_model, mark_unsupported_model}, + agent_loop::{ + ChannelStreamDispatcher, clear_unsupported_model, + commit_terminal_and_finish_channel_stream, mark_unsupported_model, + }, channels::{ deliver_channel_error, deliver_channel_replies, generate_tts_audio, send_retry_status_to_channels, @@ -335,14 +338,6 @@ pub(crate) async fn run_streaming( let trimmed = accumulated_reasoning.trim(); (!trimmed.is_empty()).then(|| trimmed.to_string()) }; - let streamed_target_keys = - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - dispatcher.completed_target_keys().await - } else { - HashSet::new() - }; - info!( run_id, input_tokens = usage.input_tokens, @@ -363,17 +358,22 @@ pub(crate) async fn run_streaming( "The provider returned an empty response (possible network error). Please try again.", Some(provider_name), ); - deliver_channel_error(state, session_key, &error_obj).await; let error_payload = ChatErrorBroadcast { run_id: run_id.to_string(), session_key: session_key.to_string(), state: "error", - error: error_obj, + error: error_obj.clone(), seq: client_seq, }; #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); + commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + channel_stream_dispatcher.as_mut(), + ) + .await; + deliver_channel_error(state, session_key, &error_obj).await; broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; return None; } @@ -436,8 +436,15 @@ pub(crate) async fn run_streaming( ); #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&final_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); - broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; + + // Channel Done and fallback replies are irreversible. Claim + // terminal ownership before either can be accepted. + let streamed_target_keys = commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + channel_stream_dispatcher.as_mut(), + ) + .await; if !is_silent { // Send push notification when chat response completes @@ -448,6 +455,7 @@ pub(crate) async fn run_streaming( } deliver_channel_replies( state, + run_id, session_key, &accumulated, desired_reply_medium, @@ -457,14 +465,16 @@ pub(crate) async fn run_streaming( } let llm_api_response = (!raw_llm_responses.is_empty()).then_some(Value::Array(raw_llm_responses)); - return Some(build_assistant_turn_output( + let mut output = build_assistant_turn_output( accumulated, UsageSnapshot::new(usage.clone(), Some(usage)), run_started.elapsed().as_millis() as u64, audio_path, reasoning, llm_api_response, - )); + ); + output.final_broadcast = Some(payload_val); + return Some(output); }, StreamEvent::Error(msg) => { let error_obj = parse_chat_error(&msg, Some(provider_name)); @@ -518,23 +528,25 @@ pub(crate) async fn run_streaming( } warn!(run_id, error = %msg, "chat stream error"); - if let Some(dispatcher) = channel_stream_dispatcher.as_mut() { - dispatcher.finish().await; - } state.set_run_error(run_id, msg.clone()).await; mark_unsupported_model(state, model_store, model_id, provider_name, &error_obj) .await; - deliver_channel_error(state, session_key, &error_obj).await; let error_payload = ChatErrorBroadcast { run_id: run_id.to_string(), session_key: session_key.to_string(), state: "error", - error: error_obj, + error: error_obj.clone(), seq: client_seq, }; #[allow(clippy::unwrap_used)] // serializing known-valid struct let payload_val = serde_json::to_value(&error_payload).unwrap(); - terminal_runs.write().await.insert(run_id.to_string()); + commit_terminal_and_finish_channel_stream( + terminal_runs, + run_id, + channel_stream_dispatcher.as_mut(), + ) + .await; + deliver_channel_error(state, session_key, &error_obj).await; broadcast(state, "chat", payload_val, BroadcastOpts::default()).await; return None; }, diff --git a/crates/chat/src/types.rs b/crates/chat/src/types.rs index 38f163cf53..0e6e837472 100644 --- a/crates/chat/src/types.rs +++ b/crates/chat/src/types.rs @@ -171,6 +171,9 @@ pub(crate) struct AssistantTurnOutput { pub audio_path: Option, pub reasoning: Option, pub llm_api_response: Option, + /// Prepared after abortable channel delivery and emitted by the run owner + /// only after the assistant message has been persisted. + pub final_broadcast: Option, } #[allow(clippy::too_many_arguments)] @@ -244,6 +247,7 @@ pub(crate) fn build_assistant_turn_output( audio_path, reasoning, llm_api_response, + final_broadcast: None, } } diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index 0bed3d1dc3..b3df7f657e 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -19,7 +19,11 @@ //! //! Design ported from openclaw's `status-reactions.ts`. -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, + time::{Duration, Instant}, +}; use { moltis_channels::{ChannelAckOutcome, ChannelActivity, ChannelOutbound}, @@ -63,14 +67,20 @@ pub fn ack_key(account_id: &str, chat_id: &str, message_id: &str) -> String { /// message that is never claimed by a run cannot grow the map without bound. /// Controllers additionally self-expire via [`MAX_LIFETIME`]. const MAX_PENDING_ACKS: usize = 512; +/// Threshold for cleaning up turns that have exceeded [`MAX_LIFETIME`]. Healthy +/// concurrent turns may temporarily exceed it. +const MAX_ACTIVE_TURNS: usize = 512; + +struct ActiveAck { + key: String, + controller: Arc, +} /// The acknowledgments a currently-executing run owns. -/// -/// `id` distinguishes successive turns on the same session so a slow terminal -/// from turn N can never clear turn N+1's state. struct ActiveTurn { - id: u64, - keys: Vec, + session_key: String, + acknowledgments: Vec, + activated_at: Instant, } #[derive(Default)] @@ -78,9 +88,11 @@ struct RegistryInner { /// Received messages with a live reaction, not yet claimed by a run. pending: HashMap>, /// Insertion order of `pending`, for oldest-first eviction. - pending_order: Vec, - /// Which acknowledgments the running turn owns, per session. + pending_order: VecDeque, + /// Controllers owned by a running turn, keyed by the run's identity. active: HashMap, + /// Current activity identity per session, used to settle a displaced turn. + active_by_session: HashMap, } /// Routes acknowledgment-reaction activity to the right inbound message(s). @@ -92,10 +104,22 @@ struct RegistryInner { /// that merely happens to share the session. In `collect` queue mode a single /// run legitimately owns several inbound messages, and every one of them /// receives the phase and terminal reactions. -#[derive(Default)] pub struct ReactionRegistry { inner: Mutex, - next_turn_id: std::sync::atomic::AtomicU64, + pending_limit: usize, + active_limit: usize, + active_lifetime: Duration, +} + +impl Default for ReactionRegistry { + fn default() -> Self { + Self { + inner: Mutex::new(RegistryInner::default()), + pending_limit: MAX_PENDING_ACKS, + active_limit: MAX_ACTIVE_TURNS, + active_lifetime: MAX_LIFETIME, + } + } } impl ReactionRegistry { @@ -112,11 +136,12 @@ impl ReactionRegistry { if let Some(previous) = inner.pending.insert(key.clone(), controller) { displaced.push(previous); } else { - inner.pending_order.push(key.clone()); + inner.pending_order.push_back(key.clone()); } - // Evict oldest entries that no active turn still owns. - while inner.pending_order.len() > MAX_PENDING_ACKS { - let oldest = inner.pending_order.remove(0); + while inner.pending_order.len() > self.pending_limit { + let Some(oldest) = inner.pending_order.pop_front() else { + break; + }; if let Some(evicted) = inner.pending.remove(&oldest) { displaced.push(evicted); } @@ -130,87 +155,120 @@ impl ReactionRegistry { } } - /// Bind the given acknowledgments to this session's now-executing run. - /// - /// Any previously active turn for the session is dropped from routing (its - /// controllers keep their own lifetime cap), so a superseded run cannot - /// keep driving reactions. - pub async fn activate(&self, session_key: &str, keys: Vec) { + /// Transfer the given acknowledgments to this now-executing run. + pub async fn activate(&self, activity_id: &str, session_key: &str, keys: Vec) { if keys.is_empty() { return; } - let id = self - .next_turn_id - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let mut inner = self.inner.lock().await; - inner - .active - .insert(session_key.to_string(), ActiveTurn { id, keys }); + let displaced = { + let mut inner = self.inner.lock().await; + let acknowledgments = keys + .into_iter() + .filter_map(|key| { + inner.pending_order.retain(|pending| pending != &key); + inner + .pending + .remove(&key) + .map(|controller| ActiveAck { key, controller }) + }) + .collect::>(); + if acknowledgments.is_empty() { + return; + } + + let mut displaced = Vec::new(); + // This is a cleanup threshold rather than a concurrency cap: live + // workers may overlap, so only turns beyond their worker lifetime + // are eligible for eviction. + if inner.active.len() >= self.active_limit { + let now = Instant::now(); + let stale_ids = inner + .active + .iter() + .filter(|(_, turn)| { + now.duration_since(turn.activated_at) >= self.active_lifetime + }) + .map(|(activity_id, _)| activity_id.clone()) + .collect::>(); + for stale_id in stale_ids { + if let Some(stale) = remove_active(&mut inner, &stale_id) { + displaced.extend(stale.acknowledgments); + } + } + } + if let Some(previous_id) = inner + .active_by_session + .get(session_key) + .filter(|previous_id| previous_id.as_str() != activity_id) + .cloned() + && let Some(previous) = remove_active(&mut inner, &previous_id) + { + displaced.extend(previous.acknowledgments); + } + if let Some(previous) = remove_active(&mut inner, activity_id) { + displaced.extend(previous.acknowledgments); + } + + inner.active.insert(activity_id.to_string(), ActiveTurn { + session_key: session_key.to_string(), + acknowledgments, + activated_at: Instant::now(), + }); + inner + .active_by_session + .insert(session_key.to_string(), activity_id.to_string()); + displaced + }; + finish_acks(displaced, ChannelAckOutcome::Cancelled).await; } - /// Forward an activity to every acknowledgment the session's run owns. + /// Forward an activity to every acknowledgment this exact run owns. /// - /// Terminal activities finalize and release those acknowledgments; the - /// release is identity-checked against the turn id so a late terminal from - /// a superseded run cannot clear the current one. - pub async fn note(&self, session_key: &str, activity: ChannelActivity) { + /// A terminal detaches the turn before any controller I/O, so later activity + /// cannot race behind it and a delayed terminal cannot address a newer run. + pub async fn note(&self, activity_id: &str, activity: ChannelActivity) { let is_terminal = matches!(activity, ChannelActivity::Finished(_)); - let (turn_id, controllers) = { - let inner = self.inner.lock().await; - let Some(turn) = inner.active.get(session_key) else { - return; - }; - let controllers: Vec<_> = turn - .keys - .iter() - .filter_map(|k| inner.pending.get(k).cloned()) - .collect(); - (turn.id, controllers) - }; - - for controller in &controllers { - controller.note(activity.clone()).await; - } - - if is_terminal { + let controllers = { let mut inner = self.inner.lock().await; - // Only release if this session's active turn is still ours. - let still_ours = inner - .active - .get(session_key) - .is_some_and(|turn| turn.id == turn_id); - if still_ours && let Some(turn) = inner.active.remove(session_key) { - for key in turn.keys { - inner.pending.remove(&key); - inner.pending_order.retain(|k| k != &key); - } + if is_terminal { + remove_active(&mut inner, activity_id) + .map(|turn| turn.acknowledgments) + .unwrap_or_default() + } else { + inner + .active + .get(activity_id) + .map(|turn| { + turn.acknowledgments + .iter() + .map(|ack| ActiveAck { + key: ack.key.clone(), + controller: Arc::clone(&ack.controller), + }) + .collect() + }) + .unwrap_or_default() } + }; + + for ack in controllers { + ack.controller.note(activity.clone()).await; } } - /// Finalize whichever turn is active for this session *right now*. + /// Finalize the exact active turn identified by `activity_id`. /// /// Used by abort, where the run future is killed and cannot signal its own - /// terminal. Taking the turn under the lock makes this turn-addressed: a - /// turn that starts afterwards can never be caught by this call, which a - /// session-addressed terminal sent after the fact could. - pub async fn finalize_active(&self, session_key: &str, outcome: ChannelAckOutcome) { - let controllers = { + /// terminal. Taking the turn under the lock ensures a later turn in the + /// same session can never be caught by this call. + pub async fn finalize_active(&self, activity_id: &str, outcome: ChannelAckOutcome) { + let acknowledgments = { let mut inner = self.inner.lock().await; - let Some(turn) = inner.active.remove(session_key) else { - return; - }; - turn.keys - .iter() - .filter_map(|k| { - inner.pending_order.retain(|pk| pk != k); - inner.pending.remove(k) - }) - .collect::>() + remove_active(&mut inner, activity_id) + .map(|turn| turn.acknowledgments) + .unwrap_or_default() }; - for controller in controllers { - controller.note(ChannelActivity::Finished(outcome)).await; - } + finish_acks(acknowledgments, outcome).await; } /// Finalize specific acknowledgments directly, for paths where no run ever @@ -219,17 +277,35 @@ impl ReactionRegistry { pub async fn finalize_keys(&self, keys: &[String], outcome: ChannelAckOutcome) { let controllers = { let mut inner = self.inner.lock().await; - // Drop any active-turn ownership of these keys too, so a later - // terminal cannot address controllers that are already resolved. - inner - .active - .retain(|_, turn| !turn.keys.iter().any(|tk| keys.contains(tk))); - keys.iter() + let mut controllers = keys + .iter() .filter_map(|k| { inner.pending_order.retain(|pk| pk != k); inner.pending.remove(k) }) - .collect::>() + .collect::>(); + let activity_ids = inner.active.keys().cloned().collect::>(); + for activity_id in activity_ids { + let Some(mut turn) = remove_active(&mut inner, &activity_id) else { + continue; + }; + let mut kept = Vec::new(); + for ack in turn.acknowledgments { + if keys.contains(&ack.key) { + controllers.push(ack.controller); + } else { + kept.push(ack); + } + } + if !kept.is_empty() { + turn.acknowledgments = kept; + inner + .active_by_session + .insert(turn.session_key.clone(), activity_id.clone()); + inner.active.insert(activity_id, turn); + } + } + controllers }; for controller in controllers { controller.note(ChannelActivity::Finished(outcome)).await; @@ -237,6 +313,27 @@ impl ReactionRegistry { } } +fn remove_active(inner: &mut RegistryInner, activity_id: &str) -> Option { + let turn = inner.active.remove(activity_id)?; + if inner + .active_by_session + .get(&turn.session_key) + .map(String::as_str) + == Some(activity_id) + { + inner.active_by_session.remove(&turn.session_key); + } + Some(turn) +} + +async fn finish_acks(acknowledgments: Vec, outcome: ChannelAckOutcome) { + for ack in acknowledgments { + ack.controller + .note(ChannelActivity::Finished(outcome)) + .await; + } +} + /// Classify a tool name into a phase emoji shortcode. /// /// Mirrors openclaw's token lists so the single reaction slot communicates what @@ -412,7 +509,11 @@ async fn run_worker( return; }, Ok(None) => { - // Sender dropped without a terminal β€” leave the current marker. + // The controller owner disappeared without a terminal. Strip + // the in-progress marker rather than leaking it indefinitely. + if let Some(cur) = current.take() { + remove_reaction(&outbound, &account_id, &chat_id, &message_id, &cur).await; + } return; }, Err(_) => { @@ -644,11 +745,13 @@ mod tests { let registry = ReactionRegistry::default(); let a = park(®istry, "msg-a").await; let b = park(®istry, "msg-b").await; - registry.activate("sess", vec!["msg-a".into()]).await; + registry + .activate("activity-a", "sess", vec!["msg-a".into()]) + .await; registry .note( - "sess", + "activity-a", ChannelActivity::Finished(ChannelAckOutcome::Success), ) .await; @@ -669,12 +772,12 @@ mod tests { let b = park(®istry, "msg-b").await; // One run answers both messages. registry - .activate("sess", vec!["msg-a".into(), "msg-b".into()]) + .activate("activity-a", "sess", vec!["msg-a".into(), "msg-b".into()]) .await; registry .note( - "sess", + "activity-a", ChannelActivity::Finished(ChannelAckOutcome::Success), ) .await; @@ -690,22 +793,95 @@ mod tests { // Turn 1 activates, is superseded by turn 2, then turn 1's terminal // arrives late. It must not release turn 2's acknowledgment. let registry = ReactionRegistry::default(); + let a = park(®istry, "msg-a").await; + registry + .activate("activity-1", "sess", vec!["msg-a".into()]) + .await; let b = park(®istry, "msg-b").await; - registry.activate("sess", vec!["msg-b".into()]).await; - registry.activate("sess", vec!["msg-b".into()]).await; // turn 2 + registry + .activate("activity-2", "sess", vec!["msg-b".into()]) + .await; + + registry + .note( + "activity-1", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + assert_eq!(ops_of(&a), vec!["+πŸ‘€", "-πŸ‘€"]); + assert_eq!(ops_of(&b), vec!["+πŸ‘€"], "late terminal touched turn 2"); registry .note( - "sess", + "activity-2", ChannelActivity::Finished(ChannelAckOutcome::Success), ) .await; tokio::time::sleep(Duration::from_millis(60)).await; - // The still-current turn resolved exactly once. assert_eq!(ops_of(&b), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"]); } + #[tokio::test] + async fn active_cleanup_threshold_does_not_cancel_healthy_live_turns() { + let registry = ReactionRegistry { + active_limit: 1, + ..ReactionRegistry::default() + }; + let a = park(®istry, "msg-a").await; + let b = park(®istry, "msg-b").await; + registry + .activate("activity-a", "sess-a", vec!["msg-a".into()]) + .await; + registry + .activate("activity-b", "sess-b", vec!["msg-b".into()]) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + + assert_eq!(ops_of(&a), vec!["+πŸ‘€"], "healthy turn was evicted"); + assert_eq!(ops_of(&b), vec!["+πŸ‘€"]); + + registry + .note( + "activity-a", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + registry + .note( + "activity-b", + ChannelActivity::Finished(ChannelAckOutcome::Success), + ) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + assert_eq!(ops_of(&a), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"]); + assert_eq!(ops_of(&b), vec!["+πŸ‘€", "+βœ…", "-πŸ‘€"]); + } + + #[tokio::test] + async fn active_cleanup_threshold_cancels_only_expired_turns() { + let registry = ReactionRegistry { + active_limit: 1, + active_lifetime: Duration::ZERO, + ..ReactionRegistry::default() + }; + let stale = park(®istry, "msg-stale").await; + let current = park(®istry, "msg-current").await; + registry + .activate("activity-stale", "sess-stale", vec!["msg-stale".into()]) + .await; + registry + .activate("activity-current", "sess-current", vec![ + "msg-current".into(), + ]) + .await; + tokio::time::sleep(Duration::from_millis(60)).await; + + assert_eq!(ops_of(&stale), vec!["+πŸ‘€", "-πŸ‘€"]); + assert_eq!(ops_of(¤t), vec!["+πŸ‘€"]); + } + #[tokio::test] async fn finalize_keys_resolves_acks_that_never_ran() { // Hook rejection / early error: no run ever claims the message. @@ -739,13 +915,13 @@ mod tests { } #[tokio::test] - async fn activity_for_unknown_session_is_a_noop() { + async fn activity_for_unknown_id_is_a_noop() { let registry = ReactionRegistry::default(); let a = park(®istry, "msg-a").await; // Never activated β€” a stray activity must not touch the parked ack. registry .note( - "other-session", + "other-activity", ChannelActivity::Finished(ChannelAckOutcome::Success), ) .await; diff --git a/crates/gateway/src/channel_webhook_dedup.rs b/crates/gateway/src/channel_webhook_dedup.rs index 9893c907a8..0eb8f00d05 100644 --- a/crates/gateway/src/channel_webhook_dedup.rs +++ b/crates/gateway/src/channel_webhook_dedup.rs @@ -7,18 +7,53 @@ use std::{collections::HashMap, time::Instant}; +/// Slack documents callback throughput at roughly 2,500 deliveries per account +/// during the five-minute retry window. Keep that full window independently for +/// every account and endpoint. +const MAX_ENTRIES_PER_SCOPE: usize = 2_500; + struct DedupeEntry { inserted_at: Instant, } +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +struct DedupeScope { + channel: String, + account_id: String, + endpoint: String, +} + +impl DedupeScope { + fn new(channel: &str, account_id: &str, endpoint: &str) -> Self { + Self { + channel: channel.to_owned(), + account_id: account_id.to_owned(), + endpoint: endpoint.to_owned(), + } + } + + fn unscoped() -> Self { + Self::new("", "", "") + } +} + +/// Result of atomically admitting work and committing its idempotency key. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ChannelWebhookAdmission { + Duplicate, + Admitted, + Rejected, +} + /// TTL-based idempotency store for channel webhook deduplication. /// /// Tracks provider-specific message IDs to detect duplicate deliveries. -/// Entries expire after `ttl` and the store caps at `max_entries`. +/// Entries expire after `ttl`. Each `(channel, account, endpoint)` partition +/// has its own capacity so traffic in one partition cannot evict another. pub struct ChannelWebhookDedupeStore { - entries: HashMap, + partitions: HashMap>, ttl: std::time::Duration, - max_entries: usize, + max_entries_per_scope: usize, } impl Default for ChannelWebhookDedupeStore { @@ -30,9 +65,9 @@ impl Default for ChannelWebhookDedupeStore { impl ChannelWebhookDedupeStore { pub fn new() -> Self { Self { - entries: HashMap::new(), + partitions: HashMap::new(), ttl: std::time::Duration::from_millis(moltis_protocol::DEDUPE_TTL_MS), - max_entries: moltis_protocol::DEDUPE_MAX_ENTRIES, + max_entries_per_scope: MAX_ENTRIES_PER_SCOPE, } } @@ -40,27 +75,85 @@ impl ChannelWebhookDedupeStore { /// If new, inserts the key and returns `false`. pub fn check_and_insert(&mut self, key: &str) -> bool { self.evict_expired(); - if self.entries.contains_key(key) { + let scope = DedupeScope::unscoped(); + if self + .partitions + .get(&scope) + .is_some_and(|entries| entries.contains_key(key)) + { + return true; + } + self.insert_new(scope, key.to_owned()); + false + } + + /// Check a provider key within a channel account and callback endpoint. + pub fn check_and_insert_scoped( + &mut self, + channel: &str, + account_id: &str, + endpoint: &str, + provider_key: &str, + ) -> bool { + self.evict_expired(); + let scope = DedupeScope::new(channel, account_id, endpoint); + if self + .partitions + .get(&scope) + .is_some_and(|entries| entries.contains_key(provider_key)) + { return true; } - if self.entries.len() >= self.max_entries - && let Some(oldest_key) = self - .entries + self.insert_new(scope, provider_key.to_owned()); + false + } + + /// Record the idempotency key only when bounded work admission succeeds. + pub fn admit_scoped( + &mut self, + channel: &str, + account_id: &str, + endpoint: &str, + provider_key: &str, + admit: impl FnOnce() -> bool, + ) -> ChannelWebhookAdmission { + self.evict_expired(); + let scope = DedupeScope::new(channel, account_id, endpoint); + if self + .partitions + .get(&scope) + .is_some_and(|entries| entries.contains_key(provider_key)) + { + return ChannelWebhookAdmission::Duplicate; + } + if !admit() { + return ChannelWebhookAdmission::Rejected; + } + self.insert_new(scope, provider_key.to_owned()); + ChannelWebhookAdmission::Admitted + } + + fn insert_new(&mut self, scope: DedupeScope, key: String) { + let entries = self.partitions.entry(scope).or_default(); + if entries.len() >= self.max_entries_per_scope + && let Some(oldest_key) = entries .iter() - .min_by_key(|(_, v)| v.inserted_at) - .map(|(k, _)| k.clone()) + .min_by_key(|(_, entry)| entry.inserted_at) + .map(|(key, _)| key.clone()) { - self.entries.remove(&oldest_key); + entries.remove(&oldest_key); } - self.entries.insert(key.to_string(), DedupeEntry { + entries.insert(key, DedupeEntry { inserted_at: Instant::now(), }); - false } fn evict_expired(&mut self) { let cutoff = Instant::now() - self.ttl; - self.entries.retain(|_, v| v.inserted_at > cutoff); + self.partitions.retain(|_, entries| { + entries.retain(|_, entry| entry.inserted_at > cutoff); + !entries.is_empty() + }); } } @@ -90,12 +183,45 @@ mod tests { assert!(store.check_and_insert("key1")); } + #[test] + fn scoped_keys_are_isolated_by_account_and_endpoint() { + let mut store = ChannelWebhookDedupeStore::new(); + assert!(!store.check_and_insert_scoped("slack", "a", "events", "trigger")); + assert!(store.check_and_insert_scoped("slack", "a", "events", "trigger")); + assert!(!store.check_and_insert_scoped("slack", "b", "events", "trigger")); + assert!(!store.check_and_insert_scoped("slack", "a", "commands", "trigger")); + } + + #[test] + fn scoped_keys_are_unambiguous() { + let mut store = ChannelWebhookDedupeStore::new(); + assert!(!store.check_and_insert_scoped("slack", "a:events", "commands", "id")); + assert!(!store.check_and_insert_scoped("slack", "a", "events:commands", "id")); + } + + #[test] + fn rejected_admission_does_not_consume_dedup_key() { + let mut store = ChannelWebhookDedupeStore::new(); + assert_eq!( + store.admit_scoped("slack", "a", "events", "event-1", || false), + ChannelWebhookAdmission::Rejected + ); + assert_eq!( + store.admit_scoped("slack", "a", "events", "event-1", || true), + ChannelWebhookAdmission::Admitted + ); + assert_eq!( + store.admit_scoped("slack", "a", "events", "event-1", || false), + ChannelWebhookAdmission::Duplicate + ); + } + #[test] fn evicts_oldest_at_capacity() { let mut store = ChannelWebhookDedupeStore { - entries: HashMap::new(), + partitions: HashMap::new(), ttl: std::time::Duration::from_secs(300), - max_entries: 2, + max_entries_per_scope: 2, }; assert!(!store.check_and_insert("a")); // Small sleep so "a" is strictly oldest @@ -106,4 +232,26 @@ mod tests { assert!(!store.check_and_insert("a")); // "a" was evicted, so it's new again assert!(store.check_and_insert("c")); // "c" is still there } + + #[test] + fn retains_retry_window_capacity_independently_per_account_and_endpoint() { + let mut store = ChannelWebhookDedupeStore::new(); + + for index in 0..MAX_ENTRIES_PER_SCOPE { + let key = format!("account-a-event-{index}"); + assert!(!store.check_and_insert_scoped("slack", "a", "events", &key)); + } + for index in 0..MAX_ENTRIES_PER_SCOPE { + let key = format!("account-b-event-{index}"); + assert!(!store.check_and_insert_scoped("slack", "b", "events", &key)); + } + for index in 0..MAX_ENTRIES_PER_SCOPE { + let key = format!("account-a-command-{index}"); + assert!(!store.check_and_insert_scoped("slack", "a", "commands", &key)); + } + + assert!(store.check_and_insert_scoped("slack", "a", "events", "account-a-event-0")); + assert!(store.check_and_insert_scoped("slack", "b", "events", "account-b-event-0")); + assert!(store.check_and_insert_scoped("slack", "a", "commands", "account-a-command-0")); + } } diff --git a/crates/gateway/src/channel_webhook_middleware.rs b/crates/gateway/src/channel_webhook_middleware.rs index 5849bf3aac..06891dc24a 100644 --- a/crates/gateway/src/channel_webhook_middleware.rs +++ b/crates/gateway/src/channel_webhook_middleware.rs @@ -5,7 +5,7 @@ //! //! 1. Signature verification (via the channel's [`ChannelWebhookVerifier`]) //! 2. Timestamp staleness rejection -//! 3. Per-(channel, account) rate limiting +//! 3. Per-(channel, account, endpoint) rate limiting //! 4. Idempotency deduplication use http::HeaderMap; @@ -20,24 +20,22 @@ use crate::{ channel_webhook_rate_limit::ChannelWebhookRateLimiter, }; -/// Run the full channel webhook verification pipeline. +/// Verify, freshness-check, and rate-limit a channel webhook without consuming +/// its idempotency key. Callers that queue work asynchronously can use this and +/// commit dedup atomically with queue admission. /// /// Steps: /// 1. Verify the cryptographic signature via the channel's verifier. /// 2. Check the timestamp is within the acceptable staleness window. -/// 3. Apply per-(channel, account) rate limiting. -/// 4. Deduplicate by provider message ID (if present). -/// -/// On success returns the verified envelope and the dedup result. -/// On failure returns a rejection that transport layers can map to responses. -pub fn channel_webhook_gate( +/// 3. Apply per-(channel, account, endpoint) rate limiting. +pub fn verify_channel_webhook( verifier: &dyn ChannelWebhookVerifier, - dedup_store: &std::sync::RwLock, rate_limiter: &ChannelWebhookRateLimiter, account_id: &str, + endpoint: &str, headers: &HeaderMap, body: &[u8], -) -> Result<(VerifiedChannelWebhook, ChannelWebhookDedupeResult), ChannelWebhookRejection> { +) -> Result { #[cfg(feature = "metrics")] let start = std::time::Instant::now(); @@ -102,10 +100,11 @@ pub fn channel_webhook_gate( return Err(rejection); } - // Step 3: Per-(channel, account) rate limiting. + // Step 3: Per-(channel, account, endpoint) rate limiting. if let Err(rejection) = rate_limiter.check( verifier.channel_type().as_str(), account_id, + endpoint, &verifier.rate_policy(), ) { #[cfg(feature = "metrics")] @@ -120,10 +119,34 @@ pub fn channel_webhook_gate( return Err(rejection); } - // Step 4: Idempotency deduplication. + Ok(envelope) +} + +/// Run the full channel webhook verification pipeline. +/// +/// On success returns the verified envelope and the dedup result. Synchronous +/// consumers can use this directly; async consumers should use +/// [`verify_channel_webhook`] and commit dedup with bounded work admission. +pub fn channel_webhook_gate( + verifier: &dyn ChannelWebhookVerifier, + dedup_store: &std::sync::RwLock, + rate_limiter: &ChannelWebhookRateLimiter, + account_id: &str, + endpoint: &str, + headers: &HeaderMap, + body: &[u8], +) -> Result<(VerifiedChannelWebhook, ChannelWebhookDedupeResult), ChannelWebhookRejection> { + let envelope = + verify_channel_webhook(verifier, rate_limiter, account_id, endpoint, headers, body)?; + let dedup_result = if let Some(ref key) = envelope.idempotency_key { let mut store = dedup_store.write().unwrap_or_else(|e| e.into_inner()); - if store.check_and_insert(key) { + if store.check_and_insert_scoped( + verifier.channel_type().as_str(), + account_id, + endpoint, + key, + ) { #[cfg(feature = "metrics")] { use moltis_metrics::{counter, labels}; @@ -214,6 +237,7 @@ mod tests { &store, &limiter, "acct1", + "events", &HeaderMap::new(), body, ); @@ -232,6 +256,7 @@ mod tests { &store, &limiter, "acct1", + "events", &HeaderMap::new(), b"{}", ); @@ -248,15 +273,74 @@ mod tests { let body = br#"{"id":"ev-dup"}"#; let headers = HeaderMap::new(); - let (_, d1) = - channel_webhook_gate(&PassVerifier, &store, &limiter, "acct1", &headers, body).unwrap(); + let (_, d1) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "events", + &headers, + body, + ) + .unwrap(); assert_eq!(d1, ChannelWebhookDedupeResult::New); - let (_, d2) = - channel_webhook_gate(&PassVerifier, &store, &limiter, "acct1", &headers, body).unwrap(); + let (_, d2) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "events", + &headers, + body, + ) + .unwrap(); assert_eq!(d2, ChannelWebhookDedupeResult::Duplicate); } + #[test] + fn gate_namespaces_dedup_by_account_and_endpoint() { + let store = make_store(); + let limiter = make_limiter(); + let body = br#"{"id":"same-provider-id"}"#; + let headers = HeaderMap::new(); + + let (_, event) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "events", + &headers, + body, + ) + .unwrap(); + let (_, command) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "commands", + &headers, + body, + ) + .unwrap(); + let (_, other_account) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct2", + "events", + &headers, + body, + ) + .unwrap(); + + assert_eq!(event, ChannelWebhookDedupeResult::New); + assert_eq!(command, ChannelWebhookDedupeResult::New); + assert_eq!(other_account, ChannelWebhookDedupeResult::New); + } + #[test] fn gate_skips_dedup_without_idempotency_key() { let store = make_store(); @@ -264,13 +348,29 @@ mod tests { let body = br#"{"text":"no id"}"#; let headers = HeaderMap::new(); - let (_, d1) = - channel_webhook_gate(&PassVerifier, &store, &limiter, "acct1", &headers, body).unwrap(); + let (_, d1) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "events", + &headers, + body, + ) + .unwrap(); assert_eq!(d1, ChannelWebhookDedupeResult::New); // Same body without id β€” still New (no dedup key) - let (_, d2) = - channel_webhook_gate(&PassVerifier, &store, &limiter, "acct1", &headers, body).unwrap(); + let (_, d2) = channel_webhook_gate( + &PassVerifier, + &store, + &limiter, + "acct1", + "events", + &headers, + body, + ) + .unwrap(); assert_eq!(d2, ChannelWebhookDedupeResult::New); } @@ -318,6 +418,7 @@ mod tests { &store, &limiter, "rate-test", + "events", &headers, b"{}", ); @@ -330,6 +431,7 @@ mod tests { &store, &limiter, "rate-test", + "events", &headers, b"{}", ); diff --git a/crates/gateway/src/channel_webhook_rate_limit.rs b/crates/gateway/src/channel_webhook_rate_limit.rs index f09866fe69..5d9a93ea19 100644 --- a/crates/gateway/src/channel_webhook_rate_limit.rs +++ b/crates/gateway/src/channel_webhook_rate_limit.rs @@ -1,6 +1,6 @@ -//! Per-(channel, account) token-bucket rate limiter for channel webhooks. +//! Per-(channel, account, endpoint) token-bucket rate limiter for channel webhooks. //! -//! Each `(channel_type, account_id)` pair gets its own bucket. Buckets are +//! Each `(channel_type, account_id, endpoint)` tuple gets its own bucket. Buckets are //! lazily created on first request and automatically evicted when stale. use std::time::{Duration, Instant}; @@ -11,14 +11,19 @@ use moltis_channels::channel_webhook_middleware::{ ChannelWebhookRatePolicy, ChannelWebhookRejection, }; -/// Composite key for per-(channel, account) rate limiting. +/// Composite key for per-(channel, account, endpoint) rate limiting. type BucketKey = String; -fn bucket_key(channel: &str, account_id: &str) -> BucketKey { - format!("{channel}:{account_id}") +fn bucket_key(channel: &str, account_id: &str, endpoint: &str) -> BucketKey { + format!( + "{}:{channel}{}:{account_id}{}:{endpoint}", + channel.len(), + account_id.len(), + endpoint.len() + ) } -/// Token-bucket state for a single (channel, account) pair. +/// Token-bucket state for a single (channel, account, endpoint) tuple. struct Bucket { tokens: f64, last_refill: Instant, @@ -41,7 +46,7 @@ impl ChannelWebhookRateLimiter { } } - /// Check the rate limit for a (channel, account) pair. + /// Check the rate limit for a (channel, account, endpoint) tuple. /// /// Returns `Ok(())` if the request is allowed, or /// `Err(ChannelWebhookRejection::RateLimited { .. })` if the bucket is exhausted. @@ -49,9 +54,10 @@ impl ChannelWebhookRateLimiter { &self, channel_type: &str, account_id: &str, + endpoint: &str, policy: &ChannelWebhookRatePolicy, ) -> Result<(), ChannelWebhookRejection> { - let key = bucket_key(channel_type, account_id); + let key = bucket_key(channel_type, account_id, endpoint); let rate_per_sec = f64::from(policy.max_requests_per_minute) / 60.0; let capacity = f64::from(policy.max_requests_per_minute + policy.burst); @@ -139,7 +145,7 @@ mod tests { let policy = default_policy(); // First request should always be allowed (bucket starts full). - assert!(limiter.check("slack", "acct1", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); } #[test] @@ -152,11 +158,11 @@ mod tests { // Exhaust the full capacity (rate + burst = 8 tokens). for _ in 0..8 { - assert!(limiter.check("slack", "acct1", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); } // Next request should be rejected. - let result = limiter.check("slack", "acct1", &policy); + let result = limiter.check("slack", "acct1", "events", &policy); assert!(matches!( result, Err(ChannelWebhookRejection::RateLimited { .. }) @@ -173,12 +179,12 @@ mod tests { // Exhaust account 1. for _ in 0..6 { - assert!(limiter.check("slack", "acct1", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); } - assert!(limiter.check("slack", "acct1", &policy).is_err()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_err()); // Account 2 should still be fine. - assert!(limiter.check("slack", "acct2", &policy).is_ok()); + assert!(limiter.check("slack", "acct2", "events", &policy).is_ok()); } #[test] @@ -190,12 +196,16 @@ mod tests { }; for _ in 0..6 { - assert!(limiter.check("slack", "acct1", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); } - assert!(limiter.check("slack", "acct1", &policy).is_err()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_err()); // Same account, different channel β€” separate bucket. - assert!(limiter.check("msteams", "acct1", &policy).is_ok()); + assert!( + limiter + .check("msteams", "acct1", "webhook", &policy) + .is_ok() + ); } #[test] @@ -203,7 +213,7 @@ mod tests { let limiter = ChannelWebhookRateLimiter::new(); let policy = default_policy(); - limiter.check("slack", "acct1", &policy).ok(); + limiter.check("slack", "acct1", "events", &policy).ok(); assert_eq!(limiter.bucket_count(), 1); // Evict with zero max_idle removes everything. @@ -220,15 +230,34 @@ mod tests { }; for _ in 0..6 { - limiter.check("slack", "acct1", &policy).ok(); + limiter.check("slack", "acct1", "events", &policy).ok(); } if let Err(ChannelWebhookRejection::RateLimited { retry_after }) = - limiter.check("slack", "acct1", &policy) + limiter.check("slack", "acct1", "events", &policy) { assert!(retry_after.as_secs_f64() > 0.0); } else { panic!("expected RateLimited"); } } + + #[test] + fn separate_endpoints_have_separate_buckets() { + let limiter = ChannelWebhookRateLimiter::new(); + let policy = ChannelWebhookRatePolicy { + max_requests_per_minute: 2, + burst: 0, + }; + + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_ok()); + assert!(limiter.check("slack", "acct1", "events", &policy).is_err()); + assert!(limiter.check("slack", "acct1", "commands", &policy).is_ok()); + assert!( + limiter + .check("slack", "acct1", "interactions", &policy) + .is_ok() + ); + } } diff --git a/crates/gateway/src/chat.rs b/crates/gateway/src/chat.rs index 96abf5f76b..cb07595851 100644 --- a/crates/gateway/src/chat.rs +++ b/crates/gateway/src/chat.rs @@ -58,28 +58,33 @@ impl ChatRuntime for GatewayChatRuntime { // ── Channel acknowledgment reactions ──────────────────────────────────── - async fn note_channel_activity(&self, session_key: &str, activity: ChannelActivity) { + async fn note_channel_activity(&self, activity_id: &str, activity: ChannelActivity) { self.state .channel_reaction_controllers - .note(session_key, activity) + .note(activity_id, activity) .await; } - async fn activate_channel_acks(&self, session_key: &str, ack_keys: Vec) { + async fn activate_channel_acks( + &self, + activity_id: &str, + session_key: &str, + ack_keys: Vec, + ) { self.state .channel_reaction_controllers - .activate(session_key, ack_keys) + .activate(activity_id, session_key, ack_keys) .await; } async fn finalize_active_channel_acks( &self, - session_key: &str, + activity_id: &str, outcome: moltis_channels::ChannelAckOutcome, ) { self.state .channel_reaction_controllers - .finalize_active(session_key, outcome) + .finalize_active(activity_id, outcome) .await; } diff --git a/crates/httpd/src/auth_middleware.rs b/crates/httpd/src/auth_middleware.rs index 612041ce5b..79db040b57 100644 --- a/crates/httpd/src/auth_middleware.rs +++ b/crates/httpd/src/auth_middleware.rs @@ -235,15 +235,13 @@ pub async fn auth_gate( } } -/// Paths that never require authentication. -#[cfg(feature = "web-ui")] /// Exactly the three Slack callback endpoints, and nothing else under the /// Slack namespace. /// /// Matching the whole `/api/channels/slack/` prefix would also expose any /// future management route added there, so the suffix is checked explicitly. #[cfg(feature = "slack")] -fn is_slack_callback_path(path: &str) -> bool { +pub(crate) fn is_slack_callback_path(path: &str) -> bool { let Some(rest) = path.strip_prefix("/api/channels/slack/") else { return false; }; @@ -255,6 +253,8 @@ fn is_slack_callback_path(path: &str) -> bool { ) } +/// Paths that never require authentication. +#[cfg(feature = "web-ui")] fn is_public_path(path: &str) -> bool { matches!( path, @@ -464,7 +464,7 @@ pub fn parse_cookie<'a>(header: &'a str, name: &str) -> Option<&'a str> { #[cfg(test)] mod tests { - #[cfg(feature = "slack")] + #[cfg(all(feature = "slack", feature = "web-ui"))] #[test] fn slack_callback_paths_bypass_gateway_auth_but_nothing_else_does() { // Slack cannot present a Moltis session; these three verify Slack's @@ -490,6 +490,164 @@ mod tests { assert!(!is_public_path(p), "{p} must stay authenticated"); } } + + #[cfg(feature = "slack")] + #[test] + fn slack_callback_helper_is_available_without_web_ui() { + assert!(is_slack_callback_path("/api/channels/slack/account/events")); + assert!(!is_slack_callback_path( + "/api/channels/slack/account/events/extra" + )); + } + + #[cfg(all(feature = "slack", feature = "web-ui"))] + #[tokio::test] + async fn composed_router_bypasses_auth_only_for_hmac_verified_slack_callbacks() + -> Result<(), Box> { + use { + axum::{ + Router, + body::Bytes, + extract::{Path, State}, + http::Request, + middleware::{Next, from_fn}, + response::IntoResponse, + routing::post, + }, + moltis_channels::ChannelWebhookVerifier as _, + moltis_gateway::channel_webhook_dedup::ChannelWebhookDedupeStore, + moltis_slack::channel_webhook_verifier::SlackChannelWebhookVerifier, + secrecy::Secret, + std::sync::RwLock, + }; + + type CallbackState = Arc>; + + async fn callback( + State(dedup): State, + Path((account_id, endpoint)): Path<(String, String)>, + headers: HeaderMap, + body: Bytes, + ) -> axum::response::Response { + let verifier = SlackChannelWebhookVerifier::new(Secret::new( + "test_signing_secret_123".to_string(), + )); + let verified = match verifier.verify(&headers, &body) { + Ok(verified) => verified, + Err(_) => return StatusCode::UNAUTHORIZED.into_response(), + }; + let duplicate = verified.idempotency_key.as_deref().is_some_and(|key| { + dedup + .write() + .unwrap_or_else(|error| error.into_inner()) + .check_and_insert_scoped("slack", &account_id, &endpoint, key) + }); + if duplicate { + return Json(serde_json::json!({ "deduplicated": true })).into_response(); + } + let challenge = serde_json::from_slice::(&body) + .ok() + .and_then(|value| value["challenge"].as_str().map(ToOwned::to_owned)); + match challenge { + Some(challenge) => { + Json(serde_json::json!({ "challenge": challenge })).into_response() + }, + None => Json(serde_json::json!({ "deduplicated": false })).into_response(), + } + } + + async fn test_auth( + request: Request, + next: Next, + ) -> axum::response::Response { + if is_public_path(request.uri().path()) { + next.run(request).await + } else { + StatusCode::UNAUTHORIZED.into_response() + } + } + + let app = Router::new() + .route( + "/api/channels/slack/{account_id}/{endpoint}", + post(callback), + ) + .layer(from_fn(test_auth)) + .with_state(Arc::new(RwLock::new(ChannelWebhookDedupeStore::new()))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + let server = tokio::spawn(async move { axum::serve(listener, app).await }); + let client = reqwest::Client::new(); + let body = r#"{"type":"url_verification","challenge":"route-ok"}"#; + let callback_url = format!("http://{addr}/api/channels/slack/account/events"); + + let signed = client + .post(&callback_url) + .header("x-slack-request-timestamp", "1700000000") + .header( + "x-slack-signature", + "v0=f2ff105195f235457c3d150889fb4f7743c7ee1e21de2fb7e68ae000efc0ca73", + ) + .body(body) + .send() + .await?; + assert_eq!(signed.status(), StatusCode::OK); + assert_eq!( + signed.json::().await?["challenge"], + "route-ok" + ); + + let unsigned = client.post(&callback_url).body(body).send().await?; + assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED); + + for (endpoint, body, signature, duplicate) in [ + ( + "commands", + "command=%2Fmoltis&trigger_id=route-trigger&text=hello\n", + "v0=ba1da8a2c1281bfc9e48a97c750a7f9906303fd4824237e9870094fdd0ff71c3", + false, + ), + ( + "interactions", + "payload=%7B%22type%22%3A%22block_actions%22%2C%22trigger_id%22%3A%22route-trigger%22%7D\n", + "v0=007a942d6e04a0f1eecdd87b8f101a1a6505668b8e38cf35b3e1cfc344b8e424", + false, + ), + ( + "commands", + "command=%2Fmoltis&trigger_id=route-trigger&text=hello\n", + "v0=ba1da8a2c1281bfc9e48a97c750a7f9906303fd4824237e9870094fdd0ff71c3", + true, + ), + ] { + let response = client + .post(format!( + "http://{addr}/api/channels/slack/account/{endpoint}" + )) + .header("x-slack-request-timestamp", "1700000000") + .header("x-slack-signature", signature) + .body(body) + .send() + .await?; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.json::().await?["deduplicated"], + duplicate, + "{endpoint}" + ); + } + + for path in [ + "/api/channels/slack/account/config", + "/api/channels/slack/account/events/extra", + ] { + let response = client.post(format!("http://{addr}{path}")).send().await?; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED, "{path}"); + } + + server.abort(); + Ok(()) + } use {super::*, sqlx::SqlitePool}; #[test] diff --git a/crates/httpd/src/request_throttle.rs b/crates/httpd/src/request_throttle.rs index 0081690e30..e52fdaf177 100644 --- a/crates/httpd/src/request_throttle.rs +++ b/crates/httpd/src/request_throttle.rs @@ -37,10 +37,15 @@ enum ThrottleScope { Api, Share, Ws, + SlackCallbackPreauth, } impl ThrottleScope { fn from_request(method: &Method, path: &str) -> Option { + #[cfg(feature = "slack")] + if method == Method::POST && crate::auth_middleware::is_slack_callback_path(path) { + return Some(Self::SlackCallbackPreauth); + } if path == "/api/auth/login" && method == Method::POST { return Some(Self::Login); } @@ -58,6 +63,10 @@ impl ThrottleScope { } None } + + fn permits_authenticated_bypass(self) -> bool { + !matches!(self, Self::SlackCallbackPreauth) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -85,6 +94,7 @@ struct ThrottleLimits { api: RateLimit, share: RateLimit, ws: RateLimit, + callback_preauth: RateLimit, } impl Default for ThrottleLimits { @@ -115,6 +125,13 @@ impl Default for ThrottleLimits { max_requests: 30, window: Duration::from_secs(60), }, + // This IP-level ceiling runs before account lookup and signature + // verification. It is intentionally above Slack's three callback + // endpoints at their verified 600/minute + 200 burst policy. + callback_preauth: RateLimit { + max_requests: 5_000, + window: Duration::from_secs(60), + }, } } } @@ -145,6 +162,7 @@ impl RequestThrottle { ThrottleScope::Api => self.limits.api, ThrottleScope::Share => self.limits.share, ThrottleScope::Ws => self.limits.ws, + ThrottleScope::SlackCallbackPreauth => self.limits.callback_preauth, } } @@ -208,6 +226,7 @@ impl RequestThrottle { self.limits.api.window, self.limits.share.window, self.limits.ws.window, + self.limits.callback_preauth.window, ] .into_iter() .max() @@ -233,7 +252,9 @@ pub async fn throttle_gate( return next.run(request).await; }; - if should_bypass_throttling(&state, request.headers(), addr).await { + if scope.permits_authenticated_bypass() + && should_bypass_throttling(&state, request.headers(), addr).await + { return next.run(request).await; } @@ -354,6 +375,37 @@ mod tests { ); } + #[cfg(feature = "slack")] + #[test] + fn exact_slack_callbacks_use_preauth_throttle_scope() { + for path in [ + "/api/channels/slack/account/events", + "/api/channels/slack/account/interactions", + "/api/channels/slack/account/commands", + ] { + assert_eq!( + ThrottleScope::from_request(&Method::POST, path), + Some(ThrottleScope::SlackCallbackPreauth), + "{path}" + ); + } + for path in [ + "/api/channels/slack/account/config", + "/api/channels/slack/account/events/extra", + "/api/channels/slack//events", + ] { + assert_eq!( + ThrottleScope::from_request(&Method::POST, path), + Some(ThrottleScope::Api), + "{path}" + ); + } + assert_eq!( + ThrottleScope::from_request(&Method::GET, "/api/channels/slack/account/events"), + Some(ThrottleScope::Api) + ); + } + #[test] fn classify_ws_request() { assert_eq!( @@ -398,6 +450,10 @@ mod tests { max_requests: 100, window: Duration::from_secs(10), }, + callback_preauth: RateLimit { + max_requests: 100, + window: Duration::from_secs(10), + }, }); let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); @@ -426,6 +482,73 @@ mod tests { )); } + #[cfg(feature = "slack")] + #[test] + fn callback_preauth_scope_limits_floods_without_authenticated_bypass() { + let limits = ThrottleLimits { + callback_preauth: RateLimit { + max_requests: 2, + window: Duration::from_secs(10), + }, + ..ThrottleLimits::default() + }; + let throttle = RequestThrottle::with_limits(limits); + let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + let now = Instant::now(); + + assert!(!ThrottleScope::SlackCallbackPreauth.permits_authenticated_bypass()); + assert!(matches!( + throttle.check_at(ip, ThrottleScope::SlackCallbackPreauth, now), + ThrottleDecision::Allowed + )); + assert!(matches!( + throttle.check_at(ip, ThrottleScope::SlackCallbackPreauth, now), + ThrottleDecision::Allowed + )); + assert!(matches!( + throttle.check_at(ip, ThrottleScope::SlackCallbackPreauth, now), + ThrottleDecision::Denied { .. } + )); + } + + #[cfg(feature = "slack")] + #[test] + fn callback_preauth_ceiling_does_not_override_verified_slack_burst() { + let throttle = RequestThrottle::new(); + assert!( + throttle.limits.callback_preauth.max_requests > 3 * (600 + 200), + "preauth must allow one account's three endpoint bursts" + ); + let verified = moltis_gateway::channel_webhook_rate_limit::ChannelWebhookRateLimiter::new(); + let policy = moltis_channels::ChannelWebhookRatePolicy { + max_requests_per_minute: 600, + burst: 200, + }; + let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + let now = Instant::now(); + + for _ in 0..800 { + assert!(matches!( + throttle.check_at(ip, ThrottleScope::SlackCallbackPreauth, now), + ThrottleDecision::Allowed + )); + assert!( + verified + .check("slack", "account", "events", &policy) + .is_ok() + ); + } + assert!(matches!( + throttle.check_at(ip, ThrottleScope::SlackCallbackPreauth, now), + ThrottleDecision::Allowed + )); + assert!( + verified + .check("slack", "account", "events", &policy) + .is_err() + ); + } + #[test] fn forwarded_ip_uses_first_xff_value() { let mut headers = HeaderMap::new(); diff --git a/crates/httpd/src/server/gateway.rs b/crates/httpd/src/server/gateway.rs index 2e3ff0ac3b..97cc914d13 100644 --- a/crates/httpd/src/server/gateway.rs +++ b/crates/httpd/src/server/gateway.rs @@ -3,7 +3,7 @@ use std::{path::PathBuf, sync::Arc}; -#[cfg(any(feature = "msteams", feature = "telephony"))] +#[cfg(any(feature = "msteams", feature = "slack", feature = "telephony"))] use moltis_channels::ChannelPlugin; use { @@ -24,6 +24,11 @@ use super::{ PreparedGateway, RouteEnhancer, builder::finalize_gateway_app, runtime::FinalizeGatewayArgs, }; +#[cfg(feature = "slack")] +fn slack_json_response(status: StatusCode, body: serde_json::Value) -> axum::response::Response { + (status, Json(body)).into_response() +} + #[cfg(feature = "tailscale")] use super::TailscaleOpts; @@ -251,7 +256,10 @@ pub async fn prepare_gateway( router }; - let mut app = finalize_gateway_app(router, app_state, config.server.http_request_logs); + // Keep every callback on the stateful base router until all routes have + // been registered. Finalization below applies the global middleware stack + // only to routes already present at that point. + let mut app = router; #[cfg(feature = "msteams")] { @@ -315,6 +323,7 @@ pub async fn prepare_gateway( &gw_state.channel_webhook_dedup, &gw_state.channel_webhook_rate_limiter, &account_id, + "webhook", &merged_headers, &body, ) { @@ -374,9 +383,12 @@ pub async fn prepare_gateway( } #[cfg(feature = "slack")] { - // Slack Events API webhook + let slack_callback_dispatcher = super::slack_callbacks::SlackCallbackDispatcher::start( + Arc::clone(&slack_webhook_plugin), + ); let slack_events_plugin = Arc::clone(&slack_webhook_plugin); let state_for_slack_events = Arc::clone(&state); + let dispatcher_for_slack_events = Arc::clone(&slack_callback_dispatcher); app = app.route( "/api/channels/slack/{account_id}/events", axum::routing::post( @@ -385,26 +397,23 @@ pub async fn prepare_gateway( body: axum::body::Bytes| { let plugin = Arc::clone(&slack_events_plugin); let gw_state = Arc::clone(&state_for_slack_events); + let dispatcher = Arc::clone(&dispatcher_for_slack_events); async move { - // Get the verifier from the plugin. let verifier = { let p = plugin.read().await; p.channel_webhook_verifier(&account_id) }; let Some(verifier) = verifier else { - return ( + return slack_json_response( StatusCode::NOT_FOUND, - Json(serde_json::json!({ "ok": false, "error": "unknown Slack account" })), - ) - .into_response(); + serde_json::json!({ "ok": false, "error": "unknown Slack account" }), + ); }; - - // Run the middleware pipeline. - match moltis_gateway::channel_webhook_middleware::channel_webhook_gate( + match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( verifier.as_ref(), - &gw_state.channel_webhook_dedup, &gw_state.channel_webhook_rate_limiter, &account_id, + "events", &headers, &body, ) { @@ -413,45 +422,53 @@ pub async fn prepare_gateway( rejection, ) }, - Ok((_, moltis_channels::ChannelWebhookDedupeResult::Duplicate)) => ( - StatusCode::OK, - Json(serde_json::json!({ "ok": true, "deduplicated": true })), - ) - .into_response(), - Ok((verified, moltis_channels::ChannelWebhookDedupeResult::New)) => { - // Dispatch to Slack plugin with verified body. - let result = { - let p = plugin.read().await; - p.ingest_verified_webhook(&account_id, &verified.body) - .await - }; - match result { - Ok(Some(challenge)) => ( + Ok(verified) => { + let payload: serde_json::Value = + match serde_json::from_slice(&verified.body) { + Ok(payload) => payload, + Err(error) => { + return slack_json_response( + StatusCode::BAD_REQUEST, + serde_json::json!({ "ok": false, "error": error.to_string() }), + ); + }, + }; + if payload.get("type").and_then(|value| value.as_str()) + == Some("url_verification") + { + return match payload + .get("challenge") + .and_then(|value| value.as_str()) + { + Some(challenge) => slack_json_response( + StatusCode::OK, + serde_json::json!({ "challenge": challenge }), + ), + None => slack_json_response( + StatusCode::BAD_REQUEST, + serde_json::json!({ "ok": false, "error": "missing Slack challenge" }), + ), + }; + } + match dispatcher.admit( + &gw_state.channel_webhook_dedup, + account_id, + verified.idempotency_key.as_deref(), + super::slack_callbacks::SlackCallbackKind::Event, + verified.body, + ) { + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate => slack_json_response( StatusCode::OK, - Json(serde_json::json!({ "challenge": challenge })), - ) - .into_response(), - Ok(None) => ( + serde_json::json!({ "ok": true, "deduplicated": true }), + ), + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => slack_json_response( StatusCode::OK, - Json(serde_json::json!({ "ok": true })), - ) - .into_response(), - Err(e) => { - let msg = e.to_string(); - if msg.contains("unknown") { - ( - StatusCode::NOT_FOUND, - Json(serde_json::json!({ "ok": false, "error": msg })), - ) - .into_response() - } else { - ( - StatusCode::BAD_REQUEST, - Json(serde_json::json!({ "ok": false, "error": msg })), - ) - .into_response() - } - }, + serde_json::json!({ "ok": true }), + ), + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( + StatusCode::SERVICE_UNAVAILABLE, + serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), + ), } }, } @@ -459,10 +476,9 @@ pub async fn prepare_gateway( }, ), ); - - // Slack interaction webhook -- receives button click payloads. let slack_interact_plugin = Arc::clone(&slack_webhook_plugin); let state_for_slack_interact = Arc::clone(&state); + let dispatcher_for_slack_interact = Arc::clone(&slack_callback_dispatcher); app = app.route( "/api/channels/slack/{account_id}/interactions", axum::routing::post( @@ -471,26 +487,23 @@ pub async fn prepare_gateway( body: axum::body::Bytes| { let plugin = Arc::clone(&slack_interact_plugin); let gw_state = Arc::clone(&state_for_slack_interact); + let dispatcher = Arc::clone(&dispatcher_for_slack_interact); async move { - // Get the verifier from the plugin. let verifier = { let p = plugin.read().await; p.channel_webhook_verifier(&account_id) }; let Some(verifier) = verifier else { - return ( + return slack_json_response( StatusCode::NOT_FOUND, - Json(serde_json::json!({ "ok": false, "error": "unknown Slack account" })), - ) - .into_response(); + serde_json::json!({ "ok": false, "error": "unknown Slack account" }), + ); }; - - // Run the middleware pipeline. - match moltis_gateway::channel_webhook_middleware::channel_webhook_gate( + match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( verifier.as_ref(), - &gw_state.channel_webhook_dedup, &gw_state.channel_webhook_rate_limiter, &account_id, + "interactions", &headers, &body, ) { @@ -499,43 +512,34 @@ pub async fn prepare_gateway( rejection, ) }, - Ok((_, moltis_channels::ChannelWebhookDedupeResult::Duplicate)) => ( - StatusCode::OK, - Json(serde_json::json!({ "ok": true, "deduplicated": true })), - ) - .into_response(), - Ok((verified, moltis_channels::ChannelWebhookDedupeResult::New)) => { - // Dispatch to Slack plugin with verified body. - let result = { - let p = plugin.read().await; - p.ingest_verified_interaction_webhook( - &account_id, - &verified.body, - ) - .await - }; - match result { - Ok(()) => ( - StatusCode::OK, - Json(serde_json::json!({ "ok": true })), - ) - .into_response(), - Err(e) => ( - StatusCode::BAD_REQUEST, - Json(serde_json::json!({ "ok": false, "error": e.to_string() })), - ) - .into_response(), - } + Ok(verified) => match dispatcher.admit( + &gw_state.channel_webhook_dedup, + account_id, + verified.idempotency_key.as_deref(), + super::slack_callbacks::SlackCallbackKind::Interaction, + verified.body, + ) { + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate => slack_json_response( + StatusCode::OK, + serde_json::json!({ "ok": true, "deduplicated": true }), + ), + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => slack_json_response( + StatusCode::OK, + serde_json::json!({ "ok": true }), + ), + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( + StatusCode::SERVICE_UNAVAILABLE, + serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), + ), }, } } }, ), ); - - // Slack slash command webhook -- receives /command payloads. let slack_cmd_plugin = Arc::clone(&slack_webhook_plugin); let state_for_slack_cmd = Arc::clone(&state); + let dispatcher_for_slack_cmd = Arc::clone(&slack_callback_dispatcher); app = app.route( "/api/channels/slack/{account_id}/commands", axum::routing::post( @@ -544,26 +548,23 @@ pub async fn prepare_gateway( body: axum::body::Bytes| { let plugin = Arc::clone(&slack_cmd_plugin); let gw_state = Arc::clone(&state_for_slack_cmd); + let dispatcher = Arc::clone(&dispatcher_for_slack_cmd); async move { - // Get the verifier from the plugin. let verifier = { let p = plugin.read().await; p.channel_webhook_verifier(&account_id) }; let Some(verifier) = verifier else { - return ( + return slack_json_response( StatusCode::NOT_FOUND, - Json(serde_json::json!({ "ok": false, "error": "unknown Slack account" })), - ) - .into_response(); + serde_json::json!({ "ok": false, "error": "unknown Slack account" }), + ); }; - - // Run the middleware pipeline. - match moltis_gateway::channel_webhook_middleware::channel_webhook_gate( + match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( verifier.as_ref(), - &gw_state.channel_webhook_dedup, &gw_state.channel_webhook_rate_limiter, &account_id, + "commands", &headers, &body, ) { @@ -572,34 +573,21 @@ pub async fn prepare_gateway( rejection, ) }, - Ok((_, moltis_channels::ChannelWebhookDedupeResult::Duplicate)) => { - // Slash commands display the response body in - // Slack, so return an empty 200 for deduped - // requests instead of JSON the user would see. - StatusCode::OK.into_response() - }, - Ok((verified, moltis_channels::ChannelWebhookDedupeResult::New)) => { - // Dispatch to Slack plugin with verified body. - let result = { - let p = plugin.read().await; - p.ingest_verified_command_webhook( - &account_id, - &verified.body, - ) - .await - }; - match result { - Ok(response_text) => ( - StatusCode::OK, - response_text, - ) - .into_response(), - Err(e) => ( - StatusCode::BAD_REQUEST, - Json(serde_json::json!({ "ok": false, "error": e.to_string() })), - ) - .into_response(), - } + Ok(verified) => match dispatcher.admit( + &gw_state.channel_webhook_dedup, + account_id, + verified.idempotency_key.as_deref(), + super::slack_callbacks::SlackCallbackKind::Command, + verified.body, + ) { + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate + | moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => { + StatusCode::OK.into_response() + }, + moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( + StatusCode::SERVICE_UNAVAILABLE, + serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), + ), }, } } @@ -607,7 +595,6 @@ pub async fn prepare_gateway( ), ); } - #[cfg(feature = "telephony")] { let telephony_plugin_for_webhook = Arc::clone(&telephony_webhook_plugin); @@ -1440,6 +1427,7 @@ pub async fn prepare_gateway( } let method_count = methods.method_names().len(); + let app = finalize_gateway_app(app, app_state, config.server.http_request_logs); super::runtime::finalize_prepared_gateway(FinalizeGatewayArgs { bind, diff --git a/crates/httpd/src/server/mod.rs b/crates/httpd/src/server/mod.rs index 53e3dde785..61920cae58 100644 --- a/crates/httpd/src/server/mod.rs +++ b/crates/httpd/src/server/mod.rs @@ -59,6 +59,8 @@ mod middleware; mod netbird; mod ngrok; mod runtime; +#[cfg(feature = "slack")] +mod slack_callbacks; mod types; // ── Re-exports ─────────────────────────────────────────────────────────────── diff --git a/crates/httpd/src/server/slack_callbacks.rs b/crates/httpd/src/server/slack_callbacks.rs new file mode 100644 index 0000000000..1b100fb50c --- /dev/null +++ b/crates/httpd/src/server/slack_callbacks.rs @@ -0,0 +1,678 @@ +use std::{ + collections::{HashMap, VecDeque}, + future::Future, + panic::AssertUnwindSafe, + sync::{Arc, Mutex, RwLock, TryLockError}, +}; + +use { + bytes::Bytes, + futures::FutureExt, + moltis_gateway::channel_webhook_dedup::{ChannelWebhookAdmission, ChannelWebhookDedupeStore}, + tokio::{ + sync::{OwnedSemaphorePermit, Semaphore, mpsc}, + task::JoinSet, + }, +}; + +const CALLBACK_QUEUE_CAPACITY: usize = 256; +const CALLBACK_WORKER_LIMIT: usize = 16; +const CALLBACK_ACCOUNT_CAPACITY: usize = CALLBACK_QUEUE_CAPACITY / CALLBACK_WORKER_LIMIT; + +#[derive(Clone, Copy)] +pub(super) enum SlackCallbackKind { + Event, + Interaction, + Command, +} + +impl SlackCallbackKind { + fn endpoint(self) -> &'static str { + match self { + Self::Event => "events", + Self::Interaction => "interactions", + Self::Command => "commands", + } + } +} + +struct SlackCallbackJob { + account_id: String, + body: Bytes, + kind: SlackCallbackKind, + _permit: OwnedSemaphorePermit, +} + +#[derive(Default)] +struct AccountQueue { + jobs: VecDeque, + active: bool, +} + +#[derive(Default)] +struct CallbackQueues { + accounts: HashMap, + ready_accounts: VecDeque, +} + +impl CallbackQueues { + fn account_has_capacity(&self, account_id: &str) -> bool { + self.accounts.get(account_id).is_none_or(|account| { + account.jobs.len() + usize::from(account.active) < CALLBACK_ACCOUNT_CAPACITY + }) + } + + fn try_enqueue(&mut self, job: SlackCallbackJob) -> bool { + let account_id = job.account_id.clone(); + if !self.account_has_capacity(&account_id) { + return false; + } + + let account = self.accounts.entry(account_id.clone()).or_default(); + let became_ready = !account.active && account.jobs.is_empty(); + account.jobs.push_back(job); + if became_ready { + self.ready_accounts.push_back(account_id); + } + true + } + + fn rollback_last(&mut self, account_id: &str) { + let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { + let _ = account.jobs.pop_back(); + !account.active && account.jobs.is_empty() + } else { + false + }; + if remove_account { + self.ready_accounts.retain(|ready| ready != account_id); + self.accounts.remove(account_id); + } + } + + fn take_next(&mut self) -> Option { + while let Some(account_id) = self.ready_accounts.pop_front() { + let Some(account) = self.accounts.get_mut(&account_id) else { + continue; + }; + let Some(job) = account.jobs.pop_front() else { + continue; + }; + account.active = true; + return Some(job); + } + None + } + + fn complete(&mut self, account_id: &str) { + let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { + account.active = false; + if account.jobs.is_empty() { + true + } else { + self.ready_accounts.push_back(account_id.to_string()); + false + } + } else { + false + }; + if remove_account { + self.accounts.remove(account_id); + } + } + + fn is_empty(&self) -> bool { + self.accounts.is_empty() + } +} + +/// Bounded callback queue with an owned supervisor for all processing tasks. +pub(super) struct SlackCallbackDispatcher { + queues: Arc>, + wake_sender: mpsc::Sender<()>, + permits: Arc, + supervisor: tokio::task::JoinHandle<()>, +} + +impl SlackCallbackDispatcher { + pub(super) fn start(plugin: Arc>) -> Arc { + let queues = Arc::new(Mutex::new(CallbackQueues::default())); + let (wake_sender, wake_receiver) = mpsc::channel(1); + let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); + let supervisor = tokio::spawn(run_supervisor(Arc::clone(&queues), wake_receiver, plugin)); + Arc::new(Self { + queues, + wake_sender, + permits, + supervisor, + }) + } + + pub(super) fn admit( + &self, + dedup_store: &RwLock, + account_id: String, + idempotency_key: Option<&str>, + kind: SlackCallbackKind, + body: Bytes, + ) -> ChannelWebhookAdmission { + admit_callback( + &self.queues, + &self.wake_sender, + &self.permits, + dedup_store, + account_id, + idempotency_key, + kind, + body, + ) + } +} + +impl Drop for SlackCallbackDispatcher { + fn drop(&mut self) { + self.supervisor.abort(); + } +} + +fn admit_callback( + queues: &Arc>, + wake_sender: &mpsc::Sender<()>, + permits: &Arc, + dedup_store: &RwLock, + account_id: String, + idempotency_key: Option<&str>, + kind: SlackCallbackKind, + body: Bytes, +) -> ChannelWebhookAdmission { + let endpoint = kind.endpoint(); + let Some(key) = idempotency_key else { + return if try_enqueue_callback(queues, wake_sender, permits, account_id, kind, body) { + ChannelWebhookAdmission::Admitted + } else { + ChannelWebhookAdmission::Rejected + }; + }; + + let job_account_id = account_id.clone(); + dedup_store + .write() + .unwrap_or_else(|error| error.into_inner()) + .admit_scoped("slack", &account_id, endpoint, key, || { + try_enqueue_callback(queues, wake_sender, permits, job_account_id, kind, body) + }) +} + +fn try_enqueue_callback( + queues: &Arc>, + wake_sender: &mpsc::Sender<()>, + permits: &Arc, + account_id: String, + kind: SlackCallbackKind, + body: Bytes, +) -> bool { + let mut queues = match queues.try_lock() { + Ok(queues) => queues, + Err(TryLockError::Poisoned(error)) => error.into_inner(), + Err(TryLockError::WouldBlock) => return false, + }; + if !queues.account_has_capacity(&account_id) { + return false; + } + let Ok(permit) = Arc::clone(permits).try_acquire_owned() else { + return false; + }; + let job = SlackCallbackJob { + account_id: account_id.clone(), + body, + kind, + _permit: permit, + }; + if !queues.try_enqueue(job) { + return false; + } + + match wake_sender.try_send(()) { + Ok(()) | Err(mpsc::error::TrySendError::Full(())) => true, + Err(mpsc::error::TrySendError::Closed(())) => { + queues.rollback_last(&account_id); + false + }, + } +} + +async fn run_supervisor( + queues: Arc>, + wake_receiver: mpsc::Receiver<()>, + plugin: Arc>, +) { + run_fair_workers(queues, wake_receiver, move |job| { + process_callback(Arc::clone(&plugin), job) + }) + .await; +} + +/// Dispatch one callback per ready account in round-robin order. An account is +/// never active on more than one worker, preserving its admission order. +async fn run_fair_workers( + queues: Arc>, + mut wake_receiver: mpsc::Receiver<()>, + process: F, +) where + F: Fn(SlackCallbackJob) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let mut workers = JoinSet::new(); + let mut worker_senders = Vec::with_capacity(CALLBACK_WORKER_LIMIT); + let mut idle_workers = (0..CALLBACK_WORKER_LIMIT).collect::>(); + let (completion_sender, mut completion_receiver) = mpsc::channel(CALLBACK_WORKER_LIMIT); + for worker_index in 0..CALLBACK_WORKER_LIMIT { + let (sender, worker_receiver) = mpsc::channel(1); + worker_senders.push(sender); + workers.spawn(run_serial_worker( + worker_index, + worker_receiver, + completion_sender.clone(), + process.clone(), + )); + } + drop(completion_sender); + + let mut wake_open = true; + loop { + while let Some(worker_index) = idle_workers.pop_front() { + let job = { + queues + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take_next() + }; + let Some(job) = job else { + idle_workers.push_front(worker_index); + break; + }; + if let Err(error) = worker_senders[worker_index].try_send(job) { + tracing::error!(worker_index, %error, "Slack callback worker stopped accepting work"); + return; + } + } + + let queues_empty = queues + .lock() + .unwrap_or_else(|error| error.into_inner()) + .is_empty(); + if !wake_open && queues_empty { + break; + } + + tokio::select! { + wake = wake_receiver.recv(), if wake_open => { + if wake.is_none() { + wake_open = false; + } + }, + completion = completion_receiver.recv() => { + let Some((worker_index, account_id)) = completion else { + tracing::error!("all Slack callback workers stopped unexpectedly"); + return; + }; + queues + .lock() + .unwrap_or_else(|error| error.into_inner()) + .complete(&account_id); + idle_workers.push_back(worker_index); + }, + result = workers.join_next(), if !workers.is_empty() => { + log_worker_result(result); + tracing::error!("Slack callback worker stopped unexpectedly"); + return; + }, + } + } + + drop(worker_senders); + while !workers.is_empty() { + log_worker_result(workers.join_next().await); + } +} + +async fn run_serial_worker( + worker_index: usize, + mut receiver: mpsc::Receiver, + completion_sender: mpsc::Sender<(usize, String)>, + process: F, +) where + F: Fn(SlackCallbackJob) -> Fut, + Fut: Future, +{ + while let Some(job) = receiver.recv().await { + let account_id = job.account_id.clone(); + let process_result = std::panic::catch_unwind(AssertUnwindSafe(|| process(job))); + match process_result { + Ok(future) => { + if AssertUnwindSafe(future).catch_unwind().await.is_err() { + tracing::error!(worker_index, %account_id, "Slack callback worker panicked"); + } + }, + Err(_) => { + tracing::error!(worker_index, %account_id, "Slack callback worker panicked"); + }, + } + if completion_sender + .send((worker_index, account_id)) + .await + .is_err() + { + return; + } + } +} + +fn log_worker_result(result: Option>) { + if let Some(Err(error)) = result { + tracing::error!("Slack callback worker failed: {error}"); + } +} + +async fn process_callback( + plugin: Arc>, + job: SlackCallbackJob, +) { + let plugin = plugin.read().await; + let result = match job.kind { + SlackCallbackKind::Event => plugin + .ingest_verified_webhook(&job.account_id, &job.body) + .await + .map(|_| ()), + SlackCallbackKind::Interaction => { + plugin + .ingest_verified_interaction_webhook(&job.account_id, &job.body) + .await + }, + SlackCallbackKind::Command => plugin + .ingest_verified_command_webhook(&job.account_id, &job.body) + .await + .map(|_| ()), + }; + if let Err(error) = result { + tracing::warn!( + account_id = %job.account_id, + endpoint = job.kind.endpoint(), + "Slack callback processing failed after acknowledgement: {error}" + ); + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn account_job( + permits: &Arc, + account_id: &str, + body: &'static [u8], + ) -> SlackCallbackJob { + SlackCallbackJob { + account_id: account_id.to_string(), + body: Bytes::from_static(body), + kind: SlackCallbackKind::Event, + _permit: Arc::clone(permits).try_acquire_owned().unwrap(), + } + } + + fn admit_without_key( + queues: &Arc>, + wake_sender: &mpsc::Sender<()>, + permits: &Arc, + dedup: &RwLock, + account_id: &str, + ) -> ChannelWebhookAdmission { + admit_callback( + queues, + wake_sender, + permits, + dedup, + account_id.to_string(), + None, + SlackCallbackKind::Event, + Bytes::from_static(b"callback"), + ) + } + + fn prior_worker_index(account_id: &str) -> usize { + use std::hash::{DefaultHasher, Hash, Hasher}; + + let mut hasher = DefaultHasher::new(); + account_id.hash(&mut hasher); + hasher.finish() as usize % CALLBACK_WORKER_LIMIT + } + + #[test] + fn failed_queue_admission_does_not_consume_dedup() { + let queues = Arc::new(Mutex::new(CallbackQueues::default())); + let (closed_sender, closed_receiver) = mpsc::channel(1); + drop(closed_receiver); + let permits = Arc::new(Semaphore::new(2)); + let dedup = RwLock::new(ChannelWebhookDedupeStore::new()); + + assert_eq!( + admit_callback( + &queues, + &closed_sender, + &permits, + &dedup, + "account".to_string(), + Some("event-1"), + SlackCallbackKind::Event, + Bytes::from_static(b"first"), + ), + ChannelWebhookAdmission::Rejected + ); + + let (wake_sender, _wake_receiver) = mpsc::channel(1); + assert_eq!( + admit_callback( + &queues, + &wake_sender, + &permits, + &dedup, + "account".to_string(), + Some("event-1"), + SlackCallbackKind::Event, + Bytes::from_static(b"retry"), + ), + ChannelWebhookAdmission::Admitted + ); + let _remaining_permit = Arc::clone(&permits).try_acquire_owned().unwrap(); + assert_eq!( + admit_callback( + &queues, + &wake_sender, + &permits, + &dedup, + "account".to_string(), + Some("event-1"), + SlackCallbackKind::Event, + Bytes::from_static(b"duplicate"), + ), + ChannelWebhookAdmission::Duplicate + ); + } + + #[test] + fn one_account_saturation_leaves_capacity_until_global_saturation() { + let queues = Arc::new(Mutex::new(CallbackQueues::default())); + let (wake_sender, _wake_receiver) = mpsc::channel(1); + let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); + let dedup = RwLock::new(ChannelWebhookDedupeStore::new()); + + for _ in 0..CALLBACK_ACCOUNT_CAPACITY { + assert_eq!( + admit_without_key(&queues, &wake_sender, &permits, &dedup, "hot-account"), + ChannelWebhookAdmission::Admitted + ); + } + assert_eq!( + admit_without_key(&queues, &wake_sender, &permits, &dedup, "hot-account"), + ChannelWebhookAdmission::Rejected + ); + assert_eq!( + permits.available_permits(), + CALLBACK_QUEUE_CAPACITY - CALLBACK_ACCOUNT_CAPACITY + ); + + assert_eq!( + admit_without_key(&queues, &wake_sender, &permits, &dedup, "cold-account"), + ChannelWebhookAdmission::Admitted + ); + for index in 0..(CALLBACK_QUEUE_CAPACITY - CALLBACK_ACCOUNT_CAPACITY - 1) { + assert_eq!( + admit_without_key( + &queues, + &wake_sender, + &permits, + &dedup, + &format!("other-account-{index}"), + ), + ChannelWebhookAdmission::Admitted + ); + } + assert_eq!(permits.available_permits(), 0); + assert_eq!( + admit_without_key(&queues, &wake_sender, &permits, &dedup, "overflow-account"), + ChannelWebhookAdmission::Rejected + ); + } + + #[tokio::test] + async fn callbacks_for_same_account_are_processed_in_admission_order() { + let queues = Arc::new(Mutex::new(CallbackQueues::default())); + let (wake_sender, wake_receiver) = mpsc::channel(1); + let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); + let first_started = Arc::new(tokio::sync::Notify::new()); + let release_first = Arc::new(tokio::sync::Notify::new()); + let second_started = Arc::new(tokio::sync::Notify::new()); + let processed = Arc::new(tokio::sync::Mutex::new(Vec::new())); + + { + let mut queues = queues.lock().unwrap(); + assert!(queues.try_enqueue(account_job(&permits, "account", b"first"))); + assert!(queues.try_enqueue(account_job(&permits, "account", b"second"))); + } + wake_sender.try_send(()).unwrap(); + + let supervisor = tokio::spawn(run_fair_workers(Arc::clone(&queues), wake_receiver, { + let first_started = Arc::clone(&first_started); + let release_first = Arc::clone(&release_first); + let second_started = Arc::clone(&second_started); + let processed = Arc::clone(&processed); + move |job: SlackCallbackJob| { + let first_started = Arc::clone(&first_started); + let release_first = Arc::clone(&release_first); + let second_started = Arc::clone(&second_started); + let processed = Arc::clone(&processed); + async move { + if job.body == Bytes::from_static(b"first") { + first_started.notify_one(); + release_first.notified().await; + processed.lock().await.push("first"); + } else { + second_started.notify_one(); + processed.lock().await.push("second"); + } + } + } + })); + + drop(wake_sender); + + first_started.notified().await; + assert!( + tokio::time::timeout( + std::time::Duration::from_millis(25), + second_started.notified() + ) + .await + .is_err(), + "the second callback started before the first completed" + ); + release_first.notify_one(); + supervisor.await.unwrap(); + + assert_eq!(*processed.lock().await, ["first", "second"]); + } + + #[tokio::test] + async fn accounts_colliding_under_prior_sharding_run_independently() { + let queues = Arc::new(Mutex::new(CallbackQueues::default())); + let (wake_sender, wake_receiver) = mpsc::channel(1); + let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); + let mut accounts_by_prior_worker = HashMap::new(); + let (hot_account, colliding_account) = (0..=CALLBACK_WORKER_LIMIT) + .find_map(|index| { + let account = format!("colliding-account-{index}"); + accounts_by_prior_worker + .insert(prior_worker_index(&account), account.clone()) + .map(|prior| (prior, account)) + }) + .unwrap(); + let hot_started = Arc::new(tokio::sync::Notify::new()); + let release_hot = Arc::new(tokio::sync::Notify::new()); + let colliding_started = Arc::new(tokio::sync::Notify::new()); + + assert!(queues.lock().unwrap().try_enqueue(account_job( + &permits, + &hot_account, + b"hot-blocking" + ))); + wake_sender.try_send(()).unwrap(); + + let supervisor = tokio::spawn(run_fair_workers(Arc::clone(&queues), wake_receiver, { + let hot_started = Arc::clone(&hot_started); + let release_hot = Arc::clone(&release_hot); + let colliding_started = Arc::clone(&colliding_started); + move |job: SlackCallbackJob| { + let hot_started = Arc::clone(&hot_started); + let release_hot = Arc::clone(&release_hot); + let colliding_started = Arc::clone(&colliding_started); + async move { + if job.body == Bytes::from_static(b"hot-blocking") { + hot_started.notify_one(); + release_hot.notified().await; + } else if job.body == Bytes::from_static(b"colliding") { + colliding_started.notify_one(); + } + } + } + })); + + hot_started.notified().await; + assert!(queues.lock().unwrap().try_enqueue(account_job( + &permits, + &colliding_account, + b"colliding" + ))); + assert!( + !matches!( + wake_sender.try_send(()), + Err(mpsc::error::TrySendError::Closed(())) + ), + "scheduler stopped before colliding account was admitted" + ); + + assert!( + tokio::time::timeout( + std::time::Duration::from_secs(1), + colliding_started.notified(), + ) + .await + .is_ok(), + "accounts colliding under the old shard hash blocked each other" + ); + + release_hot.notify_one(); + drop(wake_sender); + supervisor.await.unwrap(); + } +} diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index 6127b96df5..86c53b81a8 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -38,10 +38,8 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { }; for line in markdown.lines() { - // Fence handling. A closing fence must be a bare run of at least as - // many backticks as the opener; anything else (a longer fence used as - // content, or ```rust) is body text. Treating every ```-prefixed line - // as a toggle silently truncated code blocks that contained one. + // Fence handling. CommonMark allows a bare closing run to be longer + // than its opener; a fence with an info string remains body text. if let Some(ticks) = opening_fence_len(line) { match &code { Some((buf, open_ticks)) => { @@ -124,8 +122,7 @@ fn opening_fence_len(line: &str) -> Option { (ticks >= 3).then_some(ticks) } -/// A fence closes a block only if it is a bare run of at least `open_ticks` -/// backticks with no trailing content. +/// A bare backtick run closes a block when it is at least as long as the opener. fn is_closing_fence(line: &str, open_ticks: usize) -> bool { let trimmed = line.trim(); trimmed.len() >= open_ticks && trimmed.chars().all(|c| c == '`') @@ -259,16 +256,12 @@ mod tests { } #[test] - fn code_block_containing_a_longer_fence_is_not_truncated() { - // A ```` run inside a ``` block is content, not a terminator. - let md = "```\nbefore\n````\nafter\n```"; + fn longer_backtick_run_closes_fence() { + let md = "```\nbefore\n````\nafter"; let blocks = markdown_to_blocks(md).unwrap(); - let rendered: String = blocks - .iter() - .filter_map(|b| b["text"]["text"].as_str()) - .collect(); - assert!(rendered.contains("before"), "opening content lost"); - assert!(rendered.contains("after"), "content after inner fence lost"); + assert_eq!(blocks.len(), 2); + assert_eq!(blocks[0]["text"]["text"], "```\nbefore\n```"); + assert_eq!(blocks[1]["text"]["text"], "after"); } #[test] diff --git a/crates/slack/src/callback_worker.rs b/crates/slack/src/callback_worker.rs new file mode 100644 index 0000000000..d60600657f --- /dev/null +++ b/crates/slack/src/callback_worker.rs @@ -0,0 +1,232 @@ +use std::sync::Arc; + +use { + moltis_channels::plugin::{ChannelEventSink, ChannelReplyTarget}, + slack_morphism::prelude::SlackPushEventCallback, + tracing::warn, +}; + +use crate::{ + outbound::post_response_url, + state::{AccountStateMap, DedupKind, EventDedup}, +}; + +const CALLBACK_QUEUE_CAPACITY: usize = 256; + +pub(crate) enum CallbackJob { + Push { + event: Box, + account_id: String, + accounts: AccountStateMap, + }, + Command { + sink: Arc, + command: String, + reply_to: ChannelReplyTarget, + sender_id: String, + response_url: String, + }, + Interaction { + sink: Arc, + action_id: String, + reply_to: ChannelReplyTarget, + response_url: Option, + }, + ResponseUrl { + response_url: String, + text: String, + }, +} + +#[derive(Clone)] +pub(crate) struct CallbackQueue { + sender: tokio::sync::mpsc::Sender, + cancel: tokio_util::sync::CancellationToken, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CallbackAdmission { + Queued, + Duplicate, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CallbackAdmissionError { + Full, + Canceled, +} + +impl std::fmt::Display for CallbackAdmissionError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Full => formatter.write_str("Slack callback queue is full"), + Self::Canceled => formatter.write_str("Slack callback queue is canceled"), + } + } +} + +impl std::error::Error for CallbackAdmissionError {} + +impl CallbackQueue { + pub(crate) fn start(cancel: tokio_util::sync::CancellationToken) -> Self { + let (sender, receiver) = bounded_channel(CALLBACK_QUEUE_CAPACITY); + tokio::spawn(run_worker(receiver, cancel.clone())); + Self { sender, cancel } + } + + /// Admit work immediately, allowing the Socket Mode callback to ACK in time. + pub(crate) fn try_send(&self, job: CallbackJob) -> Result<(), CallbackAdmissionError> { + if self.cancel.is_cancelled() { + return Err(CallbackAdmissionError::Canceled); + } + match self.sender.try_send(job) { + Ok(()) => Ok(()), + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + Err(CallbackAdmissionError::Full) + }, + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { + Err(CallbackAdmissionError::Canceled) + }, + } + } + + /// Atomically check deduplication and commit it only after queue admission. + pub(crate) fn try_send_deduplicated( + &self, + dedup: &mut EventDedup, + kind: DedupKind, + id: &str, + job: CallbackJob, + ) -> Result { + if dedup.contains(kind, id) { + return Ok(CallbackAdmission::Duplicate); + } + self.try_send(job)?; + let inserted = dedup.insert_new(kind, id); + debug_assert!(inserted, "dedup changed while its lock was held"); + Ok(CallbackAdmission::Queued) + } +} + +fn bounded_channel( + capacity: usize, +) -> (tokio::sync::mpsc::Sender, tokio::sync::mpsc::Receiver) { + tokio::sync::mpsc::channel(capacity) +} + +async fn run_worker( + mut receiver: tokio::sync::mpsc::Receiver, + cancel: tokio_util::sync::CancellationToken, +) { + loop { + let job = tokio::select! { + () = cancel.cancelled() => { + receiver.close(); + while let Some(job) = receiver.recv().await { + process(job).await; + } + break; + }, + job = receiver.recv() => match job { + Some(job) => job, + None => break, + }, + }; + process(job).await; + } +} + +async fn process(job: CallbackJob) { + match job { + CallbackJob::Push { + event, + account_id, + accounts, + } => crate::socket::handle_push_event(*event, &account_id, &accounts).await, + CallbackJob::Command { + sink, + command, + reply_to, + sender_id, + response_url, + } => { + let response = sink + .dispatch_command(&command, reply_to, Some(&sender_id)) + .await + .unwrap_or_else(|error| format!("Error: {error}")); + if let Err(error) = post_response_url(&response_url, &response).await { + warn!("failed to send Slack command response: {error}"); + } + }, + CallbackJob::Interaction { + sink, + action_id, + reply_to, + response_url, + } => { + let response = sink + .dispatch_interaction(&action_id, reply_to) + .await + .unwrap_or_else(|error| format!("Error: {error}")); + if let Some(response_url) = response_url + && let Err(error) = post_response_url(&response_url, &response).await + { + warn!("failed to send Slack interaction response: {error}"); + } + }, + CallbackJob::ResponseUrl { response_url, text } => { + if let Err(error) = post_response_url(&response_url, &text).await { + warn!("failed to send Slack callback response: {error}"); + } + }, + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[tokio::test] + async fn queue_full_and_cancellation_do_not_commit_dedup() { + let cancel = tokio_util::sync::CancellationToken::new(); + let (sender, mut receiver) = bounded_channel(1); + let queue = CallbackQueue { + sender, + cancel: cancel.clone(), + }; + let mut dedup = EventDedup::default(); + let job = |text: &str| CallbackJob::ResponseUrl { + response_url: "https://hooks.slack.com/actions/test".to_string(), + text: text.to_string(), + }; + + assert_eq!( + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("first")), + Ok(CallbackAdmission::Queued) + ); + assert_eq!( + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("duplicate")), + Ok(CallbackAdmission::Duplicate) + ); + assert_eq!( + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("full")), + Err(CallbackAdmissionError::Full) + ); + assert!(!dedup.contains(DedupKind::Event, "retry")); + + assert!(receiver.recv().await.is_some()); + assert_eq!( + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("retry")), + Ok(CallbackAdmission::Queued) + ); + assert!(receiver.recv().await.is_some()); + + cancel.cancel(); + assert_eq!( + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "canceled", job("canceled")), + Err(CallbackAdmissionError::Canceled) + ); + assert!(!dedup.contains(DedupKind::Event, "canceled")); + } +} diff --git a/crates/slack/src/channel_webhook_verifier.rs b/crates/slack/src/channel_webhook_verifier.rs index bddbb511c1..6d6f968590 100644 --- a/crates/slack/src/channel_webhook_verifier.rs +++ b/crates/slack/src/channel_webhook_verifier.rs @@ -31,6 +31,49 @@ impl SlackChannelWebhookVerifier { } } +fn decode_form_value(value: &str) -> Option { + let mut decoded = Vec::with_capacity(value.len()); + let mut bytes = value.bytes(); + while let Some(byte) = bytes.next() { + match byte { + b'%' => { + let pair = [bytes.next()?, bytes.next()?]; + let hex = std::str::from_utf8(&pair).ok()?; + decoded.push(u8::from_str_radix(hex, 16).ok()?); + }, + b'+' => decoded.push(b' '), + other => decoded.push(other), + } + } + String::from_utf8(decoded).ok() +} + +fn form_value(body: &[u8], name: &str) -> Option { + let body = std::str::from_utf8(body).ok()?; + body.split('&').find_map(|pair| { + let (key, value) = pair.split_once('=')?; + (key == name).then(|| decode_form_value(value)).flatten() + }) +} + +fn idempotency_key(body: &[u8]) -> Option { + if let Some(event_id) = serde_json::from_slice::(body) + .ok() + .and_then(|value| value["event_id"].as_str().map(ToOwned::to_owned)) + { + return Some(event_id); + } + + if let Some(trigger_id) = form_value(body, "trigger_id").filter(|value| !value.is_empty()) { + return Some(trigger_id); + } + + form_value(body, "payload") + .and_then(|payload| serde_json::from_str::(&payload).ok()) + .and_then(|value| value["trigger_id"].as_str().map(ToOwned::to_owned)) + .filter(|value| !value.is_empty()) +} + impl ChannelWebhookVerifier for SlackChannelWebhookVerifier { fn verify( &self, @@ -49,6 +92,10 @@ impl ChannelWebhookVerifier for SlackChannelWebhookVerifier { .and_then(|v| v.to_str().ok()) .ok_or_else(|| ChannelWebhookRejection::MissingHeaders("x-slack-signature".into()))?; + let timestamp_epoch = timestamp.parse::().map_err(|_| { + ChannelWebhookRejection::BadSignature("invalid Slack request timestamp".into()) + })?; + if !verify_signature( self.signing_secret.expose_secret(), timestamp, @@ -60,24 +107,17 @@ impl ChannelWebhookVerifier for SlackChannelWebhookVerifier { )); } - // Extract idempotency key from JSON `event_id` field (Events API). - let idempotency_key = serde_json::from_slice::(body) - .ok() - .and_then(|v| v.get("event_id").and_then(|e| e.as_str()).map(String::from)); - - let timestamp_epoch = timestamp.parse::().ok(); - Ok(VerifiedChannelWebhook { - idempotency_key, + idempotency_key: idempotency_key(body), body: Bytes::copy_from_slice(body), - timestamp_epoch, + timestamp_epoch: Some(timestamp_epoch), }) } fn rate_policy(&self) -> moltis_channels::ChannelWebhookRatePolicy { moltis_channels::ChannelWebhookRatePolicy { - max_requests_per_minute: 30, - burst: 10, + max_requests_per_minute: 600, + burst: 200, } } @@ -175,6 +215,18 @@ mod tests { )); } + #[test] + fn malformed_timestamp_rejects_even_with_matching_signature() { + let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); + let body = b"body"; + let headers = make_signed_headers(TEST_SECRET, "not-an-epoch", body); + + assert!(matches!( + verifier.verify(&headers, body), + Err(ChannelWebhookRejection::BadSignature(_)) + )); + } + #[test] fn no_event_id_yields_none_idempotency_key() { let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); @@ -186,6 +238,39 @@ mod tests { assert!(envelope.idempotency_key.is_none()); } + #[test] + fn command_trigger_id_is_used_for_idempotency() { + let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); + let body = b"command=%2Fmoltis&trigger_id=1337.42.command&text=hello+world"; + let headers = make_signed_headers(TEST_SECRET, "1700000000", body); + + let envelope = verifier.verify(&headers, body).unwrap(); + assert_eq!(envelope.idempotency_key.as_deref(), Some("1337.42.command")); + } + + #[test] + fn interaction_payload_trigger_id_is_used_for_idempotency() { + let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); + let body = b"payload=%7B%22type%22%3A%22block_actions%22%2C%22trigger_id%22%3A%221337.42.interaction%22%7D"; + let headers = make_signed_headers(TEST_SECRET, "1700000000", body); + + let envelope = verifier.verify(&headers, body).unwrap(); + assert_eq!( + envelope.idempotency_key.as_deref(), + Some("1337.42.interaction") + ); + } + + #[test] + fn malformed_form_value_does_not_create_idempotency_key() { + let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); + let body = b"trigger_id=%GG"; + let headers = make_signed_headers(TEST_SECRET, "1700000000", body); + + let envelope = verifier.verify(&headers, body).unwrap(); + assert!(envelope.idempotency_key.is_none()); + } + #[test] fn channel_type_is_slack() { let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); @@ -193,11 +278,11 @@ mod tests { } #[test] - fn rate_policy_is_30_per_minute() { + fn rate_policy_has_callback_headroom() { let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); let policy = verifier.rate_policy(); - assert_eq!(policy.max_requests_per_minute, 30); - assert_eq!(policy.burst, 10); + assert_eq!(policy.max_requests_per_minute, 600); + assert_eq!(policy.burst, 200); } // ── Contract tests ────────────────────────────────────────────────────── diff --git a/crates/slack/src/lib.rs b/crates/slack/src/lib.rs index 40453c0f66..905cdd7198 100644 --- a/crates/slack/src/lib.rs +++ b/crates/slack/src/lib.rs @@ -5,15 +5,18 @@ //! policies, and dispatches messages to the chat session. pub mod blocks; +mod callback_worker; pub mod channel_webhook_verifier; pub mod client; pub mod commands; pub mod config; pub mod emoji; pub mod markdown; +mod native_stream; pub mod outbound; pub mod plugin; pub mod socket; +mod socket_reconnect; pub mod state; pub mod webhook; diff --git a/crates/slack/src/native_stream.rs b/crates/slack/src/native_stream.rs new file mode 100644 index 0000000000..43ab953a99 --- /dev/null +++ b/crates/slack/src/native_stream.rs @@ -0,0 +1,535 @@ +use std::time::Duration; + +use { + async_trait::async_trait, + moltis_channels::{Error as ChannelError, Result as ChannelResult}, + tracing::warn, +}; + +use crate::{client::slack_api_method_url, state::StreamRecipient}; + +/// Slack's documented limit for `markdown_text` on native stream methods. +const MAX_MARKDOWN_CHARS: usize = 12_000; + +#[derive(Debug, serde::Deserialize)] +struct NativeStreamResponse { + ok: bool, + channel: Option, + ts: Option, + error: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct NativeStream { + channel: String, + ts: String, +} + +fn native_stream_body(stream: &NativeStream, markdown_text: Option<&str>) -> serde_json::Value { + let mut body = serde_json::json!({ + "channel": stream.channel, + "ts": stream.ts, + }); + if let Some(text) = markdown_text { + body["markdown_text"] = serde_json::json!(text); + } + body +} + +fn start_native_stream_body( + channel: &str, + thread_ts: &str, + markdown_text: &str, + recipient: Option<&StreamRecipient>, +) -> serde_json::Value { + let mut body = serde_json::json!({ + "channel": channel, + "thread_ts": thread_ts, + "markdown_text": markdown_text, + }); + if !channel.starts_with('D') + && let Some(recipient) = recipient + { + body["recipient_user_id"] = serde_json::json!(recipient.user_id); + body["recipient_team_id"] = serde_json::json!(recipient.team_id); + } + body +} + +#[async_trait] +trait NativeStreamApi: Send + Sync { + async fn start( + &self, + channel: &str, + thread_ts: &str, + markdown_text: &str, + recipient: Option<&StreamRecipient>, + ) -> ChannelResult; + + async fn append(&self, stream: &NativeStream, markdown_text: &str) -> ChannelResult<()>; + + async fn stop(&self, stream: &NativeStream, markdown_text: Option<&str>) -> ChannelResult<()>; +} + +pub(crate) struct HttpNativeStreamApi { + http: reqwest::Client, + api_base_url: String, + bot_token: String, +} + +impl HttpNativeStreamApi { + pub(crate) fn new(http: reqwest::Client, api_base_url: String, bot_token: String) -> Self { + Self { + http, + api_base_url, + bot_token, + } + } +} + +#[async_trait] +impl NativeStreamApi for HttpNativeStreamApi { + async fn start( + &self, + channel: &str, + thread_ts: &str, + markdown_text: &str, + recipient: Option<&StreamRecipient>, + ) -> ChannelResult { + let body = start_native_stream_body(channel, thread_ts, markdown_text, recipient); + let response: NativeStreamResponse = self + .http + .post(slack_api_method_url( + &self.api_base_url, + "chat.startStream", + )?) + .bearer_auth(&self.bot_token) + .json(&body) + .send() + .await + .map_err(|error| ChannelError::external("chat.startStream", error))? + .json() + .await + .map_err(|error| ChannelError::external("chat.startStream parse", error))?; + + if !response.ok { + let error = response.error.as_deref().unwrap_or("unknown"); + return Err(ChannelError::unavailable(format!( + "chat.startStream failed: {error}" + ))); + } + + Ok(NativeStream { + channel: response + .channel + .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing channel"))?, + ts: response + .ts + .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing ts"))?, + }) + } + + async fn append(&self, stream: &NativeStream, markdown_text: &str) -> ChannelResult<()> { + let response: serde_json::Value = self + .http + .post(slack_api_method_url( + &self.api_base_url, + "chat.appendStream", + )?) + .bearer_auth(&self.bot_token) + .json(&native_stream_body(stream, Some(markdown_text))) + .send() + .await + .map_err(|error| ChannelError::external("chat.appendStream", error))? + .json() + .await + .map_err(|error| ChannelError::external("chat.appendStream parse", error))?; + check_ok(&response, "chat.appendStream") + } + + async fn stop(&self, stream: &NativeStream, markdown_text: Option<&str>) -> ChannelResult<()> { + let response: serde_json::Value = self + .http + .post(slack_api_method_url(&self.api_base_url, "chat.stopStream")?) + .bearer_auth(&self.bot_token) + .json(&native_stream_body(stream, markdown_text)) + .send() + .await + .map_err(|error| ChannelError::external("chat.stopStream", error))? + .json() + .await + .map_err(|error| ChannelError::external("chat.stopStream parse", error))?; + check_ok(&response, "chat.stopStream") + } +} + +fn check_ok(response: &serde_json::Value, method: &str) -> ChannelResult<()> { + if response.get("ok").and_then(serde_json::Value::as_bool) == Some(true) { + return Ok(()); + } + let error = response + .get("error") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + Err(ChannelError::unavailable(format!( + "{method} failed: {error}" + ))) +} + +pub(crate) async fn send_native_stream( + api: &HttpNativeStreamApi, + channel: &str, + thread_ts: &str, + recipient: Option<&StreamRecipient>, + throttle: Duration, + stream: &mut moltis_channels::plugin::StreamReceiver, +) -> ChannelResult<()> { + send_native_stream_with_api(api, channel, thread_ts, recipient, throttle, stream).await +} + +async fn send_native_stream_with_api( + api: &A, + channel: &str, + thread_ts: &str, + recipient: Option<&StreamRecipient>, + throttle: Duration, + stream: &mut moltis_channels::plugin::StreamReceiver, +) -> ChannelResult<()> { + use moltis_channels::plugin::StreamEvent; + + let mut pending = String::new(); + let mut native_stream = None; + let mut last_append = tokio::time::Instant::now(); + + loop { + match stream.recv().await { + Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { + pending.push_str(&chunk); + if native_stream.is_none() { + let Some(initial) = take_markdown_chunk(&mut pending) else { + continue; + }; + native_stream = Some(api.start(channel, thread_ts, &initial, recipient).await?); + last_append = tokio::time::Instant::now(); + } + if last_append.elapsed() >= throttle { + flush_appends(api, native_stream.as_ref(), &mut pending).await?; + last_append = tokio::time::Instant::now(); + } + }, + Some(StreamEvent::Done) | None => break, + Some(StreamEvent::Error(error)) => { + pending.push_str(&format!("\n\n:warning: {error}")); + break; + }, + } + } + + if native_stream.is_none() { + let Some(initial) = take_markdown_chunk(&mut pending) else { + return Ok(()); + }; + native_stream = Some(api.start(channel, thread_ts, &initial, recipient).await?); + } + let native_stream = native_stream + .as_ref() + .ok_or_else(|| ChannelError::unavailable("native Slack stream was not started"))?; + + while exceeds_markdown_limit(&pending) { + let chunk = take_markdown_chunk(&mut pending).unwrap_or_default(); + append_or_stop(api, native_stream, &chunk).await?; + } + + let final_text = (!pending.is_empty()).then_some(pending.as_str()); + if let Err(error) = api.stop(native_stream, final_text).await { + warn!(channel, "chat.stopStream failed: {error}"); + if let Err(fallback_error) = api.stop(native_stream, None).await { + warn!(channel, "fallback chat.stopStream failed: {fallback_error}"); + } + return Err(error); + } + Ok(()) +} + +async fn flush_appends( + api: &A, + stream: Option<&NativeStream>, + pending: &mut String, +) -> ChannelResult<()> { + let stream = + stream.ok_or_else(|| ChannelError::unavailable("native Slack stream was not started"))?; + while let Some(chunk) = take_markdown_chunk(pending) { + append_or_stop(api, stream, &chunk).await?; + } + Ok(()) +} + +async fn append_or_stop( + api: &A, + stream: &NativeStream, + markdown_text: &str, +) -> ChannelResult<()> { + if let Err(error) = api.append(stream, markdown_text).await { + if let Err(stop_error) = api.stop(stream, None).await { + warn!("fallback chat.stopStream failed after append error: {stop_error}"); + } + return Err(error); + } + Ok(()) +} + +fn exceeds_markdown_limit(text: &str) -> bool { + text.char_indices().nth(MAX_MARKDOWN_CHARS).is_some() +} + +fn take_markdown_chunk(buffer: &mut String) -> Option { + if buffer.is_empty() { + return None; + } + let split_at = buffer + .char_indices() + .nth(MAX_MARKDOWN_CHARS) + .map_or(buffer.len(), |(index, _)| index); + Some(buffer.drain(..split_at).collect()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use { + moltis_channels::plugin::{StreamEvent, StreamReceiver}, + std::sync::Mutex, + }; + + use super::*; + + #[derive(Clone, Debug, PartialEq, Eq)] + enum Call { + Start(String), + Append(String), + Stop(Option), + } + + #[derive(Default)] + struct FakeApi { + calls: Mutex>, + fail_append: bool, + stop_failures: Mutex, + } + + #[async_trait] + impl NativeStreamApi for FakeApi { + async fn start( + &self, + _channel: &str, + _thread_ts: &str, + markdown_text: &str, + _recipient: Option<&StreamRecipient>, + ) -> ChannelResult { + self.calls + .lock() + .unwrap() + .push(Call::Start(markdown_text.to_string())); + Ok(NativeStream { + channel: "C1".into(), + ts: "1.0".into(), + }) + } + + async fn append(&self, _stream: &NativeStream, markdown_text: &str) -> ChannelResult<()> { + self.calls + .lock() + .unwrap() + .push(Call::Append(markdown_text.to_string())); + if self.fail_append { + Err(ChannelError::unavailable("append failed")) + } else { + Ok(()) + } + } + + async fn stop( + &self, + _stream: &NativeStream, + markdown_text: Option<&str>, + ) -> ChannelResult<()> { + self.calls + .lock() + .unwrap() + .push(Call::Stop(markdown_text.map(String::from))); + let mut failures = self.stop_failures.lock().unwrap(); + if *failures == 0 { + return Ok(()); + } + *failures -= 1; + Err(ChannelError::unavailable("stop failed")) + } + } + + async fn stream(events: Vec) -> StreamReceiver { + let (sender, receiver) = tokio::sync::mpsc::channel(events.len().max(1)); + for event in events { + sender.send(event).await.unwrap(); + } + drop(sender); + receiver + } + + #[tokio::test] + async fn append_failure_returns_error_and_stops_stream() { + let api = FakeApi { + fail_append: true, + ..Default::default() + }; + let mut receiver = stream(vec![ + StreamEvent::Delta("first".into()), + StreamEvent::Delta("second".into()), + StreamEvent::Done, + ]) + .await; + + let result = + send_native_stream_with_api(&api, "C1", "1.0", None, Duration::ZERO, &mut receiver) + .await; + + assert!(result.is_err()); + assert_eq!(*api.calls.lock().unwrap(), vec![ + Call::Start("first".into()), + Call::Append("second".into()), + Call::Stop(None), + ]); + } + + #[tokio::test] + async fn native_markdown_is_unchanged_and_every_request_is_unicode_bounded() { + let api = FakeApi::default(); + let markdown = "**bold** [link](https://example.com)"; + let long = "πŸ¦€".repeat(MAX_MARKDOWN_CHARS * 2 + 1); + let mut receiver = stream(vec![ + StreamEvent::Delta(markdown.into()), + StreamEvent::Delta(long.clone()), + StreamEvent::Done, + ]) + .await; + + send_native_stream_with_api(&api, "C1", "1.0", None, Duration::ZERO, &mut receiver) + .await + .unwrap(); + + let calls = api.calls.lock().unwrap(); + assert_eq!(calls.first(), Some(&Call::Start(markdown.into()))); + let delivered = calls + .iter() + .filter_map(|call| match call { + Call::Start(text) | Call::Append(text) => Some(text.as_str()), + Call::Stop(Some(text)) => Some(text.as_str()), + Call::Stop(None) => None, + }) + .collect::(); + assert_eq!(delivered, format!("{markdown}{long}")); + assert!(calls.iter().all(|call| { + match call { + Call::Start(text) | Call::Append(text) => { + text.chars().count() <= MAX_MARKDOWN_CHARS + }, + Call::Stop(text) => text + .as_deref() + .is_none_or(|text| text.chars().count() <= MAX_MARKDOWN_CHARS), + } + })); + } + + #[tokio::test] + async fn final_stop_markdown_is_unicode_bounded() { + let api = FakeApi::default(); + let tail = "Γ©".repeat(MAX_MARKDOWN_CHARS + 1); + let mut receiver = stream(vec![ + StreamEvent::Delta("start".into()), + StreamEvent::Delta(tail.clone()), + StreamEvent::Done, + ]) + .await; + + send_native_stream_with_api( + &api, + "C1", + "1.0", + None, + Duration::from_secs(60), + &mut receiver, + ) + .await + .unwrap(); + + assert_eq!(*api.calls.lock().unwrap(), vec![ + Call::Start("start".into()), + Call::Append("Γ©".repeat(MAX_MARKDOWN_CHARS)), + Call::Stop(Some("Γ©".into())), + ]); + } + + #[tokio::test] + async fn failed_final_stop_retries_without_markdown_and_returns_error() { + let api = FakeApi { + stop_failures: Mutex::new(1), + ..Default::default() + }; + let mut receiver = stream(vec![ + StreamEvent::Delta("start".into()), + StreamEvent::Delta("tail".into()), + StreamEvent::Done, + ]) + .await; + + let result = send_native_stream_with_api( + &api, + "C1", + "1.0", + None, + Duration::from_secs(60), + &mut receiver, + ) + .await; + + assert!(result.is_err()); + assert_eq!(*api.calls.lock().unwrap(), vec![ + Call::Start("start".into()), + Call::Stop(Some("tail".into())), + Call::Stop(None), + ]); + } + + #[test] + fn official_payloads_use_channel_ts_and_markdown_text() { + let recipient = StreamRecipient { + user_id: "U123".into(), + team_id: "T123".into(), + }; + assert_eq!( + start_native_stream_body("C123", "1.0", "**hello**", Some(&recipient)), + serde_json::json!({ + "channel": "C123", + "thread_ts": "1.0", + "markdown_text": "**hello**", + "recipient_user_id": "U123", + "recipient_team_id": "T123", + }) + ); + let stream = NativeStream { + channel: "C123".into(), + ts: "2.0".into(), + }; + assert_eq!( + native_stream_body(&stream, Some("tail")), + serde_json::json!({ + "channel": "C123", + "ts": "2.0", + "markdown_text": "tail", + }) + ); + assert_eq!( + native_stream_body(&stream, None), + serde_json::json!({"channel": "C123", "ts": "2.0"}) + ); + } +} diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 4a660a6aa8..15cbd18685 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -19,10 +19,11 @@ use moltis_channels::{ use moltis_common::types::ReplyPayload; use crate::{ - client::{slack_api_method_url, slack_client_for_base_url}, + client::slack_client_for_base_url, config::StreamMode, markdown::{SLACK_MAX_MESSAGE_LEN, chunk_message, markdown_to_slack}, - state::AccountStateMap, + native_stream::{HttpNativeStreamApi, send_native_stream}, + state::{AccountStateMap, StreamRecipient}, }; /// Minimum chars before the first message is sent during streaming. @@ -39,6 +40,73 @@ fn shared_http_client() -> reqwest::Client { .clone() } +fn validate_response_url(url: &str) -> ChannelResult { + let uri = url.parse::().map_err(|error| { + ChannelError::invalid_input(format!("invalid Slack response_url: {error}")) + })?; + let url = reqwest::Url::parse(url).map_err(|error| { + ChannelError::invalid_input(format!("invalid Slack response_url: {error}")) + })?; + if url.scheme() != "https" { + return Err(ChannelError::invalid_input( + "Slack response_url must use HTTPS", + )); + } + if !url.username().is_empty() || url.password().is_some() { + return Err(ChannelError::invalid_input( + "Slack response_url must not contain credentials", + )); + } + if uri + .authority() + .and_then(http::uri::Authority::port_u16) + .is_some() + { + return Err(ChannelError::invalid_input( + "Slack response_url must not contain a port", + )); + } + if !matches!( + url.host_str(), + Some("hooks.slack.com" | "hooks.slack-gov.com") + ) { + return Err(ChannelError::invalid_input( + "Slack response_url host is not approved", + )); + } + Ok(url) +} + +fn build_response_url_http_client() -> ChannelResult { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(10)) + .build() + .map_err(|error| ChannelError::external("Slack response_url client", error)) +} + +pub(crate) async fn post_response_url(url: &str, text: &str) -> ChannelResult<()> { + let url = validate_response_url(url)?; + let response = build_response_url_http_client()? + .post(url) + .json(&serde_json::json!({ + "response_type": "ephemeral", + "replace_original": false, + "text": text, + })) + .send() + .await + .map_err(|error| ChannelError::external("Slack response_url", error))?; + if !response.status().is_success() { + return Err(ChannelError::unavailable(format!( + "Slack response_url returned HTTP {}", + response.status() + ))); + } + Ok(()) +} + /// Slack outbound message sender. pub struct SlackOutbound { pub(crate) accounts: AccountStateMap, @@ -63,25 +131,16 @@ impl SlackOutbound { Ok((client, token)) } - /// Get the thread_ts for reply threading. - fn get_thread_ts(&self, account_id: &str, to: &str, reply_to: Option<&str>) -> Option { - // If we have an explicit reply_to (message_id), use that as thread_ts. - if let Some(ts) = reply_to { - return Some(ts.to_string()); - } - - // Check if thread_replies is enabled and we have a stored thread_ts. + /// Apply the account's thread reply preference to a normal outbound reply. + /// The exact inbound root remains available separately for context and the + /// native streaming API, which requires a thread identity. + fn get_reply_thread_ts(&self, account_id: &str, reply_to: Option<&str>) -> Option { let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); - let state = accounts.get(account_id)?; - if !state.config.thread_replies { - return None; - } - // Look up by channel_id (any user). - state - .pending_threads - .iter() - .find(|(k, _)| k.starts_with(&format!("{to}:"))) - .map(|(_, ts)| ts.clone()) + accounts + .get(account_id) + .is_some_and(|state| state.config.thread_replies) + .then(|| reply_to.map(String::from)) + .flatten() } /// Whether Block Kit rich rendering is enabled for the given account. @@ -115,7 +174,9 @@ impl SlackOutbound { fn get_native_stream_config( &self, account_id: &str, - ) -> ChannelResult<(String, String, Duration)> { + channel: &str, + thread_ts: &str, + ) -> ChannelResult<(String, String, Duration, Option)> { let accounts = self.accounts.read().unwrap_or_else(|e| e.into_inner()); let state = accounts .get(account_id) @@ -124,6 +185,7 @@ impl SlackOutbound { state.config.bot_token.expose_secret().clone(), state.config.api_base_url.clone(), Duration::from_millis(state.config.edit_throttle_ms), + state.stream_recipient(channel, thread_ts).cloned(), )) } @@ -132,69 +194,13 @@ impl SlackOutbound { &self, account_id: &str, to: &str, - thread_ts: Option<&str>, + thread_ts: &str, stream: &mut StreamReceiver, ) -> ChannelResult<()> { - let (bot_token, api_base_url, throttle) = self.get_native_stream_config(account_id)?; - let http = shared_http_client(); - - let stream_id = - start_native_stream(&http, &api_base_url, &bot_token, to, thread_ts).await?; - - let mut pending = String::new(); - let mut last_append = tokio::time::Instant::now(); - - loop { - match stream.recv().await { - Some(StreamEvent::Delta(chunk) | StreamEvent::ProgressDelta(chunk)) => { - pending.push_str(&chunk); - - // Throttle appends to avoid rate limits. - if last_append.elapsed() >= throttle { - let text = markdown_to_slack(&std::mem::take(&mut pending)); - if !text.is_empty() - && let Err(e) = append_native_stream( - &http, - &api_base_url, - &bot_token, - &stream_id, - &text, - ) - .await - { - debug!(account_id, to, "chat.appendStream failed (will retry): {e}"); - // Put the text back for next attempt. - pending = text; - } - last_append = tokio::time::Instant::now(); - } - }, - Some(StreamEvent::Done) => break, - Some(StreamEvent::Error(e)) => { - pending.push_str(&format!("\n\n:warning: {e}")); - break; - }, - None => break, - } - } - - // Flush any remaining text. A failure here means the tail of the reply - // never reached the user, so it must surface: the caller then treats - // the target as undelivered and falls back to a plain send, and the - // acknowledgment cannot claim success. - if !pending.is_empty() { - let text = markdown_to_slack(&pending); - append_native_stream(&http, &api_base_url, &bot_token, &stream_id, &text) - .await - .inspect_err(|e| warn!(account_id, to, "final chat.appendStream failed: {e}"))?; - } - - // Finalize the stream. A stream left open renders as incomplete. - stop_native_stream(&http, &api_base_url, &bot_token, &stream_id) - .await - .inspect_err(|e| warn!(account_id, to, "chat.stopStream failed: {e}"))?; - - Ok(()) + let (bot_token, api_base_url, throttle, recipient) = + self.get_native_stream_config(account_id, to, thread_ts)?; + let api = HttpNativeStreamApi::new(shared_http_client(), api_base_url, bot_token); + send_native_stream(&api, to, thread_ts, recipient.as_ref(), throttle, stream).await } /// Edit-in-place streaming: post β†’ throttled edits β†’ final update. @@ -537,7 +543,7 @@ impl ChannelOutbound for SlackOutbound { reply_to: Option<&str>, ) -> ChannelResult<()> { let (client, token) = self.get_session(account_id)?; - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let thread_ts = self.get_reply_thread_ts(account_id, reply_to); let slack_text = markdown_to_slack(text); let chunks = chunk_message(&slack_text, SLACK_MAX_MESSAGE_LEN); @@ -598,7 +604,7 @@ impl ChannelOutbound for SlackOutbound { }; let (client, token) = self.get_session(account_id)?; - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let thread_ts = self.get_reply_thread_ts(account_id, reply_to); upload_file( &client, @@ -649,7 +655,7 @@ impl ChannelOutbound for SlackOutbound { reply_to: Option<&str>, ) -> ChannelResult<()> { let (client, token) = self.get_session(account_id)?; - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let thread_ts = self.get_reply_thread_ts(account_id, reply_to); let session = client.open_session(&token); let channel_id: SlackChannelId = to.into(); @@ -746,124 +752,6 @@ impl ChannelOutbound for SlackOutbound { } } -/// Start a native Slack stream via `chat.startStream`. -/// -/// Returns `(stream_id, channel)` on success. -async fn start_native_stream( - http: &reqwest::Client, - api_base_url: &str, - bot_token: &str, - channel: &str, - thread_ts: Option<&str>, -) -> ChannelResult { - let mut body = serde_json::json!({ "channel": channel }); - if let Some(ts) = thread_ts { - body["thread_ts"] = serde_json::json!(ts); - } - - let resp = http - .post(slack_api_method_url(api_base_url, "chat.startStream")?) - .bearer_auth(bot_token) - .json(&body) - .send() - .await - .map_err(|e| ChannelError::external("chat.startStream", e))?; - - let json: serde_json::Value = resp - .json() - .await - .map_err(|e| ChannelError::external("chat.startStream parse", e))?; - - if json.get("ok").and_then(|v| v.as_bool()) != Some(true) { - let err = json - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - return Err(ChannelError::unavailable(format!( - "chat.startStream failed: {err}" - ))); - } - - json.get("stream_id") - .and_then(|v| v.as_str()) - .map(String::from) - .ok_or_else(|| ChannelError::unavailable("chat.startStream: missing stream_id")) -} - -/// Append text to a native Slack stream via `chat.appendStream`. -async fn append_native_stream( - http: &reqwest::Client, - api_base_url: &str, - bot_token: &str, - stream_id: &str, - text: &str, -) -> ChannelResult<()> { - let body = serde_json::json!({ - "stream_id": stream_id, - "text": text, - }); - - let resp = http - .post(slack_api_method_url(api_base_url, "chat.appendStream")?) - .bearer_auth(bot_token) - .json(&body) - .send() - .await - .map_err(|e| ChannelError::external("chat.appendStream", e))?; - - let json: serde_json::Value = resp - .json() - .await - .map_err(|e| ChannelError::external("chat.appendStream parse", e))?; - - if json.get("ok").and_then(|v| v.as_bool()) != Some(true) { - let err = json - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - return Err(ChannelError::unavailable(format!( - "chat.appendStream failed: {err}" - ))); - } - - Ok(()) -} - -/// Finalize a native Slack stream via `chat.stopStream`. -async fn stop_native_stream( - http: &reqwest::Client, - api_base_url: &str, - bot_token: &str, - stream_id: &str, -) -> ChannelResult<()> { - let body = serde_json::json!({ "stream_id": stream_id }); - - let resp = http - .post(slack_api_method_url(api_base_url, "chat.stopStream")?) - .bearer_auth(bot_token) - .json(&body) - .send() - .await - .map_err(|e| ChannelError::external("chat.stopStream", e))?; - - let json: serde_json::Value = resp - .json() - .await - .map_err(|e| ChannelError::external("chat.stopStream parse", e))?; - - if json.get("ok").and_then(|v| v.as_bool()) != Some(true) { - let err = json - .get("error") - .and_then(|v| v.as_str()) - .unwrap_or("unknown"); - return Err(ChannelError::unavailable(format!( - "chat.stopStream failed: {err}" - ))); - } - - Ok(()) -} - #[async_trait] impl ChannelStreamOutbound for SlackOutbound { async fn send_stream( @@ -874,12 +762,18 @@ impl ChannelStreamOutbound for SlackOutbound { mut stream: StreamReceiver, ) -> ChannelResult<()> { let stream_mode = self.get_stream_mode(account_id); - let thread_ts = self.get_thread_ts(account_id, to, reply_to); + let thread_ts = self.get_reply_thread_ts(account_id, reply_to); match stream_mode { - StreamMode::Native => { - self.send_stream_native(account_id, to, thread_ts.as_deref(), &mut stream) - .await + StreamMode::Native => match thread_ts.as_deref() { + Some(thread_ts) => { + self.send_stream_native(account_id, to, thread_ts, &mut stream) + .await + }, + None => { + self.send_stream_edit_in_place(account_id, to, None, &mut stream) + .await + }, }, StreamMode::EditInPlace => { self.send_stream_edit_in_place(account_id, to, thread_ts.as_deref(), &mut stream) @@ -976,25 +870,93 @@ impl ChannelThreadContext for SlackOutbound { mod tests { use super::*; - #[test] - fn get_thread_ts_from_reply_to() { + fn outbound_with_thread_replies(thread_replies: bool) -> SlackOutbound { let accounts = std::sync::Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); - let outbound = SlackOutbound { - accounts: accounts.clone(), + let config = crate::config::SlackAccountConfig { + thread_replies, + ..Default::default() }; - // reply_to takes precedence. - let ts = outbound.get_thread_ts("acct", "C123", Some("1234567.890")); + accounts + .write() + .unwrap() + .insert("acct".to_string(), crate::state::AccountState { + account_id: "acct".to_string(), + config, + message_log: None, + event_sink: None, + cancel: tokio_util::sync::CancellationToken::new(), + bot_user_id: Some("UBOT".to_string()), + stream_recipients: Default::default(), + otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), + }); + SlackOutbound { accounts } + } + + #[test] + fn configured_thread_replies_use_exact_inbound_root() { + let outbound = outbound_with_thread_replies(true); + let ts = outbound.get_reply_thread_ts("acct", Some("1234567.890")); assert_eq!(ts, Some("1234567.890".to_string())); } #[test] - fn get_thread_ts_no_account() { - let accounts = - std::sync::Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); - let outbound = SlackOutbound { accounts }; - let ts = outbound.get_thread_ts("acct", "C123", None); - assert!(ts.is_none()); + fn disabled_thread_replies_send_normal_replies_to_channel() { + let outbound = outbound_with_thread_replies(false); + assert!( + outbound + .get_reply_thread_ts("acct", Some("1234567.890")) + .is_none() + ); + } + + #[test] + fn response_url_accepts_only_slack_https_hook_hosts() { + assert!(validate_response_url("https://hooks.slack.com/commands/T1/123/secret").is_ok()); + assert!( + validate_response_url("https://hooks.slack-gov.com/commands/T1/123/secret").is_ok() + ); + + for url in [ + "http://hooks.slack.com/commands/T1/123/secret", + "https://user@hooks.slack.com/commands/T1/123/secret", + "https://hooks.slack.com:443/commands/T1/123/secret", + "https://hooks.slack.com.evil.example/commands/T1/123/secret", + "https://slack.com/commands/T1/123/secret", + "https://127.0.0.1/commands/T1/123/secret", + ] { + assert!(validate_response_url(url).is_err(), "accepted {url}"); + } + } + + #[tokio::test] + async fn response_url_client_does_not_follow_redirects() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = [0_u8; 2048]; + let _ = socket.read(&mut request).await.unwrap(); + let response = format!( + "HTTP/1.1 302 Found\r\nLocation: http://{address}/redirected\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + ); + socket.write_all(response.as_bytes()).await.unwrap(); + tokio::time::timeout(Duration::from_millis(200), listener.accept()) + .await + .is_ok() + }); + + let response = build_response_url_http_client() + .unwrap() + .post(format!("http://{address}/initial")) + .send() + .await + .unwrap(); + assert!(response.status().is_redirection()); + assert!(!server.await.unwrap(), "redirect target was requested"); } #[test] diff --git a/crates/slack/src/plugin.rs b/crates/slack/src/plugin.rs index 54d93d0540..4e9ffbbc82 100644 --- a/crates/slack/src/plugin.rs +++ b/crates/slack/src/plugin.rs @@ -340,7 +340,7 @@ mod tests { event_sink: None, cancel: CancellationToken::new(), bot_user_id: Some("Ubot".into()), - pending_threads: HashMap::new(), + stream_recipients: Default::default(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(300)), dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), } diff --git a/crates/slack/src/socket.rs b/crates/slack/src/socket.rs index 0254904fc4..1dfa26a72d 100644 --- a/crates/slack/src/socket.rs +++ b/crates/slack/src/socket.rs @@ -6,6 +6,10 @@ use { tracing::{debug, info, warn}, }; +mod callbacks; +#[cfg(test)] +mod tests; + use moltis_channels::{ config_view::ChannelConfigView, gating::{DmPolicy, GroupPolicy, is_allowed}, @@ -21,13 +25,17 @@ use moltis_channels::{ }; use crate::{ + callback_worker::{CallbackAdmissionError, CallbackQueue}, client::validated_slack_client_for_base_url, config::SlackAccountConfig, markdown::strip_mentions, outbound::SlackOutbound, - state::{AccountState, AccountStateMap}, + socket_reconnect::{RECONNECT_INITIAL_BACKOFF, RECONNECT_STABLE_AFTER, backoff_sleep}, + state::{AccountState, AccountStateMap, StreamRecipient}, }; +use callbacks::{command_events_callback, interaction_events_callback, push_events_callback}; + const OTP_CHALLENGE_MSG: &str = "To use this bot, please enter the verification code.\n\nAsk the bot owner for the code; it is visible in the web UI under Channels > Senders.\n\nThe code expires in 5 minutes."; /// State stored in the Socket Mode listener for callback access. @@ -35,6 +43,7 @@ const OTP_CHALLENGE_MSG: &str = "To use this bot, please enter the verification struct ListenerState { account_id: String, accounts: AccountStateMap, + callback_queue: CallbackQueue, } /// Start Socket Mode for a single account. @@ -88,7 +97,7 @@ pub async fn start_socket_mode( event_sink, cancel: cancel.clone(), bot_user_id: Some(bot_user_id), - pending_threads: std::collections::HashMap::new(), + stream_recipients: Default::default(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(otp_cooldown_secs)), dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), }); @@ -120,14 +129,6 @@ pub async fn start_socket_mode( Ok(()) } -/// Initial reconnect backoff after an unexpected socket disconnect. -const RECONNECT_INITIAL_BACKOFF: std::time::Duration = std::time::Duration::from_secs(1); -/// Maximum reconnect backoff. -const RECONNECT_MAX_BACKOFF: std::time::Duration = std::time::Duration::from_secs(30); -/// A connection that stayed up at least this long is considered healthy, so the -/// backoff resets to its initial value on the next disconnect. -const RECONNECT_STABLE_AFTER: std::time::Duration = std::time::Duration::from_secs(60); - /// Run the Socket Mode listener until cancelled, reconnecting on disconnect. /// /// slack-morphism's `serve()` returns when the socket drops; on its own that @@ -143,11 +144,13 @@ async fn run_socket_listener( cancel: tokio_util::sync::CancellationToken, ) -> Result<(), Box> { let mut backoff = RECONNECT_INITIAL_BACKOFF; + let callback_queue = CallbackQueue::start(cancel.clone()); while !cancel.is_cancelled() { let listener_state = ListenerState { account_id: account_id.to_string(), accounts: accounts.clone(), + callback_queue: callback_queue.clone(), }; // Callbacks must be function pointers, so per-account data rides on the @@ -205,52 +208,6 @@ async fn run_socket_listener( Ok(()) } -/// Sleep for the current backoff (with jitter), then double it up to the cap. -/// -/// Returns `false` if cancellation fired during the sleep (caller should stop). -async fn backoff_sleep( - cancel: &tokio_util::sync::CancellationToken, - backoff: &mut std::time::Duration, -) -> bool { - let delay = jittered(*backoff); - tokio::select! { - () = cancel.cancelled() => return false, - () = tokio::time::sleep(delay) => {}, - } - *backoff = next_backoff(*backoff); - true -} - -/// Double the backoff, capped at [`RECONNECT_MAX_BACKOFF`]. -fn next_backoff(current: std::time::Duration) -> std::time::Duration { - (current * 2).min(RECONNECT_MAX_BACKOFF) -} - -/// Apply +/-25% jitter to a backoff duration to avoid reconnect thundering herds. -/// -/// Uses the wall-clock nanosecond fraction as a cheap entropy source (no `rand` -/// dependency); jitter quality is not security-relevant here. -fn jittered(base: std::time::Duration) -> std::time::Duration { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.subsec_nanos()) - .unwrap_or(0); - // Nanoseconds within the current second map to [0, 1), then to a - // symmetric +/-25% factor. Dividing by anything but NANOS_PER_SEC would - // bias the jitter to one side. - base.mul_f64(jitter_factor(nanos)) -} - -/// Map sub-second nanoseconds onto a symmetric +/-25% delay factor. -/// -/// Split out from [`jittered`] so the distribution can be asserted directly -/// rather than sampled from the clock. -fn jitter_factor(nanos: u32) -> f64 { - const NANOS_PER_SEC: f64 = 1_000_000_000.0; - let unit = f64::from(nanos) / NANOS_PER_SEC; - (0.75 + unit * 0.5).clamp(0.75, 1.25) -} - /// Error handler for Socket Mode. fn error_handler( err: Box, @@ -258,63 +215,23 @@ fn error_handler( _states: SlackClientEventsUserState, ) -> HttpStatusCode { warn!("slack socket mode error: {err}"); - HttpStatusCode::OK -} - -/// Push events callback (messages, app_mention, etc.). -async fn push_events_callback( - event: SlackPushEventCallback, - _client: Arc>, - states: SlackClientEventsUserState, -) -> UserCallbackResult<()> { - let guard = states.read().await; - let listener_state = match guard.get_user_state::() { - Some(s) => s.clone(), - None => return Ok(()), - }; - drop(guard); - - // Slack retries an envelope that is not acknowledged quickly. Drop repeats - // so a retry cannot start a second agent turn for one user message. - { - let accts = listener_state - .accounts - .read() - .unwrap_or_else(|e| e.into_inner()); - if let Some(state) = accts.get(&listener_state.account_id) { - let mut dedup = state.dedup.lock().unwrap_or_else(|e| e.into_inner()); - if !dedup.insert_new(event.event_id.as_ref()) { - debug!( - account_id = %listener_state.account_id, - event_id = %event.event_id, - "dropping duplicate slack event (retry)" - ); - return Ok(()); - } - } + if err.downcast_ref::().is_some() { + HttpStatusCode::SERVICE_UNAVAILABLE + } else { + HttpStatusCode::OK } - - // Process off the callback path: the SDK only acknowledges the envelope - // once this callback returns, and handling can involve Web API calls that - // would otherwise risk exceeding Slack's acknowledgment deadline and - // trigger a retry. - tokio::spawn(async move { - handle_push_event(event, listener_state).await; - }); - - Ok(()) } /// Handle a push event body. Runs off the Socket Mode acknowledgment path. -async fn handle_push_event(event: SlackPushEventCallback, listener_state: ListenerState) { +pub(crate) async fn handle_push_event( + event: SlackPushEventCallback, + account_id: &str, + accounts: &AccountStateMap, +) { + let team_id = event.team_id.to_string(); match event.event { SlackEventCallbackBody::Message(msg_event) => { - handle_message_event( - &listener_state.account_id, - msg_event, - &listener_state.accounts, - ) - .await; + handle_message_event(account_id, msg_event, Some(team_id), accounts).await; }, SlackEventCallbackBody::AppMention(mention_event) => { let channel = mention_event.channel.to_string(); @@ -329,39 +246,42 @@ async fn handle_push_event(event: SlackPushEventCallback, listener_state: Listen let message_ts = Some(mention_event.origin.ts.to_string()); handle_inbound( - &listener_state.account_id, + account_id, &channel, &user, text, thread_ts, message_ts, + Some(team_id), None, true, // is_mention - &listener_state.accounts, + accounts, ) .await; }, SlackEventCallbackBody::ReactionAdded(reaction_event) => { handle_reaction_event( - &listener_state.account_id, + account_id, reaction_event.user.as_ref(), reaction_event.reaction.as_ref(), &reaction_event.item, reaction_event.item_user.as_ref().map(|u| u.to_string()), + Some(team_id), true, - &listener_state.accounts, + accounts, ) .await; }, SlackEventCallbackBody::ReactionRemoved(reaction_event) => { handle_reaction_event( - &listener_state.account_id, + account_id, reaction_event.user.as_ref(), reaction_event.reaction.as_ref(), &reaction_event.item, reaction_event.item_user.as_ref().map(|u| u.to_string()), + Some(team_id), false, - &listener_state.accounts, + accounts, ) .await; }, @@ -371,151 +291,11 @@ async fn handle_push_event(event: SlackPushEventCallback, listener_state: Listen } } -/// Command events callback (slash commands). -async fn command_events_callback( - event: SlackCommandEvent, - _client: Arc>, - states: SlackClientEventsUserState, -) -> UserCallbackResult { - let guard = states.read().await; - let listener_state = match guard.get_user_state::() { - Some(s) => s.clone(), - None => { - return Ok(SlackCommandEventResponse::new( - SlackMessageContent::new().with_text("Not configured".to_string()), - )); - }, - }; - drop(guard); - - let account_id = &listener_state.account_id; - let command_text = event.command.to_string(); - let text = event.text.unwrap_or_default(); - let full_command = format!("{command_text} {text}").trim().to_string(); - let sender_id = event.user_id.to_string(); - - let event_sink = { - let accts = listener_state - .accounts - .read() - .unwrap_or_else(|e| e.into_inner()); - accts.get(account_id).and_then(|s| s.event_sink.clone()) - }; - - if let Some(sink) = event_sink { - let reply_to = ChannelReplyTarget { - ack_message_id: None, - channel_type: ChannelType::Slack, - account_id: account_id.to_string(), - chat_id: event.channel_id.to_string(), - message_id: None, - thread_id: None, - }; - match sink - .dispatch_command(&full_command, reply_to, Some(&sender_id)) - .await - { - Ok(response_text) => Ok(SlackCommandEventResponse::new( - SlackMessageContent::new().with_text(response_text), - )), - Err(e) => Ok(SlackCommandEventResponse::new( - SlackMessageContent::new().with_text(format!("Error: {e}")), - )), - } - } else { - Ok(SlackCommandEventResponse::new( - SlackMessageContent::new().with_text("Channel not configured".to_string()), - )) - } -} - -/// Interaction events callback (block actions / button clicks). -async fn interaction_events_callback( - event: SlackInteractionEvent, - _client: Arc>, - states: SlackClientEventsUserState, -) -> UserCallbackResult<()> { - let guard = states.read().await; - let listener_state = match guard.get_user_state::() { - Some(s) => s.clone(), - None => return Ok(()), - }; - drop(guard); - - // Interactions are retried like events, and a repeat would run the button's - // action twice. Dedup on the trigger id, which is unique per interaction. - if let SlackInteractionEvent::BlockActions(ref ba) = event { - let trigger = ba.trigger_id.as_ref(); - let accts = listener_state - .accounts - .read() - .unwrap_or_else(|e| e.into_inner()); - if let Some(state) = accts.get(&listener_state.account_id) { - let mut dedup = state.dedup.lock().unwrap_or_else(|e| e.into_inner()); - if !dedup.insert_new(trigger) { - debug!( - account_id = %listener_state.account_id, - "dropping duplicate slack interaction (retry)" - ); - return Ok(()); - } - } - } - - // Extract the action_id from block_actions interaction type. - let (action_id, channel_id) = match &event { - SlackInteractionEvent::BlockActions(ba) => { - let action = ba.actions.as_ref().and_then(|a| a.first()); - let channel = ba.channel.as_ref().map(|c| c.id.to_string()); - match (action, channel) { - (Some(act), Some(ch)) => (act.action_id.to_string(), ch), - _ => { - debug!("block_actions missing action or channel"); - return Ok(()); - }, - } - }, - _ => { - debug!("unhandled interaction event type"); - return Ok(()); - }, - }; - - let account_id = &listener_state.account_id; - let event_sink = { - let accts = listener_state - .accounts - .read() - .unwrap_or_else(|e| e.into_inner()); - accts.get(account_id).and_then(|s| s.event_sink.clone()) - }; - - if let Some(sink) = event_sink { - let reply_to = ChannelReplyTarget { - ack_message_id: None, - channel_type: ChannelType::Slack, - account_id: account_id.to_string(), - chat_id: channel_id, - message_id: None, - thread_id: None, - }; - match sink.dispatch_interaction(&action_id, reply_to).await { - Ok(_response) => { - // Response already sent by the gateway. - }, - Err(e) => { - debug!(account_id, action_id, "interaction dispatch failed: {e}"); - }, - } - } - - Ok(()) -} - /// Handle a Slack message event. pub(crate) async fn handle_message_event( account_id: &str, event: SlackMessageEvent, + team_id: Option, accounts: &AccountStateMap, ) { // Skip message subtypes (edits, deletes, bot messages, etc.). @@ -553,9 +333,7 @@ pub(crate) async fn handle_message_event( .unwrap_or(""); let thread_ts = event.origin.thread_ts.as_ref().map(|ts| ts.to_string()); - // The exact ts of this inbound message (for acknowledgment reactions). let message_ts = event.origin.ts.to_string(); - // Use thread_ts if available, otherwise use the message ts for threading. let reply_thread = thread_ts.or_else(|| Some(message_ts.clone())); // Detect if this is a mention. @@ -574,6 +352,7 @@ pub(crate) async fn handle_message_event( text, reply_thread, Some(message_ts), + team_id, event.sender.username.clone(), is_mention, accounts, @@ -592,6 +371,7 @@ pub(crate) async fn handle_inbound( text: &str, thread_ts: Option, message_ts: Option, + team_id: Option, username: Option, is_mention: bool, accounts: &AccountStateMap, @@ -609,11 +389,7 @@ pub(crate) async fn handle_inbound( } }; - // Determine if this is a DM or channel message. - // Slack DM channel IDs start with 'D'. let is_dm = channel_id.starts_with('D'); - - // Access control check. let access_granted = check_access( is_dm, user_id, @@ -711,12 +487,15 @@ pub(crate) async fn handle_inbound( return; } - // Store thread_ts for reply threading. - if let Some(ts) = &thread_ts { - let thread_key = format!("{channel_id}:{user_id}"); + let thread_root = thread_ts.or_else(|| message_ts.clone()); + + if let (Some(thread_root), Some(team_id)) = (&thread_root, team_id) { let mut accts = accounts.write().unwrap_or_else(|e| e.into_inner()); if let Some(state) = accts.get_mut(account_id) { - state.pending_threads.insert(thread_key, ts.clone()); + state.note_stream_recipient(channel_id, thread_root, StreamRecipient { + user_id: user_id.to_string(), + team_id, + }); } } @@ -732,7 +511,7 @@ pub(crate) async fn handle_inbound( channel_type: ChannelType::Slack, account_id: account_id.to_string(), chat_id: channel_id.to_string(), - message_id: thread_ts, + message_id: thread_root, thread_id: None, }; @@ -761,6 +540,27 @@ pub(crate) async fn handle_inbound( } } +pub(crate) fn account_access_allowed( + accounts: &AccountStateMap, + account_id: &str, + user_id: &str, + channel_id: &str, +) -> bool { + let accounts = accounts.read().unwrap_or_else(|error| error.into_inner()); + let Some(state) = accounts.get(account_id) else { + return false; + }; + check_access( + channel_id.starts_with('D'), + user_id, + channel_id, + &state.config.dm_policy, + &state.config.group_policy, + &state.config.allowlist, + &state.config.channel_allowlist, + ) +} + /// Handle a reaction_added or reaction_removed event. #[allow(clippy::too_many_arguments)] pub(crate) async fn handle_reaction_event( @@ -769,6 +569,7 @@ pub(crate) async fn handle_reaction_event( emoji: &str, item: &SlackReactionsItem, item_user: Option, + team_id: Option, added: bool, accounts: &AccountStateMap, ) { @@ -784,15 +585,13 @@ pub(crate) async fn handle_reaction_event( }, _ => return, }; - dispatch_reaction( - account_id, user_id, emoji, channel_id, message_ts, item_user, added, accounts, + account_id, user_id, emoji, channel_id, message_ts, item_user, team_id, added, accounts, ) .await; } /// Core reaction handling shared by Socket Mode and the Events API. -/// /// Always emits a [`ChannelEvent::ReactionChange`] for observers; additionally /// routes the reaction into the agent as a synthetic message when /// `reaction_triggers` is enabled and the reaction is eligible. @@ -804,6 +603,7 @@ pub(crate) async fn dispatch_reaction( channel_id: String, message_ts: String, item_user: Option, + team_id: Option, added: bool, accounts: &AccountStateMap, ) { @@ -886,7 +686,15 @@ pub(crate) async fn dispatch_reaction( ); return; } - + if let Some(team_id) = team_id { + let mut accts = accounts.write().unwrap_or_else(|e| e.into_inner()); + if let Some(state) = accts.get_mut(account_id) { + state.note_stream_recipient(&channel_id, &message_ts, StreamRecipient { + user_id: user_id.to_string(), + team_id, + }); + } + } // Thread the synthetic message under the reacted message so the agent sees // the original content as thread context (dispatch fetches thread history). let reply_to = ChannelReplyTarget { @@ -1173,249 +981,3 @@ async fn send_otp_status( ); } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn backoff_doubles_and_caps() { - use std::time::Duration; - assert_eq!(next_backoff(Duration::from_secs(1)), Duration::from_secs(2)); - assert_eq!(next_backoff(Duration::from_secs(2)), Duration::from_secs(4)); - // Caps at RECONNECT_MAX_BACKOFF (30s). - assert_eq!(next_backoff(Duration::from_secs(20)), RECONNECT_MAX_BACKOFF); - assert_eq!(next_backoff(RECONNECT_MAX_BACKOFF), RECONNECT_MAX_BACKOFF); - } - - #[test] - fn jitter_stays_within_bounds() { - use std::time::Duration; - let base = Duration::from_secs(10); - for _ in 0..200 { - let j = jittered(base); - // Exactly the documented +/-25%. Looser bounds would also accept a - // one-sided distribution, which is the bug this guards against. - assert!(j >= base.mul_f64(0.75), "jitter {j:?} below lower bound"); - assert!(j <= base.mul_f64(1.25), "jitter {j:?} above upper bound"); - } - } - - #[test] - fn jitter_is_two_sided() { - // The factor must be able to land on both sides of the base delay; - // dividing nanoseconds by the wrong constant silently made it - // always-negative, which no range assertion alone would catch. - assert!(jitter_factor(0) < 1.0, "minimum should shorten the delay"); - assert!( - jitter_factor(999_999_999) > 1.0, - "maximum should lengthen the delay" - ); - } - - #[test] - fn ack_target_dm_is_acknowledged() { - assert_eq!( - ack_reaction_target(true, true, false, Some("111.222".into())), - Some("111.222".into()) - ); - } - - #[test] - fn ack_target_mention_is_acknowledged() { - assert_eq!( - ack_reaction_target(true, false, true, Some("111.222".into())), - Some("111.222".into()) - ); - } - - #[test] - fn ack_target_unaddressed_channel_message_is_not_acknowledged() { - assert_eq!( - ack_reaction_target(true, false, false, Some("111.222".into())), - None - ); - } - - #[test] - fn ack_target_disabled_config_never_acknowledges() { - assert_eq!( - ack_reaction_target(false, true, true, Some("111.222".into())), - None - ); - } - - #[test] - fn reaction_trigger_requires_enabled_added_and_not_self() { - // Disabled β†’ never. - assert!(!reaction_should_trigger(false, true, false, true, "x", &[])); - // Removal β†’ never. - assert!(!reaction_should_trigger(true, false, false, true, "x", &[])); - // Bot's own reaction β†’ never. - assert!(!reaction_should_trigger(true, true, true, true, "eyes", &[])); - // Enabled add by another user with no allowlist β†’ trigger. - assert!(reaction_should_trigger(true, true, false, true, "x", &[])); - } - - #[test] - fn reaction_trigger_respects_emoji_allowlist() { - let allow = vec!["white_check_mark".to_string()]; - assert!(reaction_should_trigger( - true, - true, - false, - true, - "white_check_mark", - &allow - )); - assert!(!reaction_should_trigger( - true, true, false, true, "x", &allow - )); - } - - #[test] - fn dm_open_allows_anyone() { - assert!(check_access( - true, - "U123", - "D456", - &DmPolicy::Open, - &GroupPolicy::Open, - &[], - &[], - )); - } - - #[test] - fn dm_allowlist_requires_user() { - assert!(!check_access( - true, - "U999", - "D456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &["U123".to_string()], - &[], - )); - assert!(check_access( - true, - "U123", - "D456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &["U123".to_string()], - &[], - )); - } - - #[test] - fn empty_dm_allowlist_denies_all() { - assert!(!check_access( - true, - "U123", - "D456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &[], - &[], - )); - } - - #[test] - fn dm_disabled_denies_all() { - assert!(!check_access( - true, - "U123", - "D456", - &DmPolicy::Disabled, - &GroupPolicy::Open, - &[], - &[], - )); - } - - #[test] - fn channel_open_allows_any() { - assert!(check_access( - false, - "U123", - "C456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &[], - &[], - )); - } - - #[test] - fn channel_allowlist_requires_channel() { - assert!(!check_access( - false, - "U123", - "C999", - &DmPolicy::Allowlist, - &GroupPolicy::Allowlist, - &[], - &["C456".to_string()], - )); - assert!(check_access( - false, - "U123", - "C456", - &DmPolicy::Allowlist, - &GroupPolicy::Allowlist, - &[], - &["C456".to_string()], - )); - } - - #[test] - fn empty_channel_allowlist_denies_all() { - assert!(!check_access( - false, - "U123", - "C456", - &DmPolicy::Allowlist, - &GroupPolicy::Allowlist, - &[], - &[], - )); - } - - #[test] - fn removing_last_allowlist_entry_denies_access() { - let mut allowlist = vec!["U123".to_string()]; - assert!(check_access( - true, - "U123", - "D456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &allowlist, - &[], - )); - allowlist.clear(); - assert!(!check_access( - true, - "U123", - "D456", - &DmPolicy::Allowlist, - &GroupPolicy::Open, - &allowlist, - &[], - )); - } - - #[test] - fn channel_disabled_denies_all() { - assert!(!check_access( - false, - "U123", - "C456", - &DmPolicy::Allowlist, - &GroupPolicy::Disabled, - &[], - &[], - )); - } -} diff --git a/crates/slack/src/socket/callbacks.rs b/crates/slack/src/socket/callbacks.rs new file mode 100644 index 0000000000..e7f61ec75f --- /dev/null +++ b/crates/slack/src/socket/callbacks.rs @@ -0,0 +1,291 @@ +use std::sync::Arc; + +use { + slack_morphism::prelude::*, + tracing::{debug, warn}, +}; + +use moltis_channels::plugin::{ChannelReplyTarget, ChannelType}; + +use crate::{ + callback_worker::{CallbackAdmission, CallbackAdmissionError, CallbackJob}, + state::DedupKind, +}; + +use super::{ListenerState, account_access_allowed}; + +/// Push events callback (messages, app_mention, etc.). +pub(super) async fn push_events_callback( + event: SlackPushEventCallback, + _client: Arc>, + states: SlackClientEventsUserState, +) -> UserCallbackResult<()> { + let guard = states.read().await; + let listener_state = match guard.get_user_state::() { + Some(s) => s.clone(), + None => return Ok(()), + }; + drop(guard); + + let event_id = event.event_id.to_string(); + match admit_callback( + &listener_state, + DedupKind::Event, + &event_id, + CallbackJob::Push { + event: Box::new(event), + account_id: listener_state.account_id.clone(), + accounts: listener_state.accounts.clone(), + }, + ) { + Ok(CallbackAdmission::Queued) => {}, + Ok(CallbackAdmission::Duplicate) => { + debug!( + account_id = %listener_state.account_id, + event_id, + "dropping duplicate slack event (retry)" + ); + }, + Err(error) => { + warn!( + account_id = %listener_state.account_id, + event_id, + "slack push event was not admitted: {error}" + ); + return Err(Box::new(error)); + }, + } + + Ok(()) +} + +/// Command events callback (slash commands). +pub(super) async fn command_events_callback( + event: SlackCommandEvent, + _client: Arc>, + states: SlackClientEventsUserState, +) -> UserCallbackResult { + let guard = states.read().await; + let listener_state = match guard.get_user_state::() { + Some(s) => s.clone(), + None => { + return Ok(SlackCommandEventResponse::new( + SlackMessageContent::new().with_text("Not configured".to_string()), + )); + }, + }; + drop(guard); + + let account_id = &listener_state.account_id; + let command_text = event.command.to_string(); + let text = event.text.unwrap_or_default(); + let full_command = format!("{command_text} {text}").trim().to_string(); + let sender_id = event.user_id.to_string(); + let channel_id = event.channel_id.to_string(); + let trigger_id = format!("command:{}", event.trigger_id); + + if !account_access_allowed( + &listener_state.accounts, + account_id, + &sender_id, + &channel_id, + ) { + return Ok(SlackCommandEventResponse::new( + SlackMessageContent::new().with_text("Access denied.".to_string()), + )); + } + + let event_sink = { + let accts = listener_state + .accounts + .read() + .unwrap_or_else(|e| e.into_inner()); + accts.get(account_id).and_then(|s| s.event_sink.clone()) + }; + + if let Some(sink) = event_sink { + let reply_to = ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Slack, + account_id: account_id.to_string(), + chat_id: channel_id, + message_id: None, + thread_id: None, + }; + let response_url = event.response_url.0.as_str().to_string(); + match admit_callback( + &listener_state, + DedupKind::Command, + &trigger_id, + CallbackJob::Command { + sink, + command: full_command, + reply_to, + sender_id, + response_url, + }, + ) { + Ok(CallbackAdmission::Queued) => Ok(SlackCommandEventResponse::new( + SlackMessageContent::new().with_text("Command accepted.".to_string()), + )), + Ok(CallbackAdmission::Duplicate) => Ok(SlackCommandEventResponse::new( + SlackMessageContent::new().with_text("Command already accepted.".to_string()), + )), + Err(error) => { + warn!( + account_id, + trigger_id, "slack command was not admitted: {error}" + ); + Err(Box::new(error)) + }, + } + } else { + Ok(SlackCommandEventResponse::new( + SlackMessageContent::new().with_text("Channel not configured".to_string()), + )) + } +} + +/// Interaction events callback (block actions / button clicks). +pub(super) async fn interaction_events_callback( + event: SlackInteractionEvent, + _client: Arc>, + states: SlackClientEventsUserState, +) -> UserCallbackResult<()> { + let guard = states.read().await; + let listener_state = match guard.get_user_state::() { + Some(s) => s.clone(), + None => return Ok(()), + }; + drop(guard); + + let (action_id, channel_id, user_id, thread_root, response_url, trigger_id) = match &event { + SlackInteractionEvent::BlockActions(ba) => { + let action = ba.actions.as_ref().and_then(|a| a.first()); + let channel = ba.channel.as_ref().map(|c| c.id.to_string()); + let user = ba.user.as_ref().map(|user| user.id.to_string()); + let thread_root = ba.message.as_ref().map(|message| { + message + .origin + .thread_ts + .as_ref() + .unwrap_or(&message.origin.ts) + .to_string() + }); + match (action, channel, user) { + (Some(act), Some(ch), Some(user)) => ( + act.action_id.to_string(), + ch, + user, + thread_root, + ba.response_url + .as_ref() + .map(|url| url.0.as_str().to_string()), + format!("interaction:{}", ba.trigger_id), + ), + _ => { + debug!("block_actions missing action, channel, or user"); + return Ok(()); + }, + } + }, + _ => { + debug!("unhandled interaction event type"); + return Ok(()); + }, + }; + + let account_id = &listener_state.account_id; + if !account_access_allowed(&listener_state.accounts, account_id, &user_id, &channel_id) { + if let Some(response_url) = response_url { + match admit_callback( + &listener_state, + DedupKind::Interaction, + &trigger_id, + CallbackJob::ResponseUrl { + response_url, + text: "Access denied.".to_string(), + }, + ) { + Ok(CallbackAdmission::Queued | CallbackAdmission::Duplicate) => {}, + Err(error) => { + warn!( + account_id, + trigger_id, "slack denial response was not admitted: {error}" + ); + return Err(Box::new(error)); + }, + } + } + return Ok(()); + } + + let event_sink = { + let accts = listener_state + .accounts + .read() + .unwrap_or_else(|e| e.into_inner()); + accts.get(account_id).and_then(|s| s.event_sink.clone()) + }; + + if let Some(sink) = event_sink { + let reply_to = ChannelReplyTarget { + ack_message_id: None, + channel_type: ChannelType::Slack, + account_id: account_id.to_string(), + chat_id: channel_id, + message_id: thread_root, + thread_id: None, + }; + match admit_callback( + &listener_state, + DedupKind::Interaction, + &trigger_id, + CallbackJob::Interaction { + sink, + action_id, + reply_to, + response_url, + }, + ) { + Ok(CallbackAdmission::Queued) => {}, + Ok(CallbackAdmission::Duplicate) => { + debug!( + account_id, + trigger_id, "dropping duplicate slack interaction (retry)" + ); + }, + Err(error) => { + warn!( + account_id, + trigger_id, "slack interaction was not admitted: {error}" + ); + return Err(Box::new(error)); + }, + } + } + + Ok(()) +} + +fn admit_callback( + listener_state: &ListenerState, + kind: DedupKind, + id: &str, + job: CallbackJob, +) -> Result { + let accounts = listener_state + .accounts + .read() + .unwrap_or_else(|error| error.into_inner()); + let state = accounts + .get(&listener_state.account_id) + .ok_or(CallbackAdmissionError::Canceled)?; + let mut dedup = state + .dedup + .lock() + .unwrap_or_else(|error| error.into_inner()); + listener_state + .callback_queue + .try_send_deduplicated(&mut dedup, kind, id, job) +} diff --git a/crates/slack/src/socket/tests.rs b/crates/slack/src/socket/tests.rs new file mode 100644 index 0000000000..8b402e6656 --- /dev/null +++ b/crates/slack/src/socket/tests.rs @@ -0,0 +1,245 @@ +use { + super::*, + crate::socket_reconnect::{RECONNECT_MAX_BACKOFF, jitter_factor, jittered, next_backoff}, +}; + +#[test] +fn backoff_doubles_and_caps() { + use std::time::Duration; + assert_eq!(next_backoff(Duration::from_secs(1)), Duration::from_secs(2)); + assert_eq!(next_backoff(Duration::from_secs(2)), Duration::from_secs(4)); + // Caps at RECONNECT_MAX_BACKOFF (30s). + assert_eq!(next_backoff(Duration::from_secs(20)), RECONNECT_MAX_BACKOFF); + assert_eq!(next_backoff(RECONNECT_MAX_BACKOFF), RECONNECT_MAX_BACKOFF); +} + +#[test] +fn jitter_stays_within_bounds() { + use std::time::Duration; + let base = Duration::from_secs(10); + for _ in 0..200 { + let j = jittered(base); + // Exactly the documented +/-25%. Looser bounds would also accept a + // one-sided distribution, which is the bug this guards against. + assert!(j >= base.mul_f64(0.75), "jitter {j:?} below lower bound"); + assert!(j <= base.mul_f64(1.25), "jitter {j:?} above upper bound"); + } +} + +#[test] +fn jitter_is_two_sided() { + // The factor must be able to land on both sides of the base delay; + // dividing nanoseconds by the wrong constant silently made it + // always-negative, which no range assertion alone would catch. + assert!(jitter_factor(0) < 1.0, "minimum should shorten the delay"); + assert!( + jitter_factor(999_999_999) > 1.0, + "maximum should lengthen the delay" + ); +} + +#[test] +fn ack_target_dm_is_acknowledged() { + assert_eq!( + ack_reaction_target(true, true, false, Some("111.222".into())), + Some("111.222".into()) + ); +} + +#[test] +fn ack_target_mention_is_acknowledged() { + assert_eq!( + ack_reaction_target(true, false, true, Some("111.222".into())), + Some("111.222".into()) + ); +} + +#[test] +fn ack_target_unaddressed_channel_message_is_not_acknowledged() { + assert_eq!( + ack_reaction_target(true, false, false, Some("111.222".into())), + None + ); +} + +#[test] +fn ack_target_disabled_config_never_acknowledges() { + assert_eq!( + ack_reaction_target(false, true, true, Some("111.222".into())), + None + ); +} + +#[test] +fn reaction_trigger_requires_enabled_added_and_not_self() { + // Disabled β†’ never. + assert!(!reaction_should_trigger(false, true, false, true, "x", &[])); + // Removal β†’ never. + assert!(!reaction_should_trigger(true, false, false, true, "x", &[])); + // Bot's own reaction β†’ never. + assert!(!reaction_should_trigger(true, true, true, true, "eyes", &[])); + // Enabled add by another user with no allowlist β†’ trigger. + assert!(reaction_should_trigger(true, true, false, true, "x", &[])); +} + +#[test] +fn reaction_trigger_respects_emoji_allowlist() { + let allow = vec!["white_check_mark".to_string()]; + assert!(reaction_should_trigger( + true, + true, + false, + true, + "white_check_mark", + &allow + )); + assert!(!reaction_should_trigger( + true, true, false, true, "x", &allow + )); +} + +#[test] +fn dm_open_allows_anyone() { + assert!(check_access( + true, + "U123", + "D456", + &DmPolicy::Open, + &GroupPolicy::Open, + &[], + &[], + )); +} + +#[test] +fn dm_allowlist_requires_user() { + assert!(!check_access( + true, + "U999", + "D456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &["U123".to_string()], + &[], + )); + assert!(check_access( + true, + "U123", + "D456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &["U123".to_string()], + &[], + )); +} + +#[test] +fn empty_dm_allowlist_denies_all() { + assert!(!check_access( + true, + "U123", + "D456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &[], + &[], + )); +} + +#[test] +fn dm_disabled_denies_all() { + assert!(!check_access( + true, + "U123", + "D456", + &DmPolicy::Disabled, + &GroupPolicy::Open, + &[], + &[], + )); +} + +#[test] +fn channel_open_allows_any() { + assert!(check_access( + false, + "U123", + "C456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &[], + &[], + )); +} + +#[test] +fn channel_allowlist_requires_channel() { + assert!(!check_access( + false, + "U123", + "C999", + &DmPolicy::Allowlist, + &GroupPolicy::Allowlist, + &[], + &["C456".to_string()], + )); + assert!(check_access( + false, + "U123", + "C456", + &DmPolicy::Allowlist, + &GroupPolicy::Allowlist, + &[], + &["C456".to_string()], + )); +} + +#[test] +fn empty_channel_allowlist_denies_all() { + assert!(!check_access( + false, + "U123", + "C456", + &DmPolicy::Allowlist, + &GroupPolicy::Allowlist, + &[], + &[], + )); +} + +#[test] +fn removing_last_allowlist_entry_denies_access() { + let mut allowlist = vec!["U123".to_string()]; + assert!(check_access( + true, + "U123", + "D456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &allowlist, + &[], + )); + allowlist.clear(); + assert!(!check_access( + true, + "U123", + "D456", + &DmPolicy::Allowlist, + &GroupPolicy::Open, + &allowlist, + &[], + )); +} + +#[test] +fn channel_disabled_denies_all() { + assert!(!check_access( + false, + "U123", + "C456", + &DmPolicy::Allowlist, + &GroupPolicy::Disabled, + &[], + &[], + )); +} diff --git a/crates/slack/src/socket_reconnect.rs b/crates/slack/src/socket_reconnect.rs new file mode 100644 index 0000000000..b012a8fa01 --- /dev/null +++ b/crates/slack/src/socket_reconnect.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +pub(crate) const RECONNECT_INITIAL_BACKOFF: Duration = Duration::from_secs(1); +pub(crate) const RECONNECT_MAX_BACKOFF: Duration = Duration::from_secs(30); +pub(crate) const RECONNECT_STABLE_AFTER: Duration = Duration::from_secs(60); + +/// Sleep for the current jittered backoff and double it up to the cap. +pub(crate) async fn backoff_sleep( + cancel: &tokio_util::sync::CancellationToken, + backoff: &mut Duration, +) -> bool { + tokio::select! { + () = cancel.cancelled() => return false, + () = tokio::time::sleep(jittered(*backoff)) => {}, + } + *backoff = next_backoff(*backoff); + true +} + +pub(crate) fn next_backoff(current: Duration) -> Duration { + (current * 2).min(RECONNECT_MAX_BACKOFF) +} + +/// Apply +/-25% jitter using wall-clock nanoseconds as non-security entropy. +pub(crate) fn jittered(base: Duration) -> Duration { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|duration| duration.subsec_nanos()) + .unwrap_or(0); + base.mul_f64(jitter_factor(nanos)) +} + +pub(crate) fn jitter_factor(nanos: u32) -> f64 { + const NANOS_PER_SEC: f64 = 1_000_000_000.0; + let unit = f64::from(nanos) / NANOS_PER_SEC; + (0.75 + unit * 0.5).clamp(0.75, 1.25) +} diff --git a/crates/slack/src/state.rs b/crates/slack/src/state.rs index 59d54f8d1a..193ac1bb51 100644 --- a/crates/slack/src/state.rs +++ b/crates/slack/src/state.rs @@ -1,6 +1,7 @@ use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashMap, VecDeque}, sync::{Arc, Mutex, RwLock}, + time::{Duration, Instant}, }; use { @@ -13,34 +14,165 @@ use crate::config::SlackAccountConfig; /// Shared account state map. pub type AccountStateMap = Arc>>; -/// Bounded set of recently seen Slack event ids. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StreamRecipient { + pub user_id: String, + pub team_id: String, +} + +type StreamRecipientKey = (String, String); + +#[derive(Clone)] +struct StreamRecipientEntry { + recipient: StreamRecipient, + last_seen: Instant, +} + +/// Bounded, TTL-limited recipient cache ordered by last observation. +#[derive(Default)] +pub struct StreamRecipients { + entries: HashMap, + order: VecDeque, +} + +impl StreamRecipients { + const MAX: usize = 2048; + const TTL: Duration = Duration::from_secs(30 * 60); + + fn get(&self, channel_id: &str, thread_root: &str) -> Option<&StreamRecipient> { + let entry = self + .entries + .get(&(channel_id.to_string(), thread_root.to_string()))?; + (entry.last_seen.elapsed() <= Self::TTL).then_some(&entry.recipient) + } + + fn note(&mut self, channel_id: &str, thread_root: &str, recipient: StreamRecipient) { + self.note_at_with_limit( + channel_id, + thread_root, + recipient, + Instant::now(), + Self::MAX, + ); + } + + fn note_at_with_limit( + &mut self, + channel_id: &str, + thread_root: &str, + recipient: StreamRecipient, + now: Instant, + max: usize, + ) { + while let Some(oldest) = self.order.front() { + let expired = self + .entries + .get(oldest) + .is_none_or(|entry| now.saturating_duration_since(entry.last_seen) > Self::TTL); + if !expired { + break; + } + if let Some(oldest) = self.order.pop_front() { + self.entries.remove(&oldest); + } + } + + let key = (channel_id.to_string(), thread_root.to_string()); + if self.entries.contains_key(&key) { + self.order.retain(|existing| existing != &key); + } else { + while self.entries.len() >= max.max(1) { + let Some(oldest) = self.order.pop_front() else { + break; + }; + self.entries.remove(&oldest); + } + } + + self.entries.insert(key.clone(), StreamRecipientEntry { + recipient, + last_seen: now, + }); + self.order.push_back(key); + } +} + +/// Category of Slack callback identifier being deduplicated. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum DedupKind { + Event, + Command, + Interaction, +} + +#[derive(Default)] +struct DedupPartition { + seen: HashMap, + order: VecDeque<(String, Instant)>, +} + +/// Bounded, per-kind set of recently seen Slack callback ids. /// -/// Slack retries an envelope when it is not acknowledged in time, so the same -/// event can arrive more than once. Without this a retry would start a second -/// agent turn for one user message. +/// Slack retries callbacks for up to five minutes. Each account and callback +/// kind retains the documented maximum of roughly 2,500 ids for that window, +/// so traffic in one category cannot evict another category's retry keys. #[derive(Default)] pub struct EventDedup { - seen: HashSet, - order: VecDeque, + partitions: HashMap, } impl EventDedup { - /// Maximum retained ids. Slack retries within minutes, so this is ample. - const MAX: usize = 2048; + const MAX_PER_KIND: usize = 2_500; + const RETRY_WINDOW: Duration = Duration::from_secs(5 * 60); /// Record an event id, returning `true` if it had not been seen before. - pub fn insert_new(&mut self, event_id: &str) -> bool { - if !self.seen.insert(event_id.to_string()) { + pub fn insert_new(&mut self, kind: DedupKind, event_id: &str) -> bool { + self.insert_new_at(kind, event_id, Instant::now()) + } + + /// Return whether an id is still inside the retry window without recording it. + pub(crate) fn contains(&mut self, kind: DedupKind, event_id: &str) -> bool { + self.contains_at(kind, event_id, Instant::now()) + } + + fn contains_at(&mut self, kind: DedupKind, event_id: &str, now: Instant) -> bool { + let partition = self.partitions.entry(kind).or_default(); + Self::prune_partition(partition, now); + partition.seen.contains_key(event_id) + } + + fn insert_new_at(&mut self, kind: DedupKind, event_id: &str, now: Instant) -> bool { + let partition = self.partitions.entry(kind).or_default(); + Self::prune_partition(partition, now); + + if partition.seen.contains_key(event_id) { return false; } - self.order.push_back(event_id.to_string()); - while self.order.len() > Self::MAX { - if let Some(old) = self.order.pop_front() { - self.seen.remove(&old); + + while partition.seen.len() >= Self::MAX_PER_KIND { + let Some((oldest_id, inserted_at)) = partition.order.pop_front() else { + break; + }; + if partition.seen.get(&oldest_id) == Some(&inserted_at) { + partition.seen.remove(&oldest_id); } } + partition.seen.insert(event_id.to_string(), now); + partition.order.push_back((event_id.to_string(), now)); true } + + fn prune_partition(partition: &mut DedupPartition, now: Instant) { + while let Some((oldest_id, inserted_at)) = partition.order.front() { + if now.saturating_duration_since(*inserted_at) <= Self::RETRY_WINDOW { + break; + } + if partition.seen.get(oldest_id) == Some(inserted_at) { + partition.seen.remove(oldest_id); + } + partition.order.pop_front(); + } + } } /// Per-account runtime state. @@ -52,10 +184,129 @@ pub struct AccountState { pub cancel: CancellationToken, /// Bot user ID obtained from `auth.test` β€” signals the connection is ready. pub bot_user_id: Option, - /// Pending thread timestamps keyed by `channel_id:user_id`. - /// Used to route replies into the correct thread. - pub pending_threads: HashMap, + /// Native-stream recipients keyed by the exact `(channel, thread root)`. + pub stream_recipients: StreamRecipients, pub otp: Mutex, /// Recently processed event ids, for retry idempotency. pub dedup: Mutex, } + +impl AccountState { + pub fn note_stream_recipient( + &mut self, + channel_id: &str, + thread_root: &str, + recipient: StreamRecipient, + ) { + self.stream_recipients + .note(channel_id, thread_root, recipient); + } + + pub fn stream_recipient( + &self, + channel_id: &str, + thread_root: &str, + ) -> Option<&StreamRecipient> { + self.stream_recipients.get(channel_id, thread_root) + } +} + +#[cfg(test)] +mod tests { + use { + super::{DedupKind, EventDedup, StreamRecipient, StreamRecipients}, + std::time::{Duration, Instant}, + }; + + fn recipient(user_id: &str) -> StreamRecipient { + StreamRecipient { + user_id: user_id.to_string(), + team_id: "T1".to_string(), + } + } + + #[test] + fn event_dedup_retains_documented_five_minute_throughput() { + let now = Instant::now(); + let mut dedup = EventDedup::default(); + for index in 0..EventDedup::MAX_PER_KIND { + assert!(dedup.insert_new_at(DedupKind::Event, &index.to_string(), now)); + } + assert!(!dedup.insert_new_at(DedupKind::Event, "0", now + EventDedup::RETRY_WINDOW)); + + let after_window = now + EventDedup::RETRY_WINDOW + Duration::from_millis(1); + assert!(dedup.insert_new_at(DedupKind::Event, "0", after_window)); + } + + #[test] + fn event_kinds_have_separate_capacity_partitions() { + let now = Instant::now(); + let mut dedup = EventDedup::default(); + for index in 0..EventDedup::MAX_PER_KIND { + assert!(dedup.insert_new_at(DedupKind::Event, &index.to_string(), now)); + } + assert!(dedup.insert_new_at(DedupKind::Command, "same", now)); + assert!(dedup.insert_new_at(DedupKind::Interaction, "same", now)); + assert!(!dedup.insert_new_at(DedupKind::Command, "same", now)); + } + + #[test] + fn stream_recipient_refresh_does_not_evict_and_updates_age_order() { + let now = Instant::now(); + let mut recipients = StreamRecipients::default(); + recipients.note_at_with_limit("C1", "1.0", recipient("U1"), now, 2); + recipients.note_at_with_limit( + "C1", + "2.0", + recipient("U2"), + now + Duration::from_secs(1), + 2, + ); + + recipients.note_at_with_limit( + "C1", + "1.0", + recipient("U1-new"), + now + Duration::from_secs(2), + 2, + ); + assert_eq!(recipients.get("C1", "1.0"), Some(&recipient("U1-new"))); + assert_eq!(recipients.get("C1", "2.0"), Some(&recipient("U2"))); + + recipients.note_at_with_limit( + "C1", + "3.0", + recipient("U3"), + now + Duration::from_secs(3), + 2, + ); + assert!(recipients.get("C1", "2.0").is_none()); + assert!(recipients.get("C1", "1.0").is_some()); + assert!(recipients.get("C1", "3.0").is_some()); + } + + #[test] + fn stream_recipient_evicts_expired_entries_before_fresh_entries() { + let now = Instant::now(); + let mut recipients = StreamRecipients::default(); + recipients.note_at_with_limit("C1", "old", recipient("U-old"), now, 2); + recipients.note_at_with_limit( + "C1", + "fresh", + recipient("U-fresh"), + now + StreamRecipients::TTL, + 2, + ); + recipients.note_at_with_limit( + "C1", + "new", + recipient("U-new"), + now + StreamRecipients::TTL + Duration::from_millis(1), + 2, + ); + + assert!(recipients.get("C1", "old").is_none()); + assert!(recipients.get("C1", "fresh").is_some()); + assert!(recipients.get("C1", "new").is_some()); + } +} diff --git a/crates/slack/src/webhook.rs b/crates/slack/src/webhook.rs index 82a6813a2b..c3efc4f8ae 100644 --- a/crates/slack/src/webhook.rs +++ b/crates/slack/src/webhook.rs @@ -77,7 +77,7 @@ pub async fn register_events_api_account( event_sink, cancel, bot_user_id: Some(bot_user_id), - pending_threads: std::collections::HashMap::new(), + stream_recipients: Default::default(), otp: std::sync::Mutex::new(moltis_channels::otp::OtpState::new(otp_cooldown_secs)), dedup: std::sync::Mutex::new(crate::state::EventDedup::default()), }); @@ -250,9 +250,24 @@ pub async fn handle_verified_interaction_webhook( .and_then(|c| c.get("id")) .and_then(|v| v.as_str()) .unwrap_or(""); + let user_id = payload + .get("user") + .and_then(|user| user.get("id")) + .and_then(|value| value.as_str()) + .unwrap_or(""); + + if action_id.is_empty() || channel_id.is_empty() || user_id.is_empty() { + debug!( + account_id, + "interaction missing action_id, channel, or user" + ); + return Ok(()); + } - if action_id.is_empty() || channel_id.is_empty() { - debug!(account_id, "interaction missing action_id or channel"); + if !crate::socket::account_access_allowed(accounts, account_id, user_id, channel_id) { + if let Some(response_url) = payload.get("response_url").and_then(|value| value.as_str()) { + let _ = crate::outbound::post_response_url(response_url, "Access denied.").await; + } return Ok(()); } @@ -267,14 +282,20 @@ pub async fn handle_verified_interaction_webhook( channel_type: ChannelType::Slack, account_id: account_id.to_string(), chat_id: channel_id.to_string(), - message_id: None, + message_id: interaction_thread_root(&payload), thread_id: None, }; - match sink.dispatch_interaction(action_id, reply_to).await { - Ok(_) => {}, - Err(e) => { - debug!(account_id, action_id, "interaction dispatch failed: {e}"); - }, + let response = sink + .dispatch_interaction(action_id, reply_to) + .await + .unwrap_or_else(|error| format!("Error: {error}")); + if let Some(response_url) = payload.get("response_url").and_then(|value| value.as_str()) + && let Err(error) = crate::outbound::post_response_url(response_url, &response).await + { + debug!( + account_id, + action_id, "interaction response failed: {error}" + ); } } @@ -298,6 +319,7 @@ pub async fn handle_verified_command_webhook( let text = extract_form_field(body_str, "text").unwrap_or_default(); let user_id = extract_form_field(body_str, "user_id").unwrap_or_default(); let channel_id = extract_form_field(body_str, "channel_id").unwrap_or_default(); + let response_url = extract_form_field(body_str, "response_url"); if command.is_empty() { return Err(moltis_channels::Error::invalid_input( @@ -305,6 +327,20 @@ pub async fn handle_verified_command_webhook( )); } + if user_id.is_empty() || channel_id.is_empty() { + return Err(moltis_channels::Error::invalid_input( + "missing user_id or channel_id in slash command payload", + )); + } + + if !crate::socket::account_access_allowed(accounts, account_id, &user_id, &channel_id) { + let response = "Access denied.".to_string(); + if let Some(response_url) = response_url.as_deref() { + let _ = crate::outbound::post_response_url(response_url, &response).await; + } + return Ok(response); + } + let full_command = format!("{command} {text}").trim().to_string(); let event_sink = { @@ -326,13 +362,19 @@ pub async fn handle_verified_command_webhook( } else { Some(user_id.as_str()) }; - match sink.dispatch_command(&full_command, reply_to, sender).await { - Ok(response_text) => Ok(response_text), - Err(e) => { - debug!(account_id, %full_command, "command dispatch failed: {e}"); - Ok(format!("Error: {e}")) - }, + let response = sink + .dispatch_command(&full_command, reply_to, sender) + .await + .unwrap_or_else(|error| { + debug!(account_id, %full_command, "command dispatch failed: {error}"); + format!("Error: {error}") + }); + if let Some(response_url) = response_url.as_deref() + && let Err(error) = crate::outbound::post_response_url(response_url, &response).await + { + debug!(account_id, %full_command, "command response failed: {error}"); } + Ok(response) } else { Ok("Channel not configured".to_string()) } @@ -344,6 +386,10 @@ async fn dispatch_event_callback( payload: &serde_json::Value, accounts: &AccountStateMap, ) { + let team_id = payload + .get("team_id") + .and_then(|value| value.as_str()) + .map(String::from); let Some(event) = payload.get("event") else { debug!("event_callback missing event field"); return; @@ -356,7 +402,8 @@ async fn dispatch_event_callback( // Parse as SlackMessageEvent via serde. match serde_json::from_value::(event.clone()) { Ok(msg_event) => { - crate::socket::handle_message_event(account_id, msg_event, accounts).await; + crate::socket::handle_message_event(account_id, msg_event, team_id, accounts) + .await; }, Err(e) => { debug!(account_id, "failed to parse message event: {e}"); @@ -376,7 +423,7 @@ async fn dispatch_event_callback( if !channel.is_empty() && !user.is_empty() { crate::socket::handle_inbound( - account_id, channel, user, text, thread_ts, message_ts, None, + account_id, channel, user, text, thread_ts, message_ts, team_id, None, true, // is_mention accounts, ) @@ -406,6 +453,7 @@ async fn dispatch_event_callback( item_channel.to_string(), message_ts.to_string(), item_user, + team_id, added, accounts, ) @@ -450,66 +498,25 @@ pub async fn handle_interaction_webhook( )); } - // Parse form-encoded body to extract `payload` field. - let body_str = std::str::from_utf8(body) - .map_err(|e| moltis_channels::Error::invalid_input(format!("invalid utf-8: {e}")))?; - - let payload_json = extract_form_payload(body_str).ok_or_else(|| { - moltis_channels::Error::invalid_input("missing payload field in interaction") - })?; - - let payload: serde_json::Value = serde_json::from_str(&payload_json)?; - - // Extract action from block_actions. - let interaction_type = payload.get("type").and_then(|v| v.as_str()).unwrap_or(""); - - if interaction_type != "block_actions" { - debug!(account_id, interaction_type, "unhandled interaction type"); - return Ok(()); - } - - let action_id = payload - .get("actions") - .and_then(|a| a.as_array()) - .and_then(|a| a.first()) - .and_then(|a| a.get("action_id")) - .and_then(|v| v.as_str()) - .unwrap_or(""); - - let channel_id = payload - .get("channel") - .and_then(|c| c.get("id")) - .and_then(|v| v.as_str()) - .unwrap_or(""); - - if action_id.is_empty() || channel_id.is_empty() { - debug!(account_id, "interaction missing action_id or channel"); - return Ok(()); - } - - let event_sink = { - let accts = accounts.read().unwrap_or_else(|e| e.into_inner()); - accts.get(account_id).and_then(|s| s.event_sink.clone()) - }; - - if let Some(sink) = event_sink { - let reply_to = ChannelReplyTarget { - ack_message_id: None, - channel_type: ChannelType::Slack, - account_id: account_id.to_string(), - chat_id: channel_id.to_string(), - message_id: None, - thread_id: None, - }; - match sink.dispatch_interaction(action_id, reply_to).await { - Ok(_) => {}, - Err(e) => { - debug!(account_id, action_id, "interaction dispatch failed: {e}"); - }, - } - } + handle_verified_interaction_webhook(account_id, body, accounts).await +} - Ok(()) +fn interaction_thread_root(payload: &serde_json::Value) -> Option { + payload + .get("message") + .and_then(|message| { + message + .get("thread_ts") + .or_else(|| message.get("ts")) + .and_then(|value| value.as_str()) + }) + .or_else(|| { + payload + .get("container") + .and_then(|container| container.get("message_ts")) + .and_then(|value| value.as_str()) + }) + .map(String::from) } /// Extract a named field from a `application/x-www-form-urlencoded` body. @@ -561,12 +568,14 @@ mod tests { /// Mock sink that records the command string passed to `dispatch_command`. struct RecordingSink { commands: Mutex>, + interactions: Mutex>, } impl RecordingSink { fn new() -> Self { Self { commands: Mutex::new(Vec::new()), + interactions: Mutex::new(Vec::new()), } } } @@ -593,6 +602,18 @@ mod tests { Ok("ok".to_string()) } + async fn dispatch_interaction( + &self, + callback_data: &str, + reply_to: ChannelReplyTarget, + ) -> ChannelResult { + self.interactions + .lock() + .unwrap() + .push((callback_data.to_string(), reply_to)); + Ok("updated".to_string()) + } + async fn request_disable_account( &self, _channel_type: &str, @@ -730,7 +751,7 @@ mod tests { event_sink: None, cancel: tokio_util::sync::CancellationToken::new(), bot_user_id: Some("B123".to_string()), - pending_threads: std::collections::HashMap::new(), + stream_recipients: Default::default(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), dedup: Mutex::new(crate::state::EventDedup::default()), }); @@ -761,7 +782,7 @@ mod tests { event_sink: Some(sink.clone()), cancel: tokio_util::sync::CancellationToken::new(), bot_user_id: Some("B123".to_string()), - pending_threads: std::collections::HashMap::new(), + stream_recipients: Default::default(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), dedup: Mutex::new(crate::state::EventDedup::default()), }); @@ -797,7 +818,7 @@ mod tests { event_sink: Some(sink.clone()), cancel: tokio_util::sync::CancellationToken::new(), bot_user_id: Some("B123".to_string()), - pending_threads: std::collections::HashMap::new(), + stream_recipients: Default::default(), otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), dedup: Mutex::new(crate::state::EventDedup::default()), }); @@ -812,4 +833,161 @@ mod tests { assert_eq!(dispatched.len(), 1); assert_eq!(dispatched[0], "/model gpt-4o"); } + + #[tokio::test] + async fn slash_command_denied_by_channel_policy_is_not_dispatched() { + let sink = Arc::new(RecordingSink::new()); + let accounts: AccountStateMap = + Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let config = SlackAccountConfig { + group_policy: moltis_channels::gating::GroupPolicy::Disabled, + ..Default::default() + }; + { + let mut accts = accounts.write().unwrap(); + accts.insert("acct1".to_string(), AccountState { + account_id: "acct1".to_string(), + config, + message_log: None, + event_sink: Some(sink.clone()), + cancel: tokio_util::sync::CancellationToken::new(), + bot_user_id: Some("B123".to_string()), + stream_recipients: Default::default(), + otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), + }); + } + + let body = b"command=%2Fnew&text=&user_id=U123&channel_id=C456"; + let response = handle_verified_command_webhook("acct1", body, &accounts) + .await + .unwrap(); + assert_eq!(response, "Access denied."); + assert!(sink.commands.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn interaction_preserves_exact_thread_root() { + let sink = Arc::new(RecordingSink::new()); + let accounts: AccountStateMap = + Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + { + let mut accts = accounts.write().unwrap(); + accts.insert("acct1".to_string(), AccountState { + account_id: "acct1".to_string(), + config: SlackAccountConfig::default(), + message_log: None, + event_sink: Some(sink.clone()), + cancel: tokio_util::sync::CancellationToken::new(), + bot_user_id: Some("B123".to_string()), + stream_recipients: Default::default(), + otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), + }); + } + let payload = serde_json::json!({ + "type": "block_actions", + "user": { "id": "U123" }, + "channel": { "id": "C456" }, + "message": { "ts": "200.2", "thread_ts": "100.1" }, + "actions": [{ "action_id": "model_switch:gpt" }], + }); + let body = format!("payload={payload}"); + + handle_verified_interaction_webhook("acct1", body.as_bytes(), &accounts) + .await + .unwrap(); + + let interactions = sink.interactions.lock().unwrap(); + assert_eq!(interactions.len(), 1); + assert_eq!(interactions[0].0, "model_switch:gpt"); + assert_eq!(interactions[0].1.message_id.as_deref(), Some("100.1")); + } + + #[tokio::test] + async fn reaction_trigger_registers_native_stream_recipient_for_exact_root() { + let sink = Arc::new(RecordingSink::new()); + let accounts: AccountStateMap = + Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let config = SlackAccountConfig { + reaction_triggers: true, + ..Default::default() + }; + accounts + .write() + .unwrap() + .insert("acct1".to_string(), AccountState { + account_id: "acct1".to_string(), + config, + message_log: None, + event_sink: Some(sink), + cancel: tokio_util::sync::CancellationToken::new(), + bot_user_id: Some("B123".to_string()), + stream_recipients: Default::default(), + otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), + }); + let payload = serde_json::json!({ + "type": "event_callback", + "team_id": "T123", + "event": { + "type": "reaction_added", + "user": "U456", + "reaction": "white_check_mark", + "item_user": "B123", + "item": { + "type": "message", + "channel": "C789", + "ts": "100.200" + } + } + }); + + handle_verified_webhook("acct1", payload.to_string().as_bytes(), &accounts) + .await + .unwrap(); + + let accts = accounts.read().unwrap(); + let recipient = accts["acct1"].stream_recipient("C789", "100.200").unwrap(); + assert_eq!(recipient.user_id, "U456"); + assert_eq!(recipient.team_id, "T123"); + } + + #[tokio::test] + async fn interaction_denied_by_channel_policy_is_not_dispatched() { + let sink = Arc::new(RecordingSink::new()); + let accounts: AccountStateMap = + Arc::new(std::sync::RwLock::new(std::collections::HashMap::new())); + let config = SlackAccountConfig { + group_policy: moltis_channels::gating::GroupPolicy::Disabled, + ..Default::default() + }; + { + let mut accts = accounts.write().unwrap(); + accts.insert("acct1".to_string(), AccountState { + account_id: "acct1".to_string(), + config, + message_log: None, + event_sink: Some(sink.clone()), + cancel: tokio_util::sync::CancellationToken::new(), + bot_user_id: Some("B123".to_string()), + stream_recipients: Default::default(), + otp: Mutex::new(moltis_channels::otp::OtpState::new(300)), + dedup: Mutex::new(crate::state::EventDedup::default()), + }); + } + let payload = serde_json::json!({ + "type": "block_actions", + "user": { "id": "U123" }, + "channel": { "id": "C456" }, + "message": { "ts": "100.1" }, + "actions": [{ "action_id": "model_switch:gpt" }], + }); + let body = format!("payload={payload}"); + + handle_verified_interaction_webhook("acct1", body.as_bytes(), &accounts) + .await + .unwrap(); + assert!(sink.interactions.lock().unwrap().is_empty()); + } } diff --git a/crates/web/ui/e2e/specs/channels-slack.spec.js b/crates/web/ui/e2e/specs/channels-slack.spec.js index dd4df8bff2..6d27355835 100644 --- a/crates/web/ui/e2e/specs/channels-slack.spec.js +++ b/crates/web/ui/e2e/specs/channels-slack.spec.js @@ -48,6 +48,57 @@ async function installSlackChannelMock(page, channel) { } test.describe("Slack channel settings", () => { + test("setup guide lists required scopes, events, and Events API routes", async ({ page }) => { + const pageErrors = watchPageErrors(page); + await navigateAndWait(page, "/settings/channels"); + await waitForWsConnected(page); + + await page.getByRole("button", { name: "Connect Slack", exact: true }).click(); + const guide = page.getByTestId("slack-setup-guide"); + await expect(guide).toBeVisible(); + + const scopes = [ + "app_mentions:read", + "chat:write", + "files:write", + "im:history", + "reactions:write", + "reactions:read", + "channels:history", + "groups:history", + "mpim:history", + "connections:write", + ]; + for (const scope of scopes) { + await expect(guide.getByText(scope, { exact: true })).toBeVisible(); + } + + const events = [ + "app_mention", + "message.im", + "reaction_added", + "message.channels", + "message.groups", + "message.mpim", + ]; + for (const event of events) { + await expect(guide.getByText(event, { exact: true })).toBeVisible(); + } + + const routes = [ + "https://your-host/api/channels/slack//events", + "https://your-host/api/channels/slack//interactions", + "https://your-host/api/channels/slack//commands", + ]; + for (const route of routes) { + await expect(guide.getByText(route, { exact: true })).toBeVisible(); + } + + await expect(guide).toContainText("mention_mode = always"); + await expect(guide).toContainText("Each scope permits access; its paired event delivers messages"); + expect(pageErrors).toEqual([]); + }); + test("ack_reactions and reaction_triggers toggles round-trip through the edit modal", async ({ page }) => { const pageErrors = watchPageErrors(page); await navigateAndWait(page, "/settings/channels"); diff --git a/crates/web/ui/src/onboarding/steps/ChannelStep.tsx b/crates/web/ui/src/onboarding/steps/ChannelStep.tsx index 4445c82fad..fc4f16f3f1 100644 --- a/crates/web/ui/src/onboarding/steps/ChannelStep.tsx +++ b/crates/web/ui/src/onboarding/steps/ChannelStep.tsx @@ -661,7 +661,10 @@ function SlackForm({ onConnected, error, setError }: ChannelFormProps): VNode { return (
-
+
How to set up a Slack bot 1. Go to{" "} @@ -676,17 +679,43 @@ function SlackForm({ onConnected, error, setError }: ChannelFormProps): VNode { and create a new app - 2. Under OAuth & Permissions, add bot scopes: chat:write,{" "} - channels:history,{" "} - im:history,{" "} - app_mentions:read + 2. Under OAuth & Permissions, add bot scopes:{" "} + app_mentions:read,{" "} + chat:write,{" "} + files:write,{" "} + im:history, and{" "} + reactions:write + + + 3. Subscribe to bot events: app_mention and{" "} + message.im. For reaction triggers, also add the{" "} + reactions:read scope and{" "} + reaction_added event - 3. Install the app to your workspace and copy the Bot User OAuth Token - 4. For Socket Mode: enable it and generate an App-Level Token with{" "} + 4. For mention_mode = always in public channels, add both{" "} + channels:history and{" "} + message.channels. For private channels, add{" "} + groups:history and{" "} + message.groups. For MPIMs, add{" "} + mpim:history and{" "} + message.mpim. Each scope permits access; its paired event + delivers messages + + 5. Install the app to your workspace and copy the Bot User OAuth Token + + 6. For Socket Mode: enable it and generate an App-Level Token with{" "} connections:write scope - 5. For Events API: set the Request URL to your server’s webhook endpoint + + 7. For Events API, set these Request URLs (replace <id>{" "} + with the Account ID below): Event Subscriptions{" "} + https://your-host/api/channels/slack/<id>/events, + Interactivity{" "} + https://your-host/api/channels/slack/<id>/interactions, + Slash Commands{" "} + https://your-host/api/channels/slack/<id>/commands +
diff --git a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx index 3085da9bb5..a8fcb9bbf3 100644 --- a/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx +++ b/crates/web/ui/src/pages/channels/modals/AddSlackModal.tsx @@ -106,7 +106,7 @@ export function AddSlackModal(): VNode { title="Connect Slack" >
-
+
How to set up a Slack bot
@@ -122,28 +122,38 @@ export function AddSlackModal(): VNode { and create a new app
- 2. Under OAuth & Permissions, add bot scopes: chat:write,{" "} - channels:history,{" "} - im:history,{" "} + 2. Under OAuth & Permissions, add bot scopes:{" "} app_mentions:read,{" "} - reactions:write (acknowledgment reactions),{" "} - reactions:read (only for reaction triggers) + chat:write,{" "} + files:write,{" "} + im:history, and{" "} + reactions:write
- 3. Install the app to your workspace and copy the Bot User OAuth Token + 3. Subscribe to bot events: app_mention and{" "} + message.im. For reaction triggers, also add the{" "} + reactions:read scope and{" "} + reaction_added event
- 4. For Socket Mode: enable Socket Mode and generate an App-Level Token with{" "} - connections:write scope + 4. For mention_mode = always in public channels, add both{" "} + channels:history and{" "} + message.channels. For private channels, add{" "} + groups:history and{" "} + message.groups. For MPIMs, add{" "} + mpim:history and{" "} + message.mpim. Each scope permits access; its paired event + delivers messages +
+
+ 5. Install the app to your workspace and copy the Bot User OAuth Token
- 5. Under Event Subscriptions, subscribe to bot events:{" "} - message.im,{" "} - app_mention, and{" "} - reaction_added if you use reaction triggers + 6. For Socket Mode: enable Socket Mode and generate an App-Level Token with{" "} + connections:write scope
- 6. For Events API, set these Request URLs (replace{" "} + 7. For Events API, set these Request URLs (replace{" "} <id> with the Account ID below): Event Subscriptions{" "} https://your-host/api/channels/slack/<id>/events, Interactivity{" "} diff --git a/docs/src/slack.md b/docs/src/slack.md index c49b9909af..0be470e7b2 100644 --- a/docs/src/slack.md +++ b/docs/src/slack.md @@ -43,22 +43,34 @@ Before configuring Moltis, create a Slack app: 3. Navigate to **OAuth & Permissions** and add these Bot Token Scopes: - `app_mentions:read` β€” read @mentions - `chat:write` β€” send messages + - `files:write` β€” upload files generated by the agent - `im:history` β€” read DM history - - `im:read` β€” view DM metadata - - `channels:history` β€” read channel messages (for `mention_mode = "always"`) - `reactions:write` β€” add acknowledgment reactions (πŸ‘€/βœ…/❌; see [Acknowledgment Reactions](#acknowledgment-reactions)) - `reactions:read` β€” read reactions (only for inbound reaction triggers) -4. Click **Install to Workspace** and copy the **Bot User OAuth Token** (`xoxb-...`) -5. For Socket Mode (recommended): +4. Under **Event Subscriptions > Subscribe to bot events**, add: + - `app_mention` β€” channel messages that @mention the bot + - `message.im` β€” direct messages to the bot (paired with `im:history`) + - `reaction_added` β€” only if using inbound reaction triggers (paired with `reactions:read`) +5. If you use `mention_mode = "always"`, add the matching scope and bot event for every conversation type where the bot should receive all messages: + + | Conversation type | Bot Token Scope | Bot event | + |-------------------|-----------------|-----------| + | Public channels | `channels:history` | `message.channels` | + | Private channels | `groups:history` | `message.groups` | + | Multiparty direct messages (MPIMs) | `mpim:history` | `message.mpim` | + + Both entries in a row are required: the history scope permits access, while + the `message.*` event delivers new messages to Moltis. Private channels and + MPIMs are handled as channels by Moltis, so `group_policy`, `mention_mode`, + and `channel_allowlist` apply to them. +6. Click **Install to Workspace** and copy the **Bot User OAuth Token** (`xoxb-...`) +7. For Socket Mode (recommended): - Go to **Socket Mode** and enable it - Generate an **App-Level Token** (`xapp-...`) with the `connections:write` scope -6. For Events API mode: +8. For Events API mode: - Go to **Event Subscriptions** and enable it - - Set the Request URL to your Moltis instance endpoint + - Set the exact URLs shown in [Events API Mode](#events-api-mode) - Copy the **Signing Secret** from **Basic Information** -7. Under **Event Subscriptions > Subscribe to bot events**, add: - - `app_mention` β€” when someone @mentions the bot - - `message.im` β€” direct messages to the bot ```admonish warning The bot token and app token are secrets β€” treat them like passwords. Never @@ -255,6 +267,12 @@ Slack user ID, such as `U0123456789`, to the account allowlist. | `"always"` | Bot responds to every message in allowed channels | | `"none"` | Bot never responds in channels (useful for DM-only bots) | +Slack only sends general public-channel messages when the app has both the +`channels:history` Bot Token Scope and the `message.channels` bot event. For +private channels, use `groups:history` with `message.groups`; for MPIMs, use +`mpim:history` with `message.mpim`. The default `"mention"` mode only needs the +`app_mentions:read` scope and `app_mention` event for channel mentions. + ### Allowlist Matching Allowlist entries support: @@ -275,6 +293,17 @@ Slack supports three streaming modes: The `edit_in_place` mode throttles updates to `edit_throttle_ms` milliseconds (default: 500) to avoid Slack API rate limits. +Native mode calls those three methods directly at `api_base_url` with the bot +token and requires a thread target. With `thread_replies = false`, Moltis falls +back to top-level edit-in-place streaming instead. For threaded replies, Moltis +does not probe whether the Slack app, workspace, or compatible proxy supports +the native methods, and a native API failure does not switch an active stream to +edit-in-place mode. Native requests send standard Markdown unchanged through +Slack's `markdown_text` field and use `edit_throttle_ms`. When +`rich_blocks = true`, rich rendering takes precedence: streaming (including +native streaming) is disabled and the completed response is sent once through +the Block Kit path. + ## Thread Replies By default (`thread_replies = true`), the bot replies in a thread attached to From e1c288a4be72b6ce2723c72fd60d3992a09e1d8e Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Tue, 28 Jul 2026 17:54:07 -0700 Subject: [PATCH 17/23] fix(web): distinguish models from ACP picker entries The reasoning-variant E2E assertion counted ACP agents after main moved them into the model picker because both row types expose provider metadata. Add explicit row identities and scope the assertion to model rows so it continues validating reasoning variant filtering. --- crates/web/ui/e2e/specs/reasoning-toggle.spec.js | 4 +--- crates/web/ui/src/models.ts | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/web/ui/e2e/specs/reasoning-toggle.spec.js b/crates/web/ui/e2e/specs/reasoning-toggle.spec.js index 8cb1c3dabe..23a8bf9663 100644 --- a/crates/web/ui/e2e/specs/reasoning-toggle.spec.js +++ b/crates/web/ui/e2e/specs/reasoning-toggle.spec.js @@ -197,9 +197,7 @@ test.describe("reasoning effort toggle", () => { const modelBtn = page.locator("#modelComboBtn"); await modelBtn.click(); - const items = page - .locator("#modelDropdownList .model-dropdown-item") - .filter({ has: page.locator(".model-item-provider") }); + const items = page.locator("#modelDropdownList .model-dropdown-item[data-model-id]"); // Only the base model should appear among the model entries, not the 3 reasoning variants. await expect(items).toHaveCount(1); await expect(items.first()).toContainText("Claude Opus 4.5"); diff --git a/crates/web/ui/src/models.ts b/crates/web/ui/src/models.ts index d55a138aa3..200b3c0049 100644 --- a/crates/web/ui/src/models.ts +++ b/crates/web/ui/src/models.ts @@ -286,6 +286,7 @@ export function closeModelDropdown(): void { function buildModelItem(m: ModelInfo, currentId: string): HTMLDivElement { const el = document.createElement("div"); el.className = "model-dropdown-item"; + el.dataset.modelId = m.id; if (m.id === currentId) el.classList.add("selected"); if (m.unsupported) el.classList.add("model-dropdown-item-unsupported"); @@ -330,6 +331,7 @@ function buildModelItem(m: ModelInfo, currentId: string): HTMLDivElement { function buildAcpItem(agent: ExternalAgentInfo, currentKind: string): HTMLDivElement { const el = document.createElement("div"); el.className = "model-dropdown-item"; + el.dataset.externalAgentKind = agent.kind; if (agent.kind === currentKind) el.classList.add("selected"); el.title = `${agent.name} (${agent.kind})`; From cacd299c45fa56d788ae861742b879a8fdcac9c4 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 09:54:13 -0700 Subject: [PATCH 18/23] fix(channels): stabilize callback and persistence lifecycle Slack response URL replies rebuilt their restricted HTTP client for every callback, preventing connection reuse. Keep one lazily initialized client while preserving redirect and timeout restrictions, and cover the pending acknowledgment cap boundary explicitly.\n\nCoverage also exposed a sled reopen race: the owning database handle was dropped before its cloned tree handles. Drop the database last so all trees release before the file lock is reopened. --- crates/gateway/src/channel_reactions.rs | 16 ++++++++++++ crates/slack/src/outbound.rs | 34 ++++++++++++++++++------- crates/whatsapp/src/sled_store.rs | 4 +-- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/crates/gateway/src/channel_reactions.rs b/crates/gateway/src/channel_reactions.rs index b3df7f657e..f537037022 100644 --- a/crates/gateway/src/channel_reactions.rs +++ b/crates/gateway/src/channel_reactions.rs @@ -788,6 +788,22 @@ mod tests { } } + #[tokio::test] + async fn pending_limit_evicts_only_after_crossing_the_boundary() { + let registry = ReactionRegistry { + pending_limit: 1, + ..ReactionRegistry::default() + }; + let oldest = park(®istry, "msg-oldest").await; + assert_eq!(ops_of(&oldest), vec!["+πŸ‘€"]); + + let newest = park(®istry, "msg-newest").await; + tokio::time::sleep(Duration::from_millis(60)).await; + + assert_eq!(ops_of(&oldest), vec!["+πŸ‘€", "-πŸ‘€"]); + assert_eq!(ops_of(&newest), vec!["+πŸ‘€"]); + } + #[tokio::test] async fn superseded_turn_terminal_does_not_clear_the_current_turn() { // Turn 1 activates, is superseded by turn 2, then turn 1's terminal diff --git a/crates/slack/src/outbound.rs b/crates/slack/src/outbound.rs index 15cbd18685..2d4305e3db 100644 --- a/crates/slack/src/outbound.rs +++ b/crates/slack/src/outbound.rs @@ -77,18 +77,27 @@ fn validate_response_url(url: &str) -> ChannelResult { Ok(url) } -fn build_response_url_http_client() -> ChannelResult { - reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .connect_timeout(Duration::from_secs(3)) - .timeout(Duration::from_secs(10)) - .build() - .map_err(|error| ChannelError::external("Slack response_url client", error)) +fn response_url_http_client() -> ChannelResult<&'static reqwest::Client> { + static CLIENT: std::sync::OnceLock> = + std::sync::OnceLock::new(); + match CLIENT.get_or_init(|| { + reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(Duration::from_secs(3)) + .timeout(Duration::from_secs(10)) + .build() + .map_err(|error| error.to_string()) + }) { + Ok(client) => Ok(client), + Err(error) => Err(ChannelError::unavailable(format!( + "failed to build Slack response_url client: {error}" + ))), + } } pub(crate) async fn post_response_url(url: &str, text: &str) -> ChannelResult<()> { let url = validate_response_url(url)?; - let response = build_response_url_http_client()? + let response = response_url_http_client()? .post(url) .json(&serde_json::json!({ "response_type": "ephemeral", @@ -930,6 +939,13 @@ mod tests { } } + #[test] + fn response_url_client_is_reused() { + let first = response_url_http_client().unwrap(); + let second = response_url_http_client().unwrap(); + assert!(std::ptr::eq(first, second)); + } + #[tokio::test] async fn response_url_client_does_not_follow_redirects() { use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -949,7 +965,7 @@ mod tests { .is_ok() }); - let response = build_response_url_http_client() + let response = response_url_http_client() .unwrap() .post(format!("http://{address}/initial")) .send() diff --git a/crates/whatsapp/src/sled_store.rs b/crates/whatsapp/src/sled_store.rs index 74dd636300..c37866aa14 100644 --- a/crates/whatsapp/src/sled_store.rs +++ b/crates/whatsapp/src/sled_store.rs @@ -37,8 +37,6 @@ fn hex_encode(bytes: &[u8]) -> String { /// Persistent store backed by sled, implementing all wacore storage traits. pub struct SledStore { - #[allow(dead_code)] - db: sled::Db, identities: sled::Tree, sessions: sled::Tree, prekeys: sled::Tree, @@ -58,6 +56,8 @@ pub struct SledStore { tc_tokens: sled::Tree, sent_messages: sled::Tree, msg_secrets: sled::Tree, + // Drop the cloned tree handles before the database releases its file lock. + db: sled::Db, } fn json_err(e: serde_json::Error) -> StoreError { From 1c7a8420810abd538e6b8f94080404eebd525989 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 11:04:24 -0700 Subject: [PATCH 19/23] fix(scripts): parse CRLF Cargo manifests Targeted local validation extracted package names with a trailing carriage return when a crate manifest used CRLF line endings, causing Cargo to reject the package argument. Consume trailing line whitespace when parsing manifest names so LF and CRLF files produce identical commands. --- scripts/local-validate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-validate.sh b/scripts/local-validate.sh index 103d924b46..3584a4a589 100755 --- a/scripts/local-validate.sh +++ b/scripts/local-validate.sh @@ -238,7 +238,7 @@ package_name_for_path() { while [[ "$dir" != "." && "$dir" != "/" ]]; do if [[ -f "$dir/Cargo.toml" ]]; then - sed -nE 's/^name[[:space:]]*=[[:space:]]*"([^"]+)"/\1/p' "$dir/Cargo.toml" | head -n1 + sed -nE 's/^name[[:space:]]*=[[:space:]]*"([^"]+)"[[:space:]]*$/\1/p' "$dir/Cargo.toml" | head -n1 return fi dir="$(dirname "$dir")" From ce0073a5b4c6826e10037ee96185ffb33fa37495 Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 13:27:27 -0700 Subject: [PATCH 20/23] refactor(slack): deduplicate callback routes and restore stripped docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three Slack callback endpoints (events, interactions, commands) were registered as three near-identical 70-line closures inside gateway.rs, which had grown to 1470 of the 1500-line CI limit as a result. Extract one `handle_slack_callback` parameterized by `SlackCallbackKind`, with the per-endpoint differences expressed as data: - `endpoint()` names the rate-limit/dedupe scope (already existed) - `ack_response()` holds the one real behavioral difference β€” slash commands render their response body in the Slack client, so they acknowledge with a bare 200 while events and interactions answer JSON - `event_preflight()` keeps the events-only cases that must answer synchronously: the url_verification handshake and a malformed body gateway.rs drops from 1470 to 1297 lines, and the extracted units are now directly testable (7 new tests covering the handshake, malformed bodies, per-endpoint ack shape, and the 503-on-saturation contract). Separately, restore documentation this branch deleted to fit files under the size limit. channels.rs was 1487 lines on main and the branch added ~250 lines of ack-tracking, so it paid for them by stripping 60 comment lines β€” including the frontend `sessionPath()` contract, the compaction notice's user-visible behavior, and the "targets are peeked, not drained" rationale that explains why an in-flight run can still reply. Splitting instead of stripping is what CLAUDE.md asks for, so move TTS synthesis into channels/tts.rs and flatten the deeply nested Telegram voice branch into `deliver_reply_to_target` with guard clauses, which buys back the room and restores every comment. Also fixed along the way: - Two doc comments had been left attached to the wrong item. A deleted const's docs dangled on top of `register_channel_reaction_controller` (and claimed it keys by session, not by ack key), and `finalize_channel_acks`'s docs had migrated onto `finalize_active_channel_acks`, leaving the former undocumented and the latter describing the wrong function. - `prepare_collected` set `_ack_keys` from a computed Vec and then re-parsed that same Vec back out of the JSON to return it. Compute it once; the two branches now share the tail instead of duplicating it. - Replace a hand-rolled percent-decoder in the Slack webhook verifier with `form_urlencoded`. Slack posts form bodies where a space is `+`, and a mature decoder is the right tool. Because `form_urlencoded` decodes leniently (a malformed `%GG` survives verbatim), keep the branch's strictness explicitly: a trigger id containing `%` or U+FFFD is decode garbage and must not become a dedupe key, since a low-entropy key would let unrelated callbacks collide and be dropped. - The gateway wrote the literal `"_ack_keys"` while moltis-chat read it from a private const. Export the const and use it on both sides. --- Cargo.lock | 1 + Cargo.toml | 4 +- crates/chat/src/channel_acks.rs | 5 +- crates/chat/src/channels.rs | 515 ++++++++---------- crates/chat/src/channels/tests.rs | 1 + crates/chat/src/channels/tts.rs | 153 ++++++ crates/chat/src/lib.rs | 2 +- crates/chat/src/runtime.rs | 6 +- .../chat/src/service/chat_impl/queue_drain.rs | 65 ++- crates/gateway/src/channel_events.rs | 14 +- crates/gateway/src/channel_events/dispatch.rs | 2 +- crates/httpd/src/server/gateway.rs | 253 ++------- crates/httpd/src/server/slack_callbacks.rs | 214 +++++++- crates/slack/Cargo.toml | 1 + crates/slack/src/channel_webhook_verifier.rs | 64 ++- 15 files changed, 725 insertions(+), 575 deletions(-) create mode 100644 crates/chat/src/channels/tts.rs diff --git a/Cargo.lock b/Cargo.lock index 34c2b47361..36336bb443 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8327,6 +8327,7 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", + "form_urlencoded", "hmac 0.12.1", "http 1.4.2", "moltis-channels", diff --git a/Cargo.toml b/Cargo.toml index aeb9e23b25..8a8cf9e6c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -299,7 +299,9 @@ tower = "0.5" tower-http = "0.6" url = "2" urlencoding = "2" -web-push = "0.11" +# Form bodies (application/x-www-form-urlencoded) +form_urlencoded = "1" +web-push = "0.11" # Metrics metrics = "0.24" metrics-exporter-prometheus = "0.16" diff --git a/crates/chat/src/channel_acks.rs b/crates/chat/src/channel_acks.rs index f390a034c3..27d909a508 100644 --- a/crates/chat/src/channel_acks.rs +++ b/crates/chat/src/channel_acks.rs @@ -11,7 +11,10 @@ use serde_json::Value; use crate::runtime::ChatRuntime; /// Params key carrying the acknowledgment identities a call is responsible for. -pub(crate) const ACK_KEYS_PARAM: &str = "_ack_keys"; +/// +/// Exported so the channel dispatch layer sets the same key this module reads, +/// instead of both sides hardcoding the string. +pub const ACK_KEYS_PARAM: &str = "_ack_keys"; /// Read the acknowledgment identities from `chat.send` params. pub(crate) fn ack_keys_from_params(params: &Value) -> Vec { diff --git a/crates/chat/src/channels.rs b/crates/chat/src/channels.rs index 9ce2ef4150..2842ab8620 100644 --- a/crates/chat/src/channels.rs +++ b/crates/chat/src/channels.rs @@ -1,9 +1,12 @@ -//! Channel delivery, TTS, push notifications, tool status, screenshots, documents, and location. +//! Channel delivery, push notifications, tool status, screenshots, documents, +//! and location. +//! +//! TTS synthesis lives in [`tts`]; everything here is about getting a finished +//! reply out to the channel targets a session accumulated during its run. use std::{collections::HashSet, sync::Arc, time::Duration}; use { - serde::{Deserialize, Serialize}, serde_json::Value, tokio::task::JoinSet, tracing::{debug, info, warn}, @@ -12,12 +15,24 @@ use { use moltis_sessions::store::SessionStore; use crate::{ - agent_loop::ChannelReplyTargetKey, channel_acks::note_delivery_failed, compaction_run, error, - runtime::ChatRuntime, types::*, + agent_loop::ChannelReplyTargetKey, channel_acks::note_delivery_failed, + channels::tts::build_tts_payload, compaction_run, runtime::ChatRuntime, types::*, }; +pub(crate) use crate::channels::tts::generate_tts_audio; + +/// Cap on the final, irreversible channel I/O (last reply or terminal error). +/// +/// The run has already committed its terminal by the time these are sent, so a +/// channel API that never answers must not hold the run's task β€” and its +/// session permit β€” open indefinitely. Overrunning it is reported as a delivery +/// failure so the acknowledgment reaction shows ❌ rather than βœ…. const FINAL_CHANNEL_IO_TIMEOUT: Duration = Duration::from_secs(30); +/// Build the SPA URL for a push notification click-through. +/// +/// Must match the frontend `sessionPath()` in `router.ts`: +/// `/chats/${key.replace(/:/g, "/")}`. #[cfg(any(feature = "push-notifications", test))] pub(crate) fn push_notification_url(session_key: &str) -> String { format!("/chats/{}", session_key.replace(':', "/")) @@ -29,6 +44,7 @@ pub(crate) async fn send_chat_push_notification( session_key: &str, text: &str, ) { + // Create a short summary of the response (first 100 chars). let summary = if text.len() > 100 { format!("{}…", truncate_at_char_boundary(text, 100)) } else { @@ -51,6 +67,14 @@ pub(crate) async fn send_chat_push_notification( } } +/// Drain any pending channel reply targets for a session and send the +/// response text back to each originating channel via outbound. +/// +/// Each delivery runs in its own spawned task so slow network calls don't block +/// each other or the chat pipeline. The whole fan-out is bounded by +/// [`FINAL_CHANNEL_IO_TIMEOUT`]; overrunning it (or any individual send +/// failing) marks the turn's acknowledgment as a delivery failure, since the +/// user never received the reply. pub(crate) async fn deliver_channel_replies( state: &Arc, activity_id: &str, @@ -92,6 +116,8 @@ async fn deliver_channel_replies_inner( let drained_targets = state.drain_channel_replies(session_key).await; let mut targets = Vec::with_capacity(drained_targets.len()); let mut streamed_targets = Vec::new(); + // When the reply medium is voice we must still deliver TTS audio even if + // the text was already streamed β€” skip the stream dedupe entirely. if desired_reply_medium != ReplyMedium::Voice && !streamed_target_keys.is_empty() { for target in drained_targets { let key = ChannelReplyTargetKey::from(&target); @@ -189,6 +215,8 @@ async fn deliver_channel_replies_inner( .await; } +/// Format buffered status log entries into a Telegram expandable blockquote HTML. +/// Returns an empty string if there are no entries. fn format_logbook_html(entries: &[String]) -> String { if entries.is_empty() { return String::new(); @@ -269,6 +297,16 @@ fn format_channel_error_message(error_obj: &Value) -> String { format!("⚠️ {title}: {detail}") } +/// Format a user-facing notice announcing that a session was compacted. +/// +/// Shown verbatim to channel users (Telegram, Discord, WhatsApp, etc.) and +/// kept short so small mobile clients don't wrap the whole thing. +/// +/// When `include_settings_hint` is false, the "Change chat.compaction.mode…" +/// footer is omitted so users who have set +/// `chat.compaction.show_settings_hint = false` don't see the repetitive +/// hint on every compaction. Mode + token lines are always included. +/// The LLM retry path never sees this text regardless. fn format_channel_compaction_notice( outcome: &compaction_run::CompactionOutcome, include_settings_hint: bool, @@ -310,6 +348,14 @@ fn format_channel_compaction_notice( } } +/// Send a silent "session compacted" notice to pending channel targets +/// without draining them. +/// +/// Mirrors [`send_retry_status_to_channels`]: the targets are *peeked*, +/// not drained, so the in-flight agent run can still deliver its final +/// reply to them afterward. Uses `send_text_silent` so the channel +/// integration doesn't count it toward user-visible interactive replies +/// (no TTS, no delivery receipts beyond the channel's own). pub(crate) async fn notify_channels_of_compaction( state: &Arc, session_key: &str, @@ -354,6 +400,8 @@ pub(crate) async fn notify_channels_of_compaction( } } +/// Send a short retry status update to pending channel targets without draining +/// them. The final reply (or terminal error) will still use the same targets. pub(crate) async fn send_retry_status_to_channels( state: &Arc, session_key: &str, @@ -502,162 +550,19 @@ async fn deliver_channel_replies_to_targets( streamed_target_keys.contains(&ChannelReplyTargetKey::from(&target)); let to = target.outbound_to().into_owned(); tasks.spawn(async move { - let tts_payload = match desired_reply_medium { - ReplyMedium::Voice => build_tts_payload(&state, &session_key, &target, &text).await, - ReplyMedium::Text => None, - }; - let reply_to = target.message_id.as_deref(); - match target.channel_type { - moltis_channels::ChannelType::Telegram => match tts_payload { - Some(mut payload) => { - let transcript = std::mem::take(&mut payload.text); - - if text_already_streamed { - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - note_delivery_failed(&state, &activity_id).await; - } - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - #[cfg(feature = "telegram")] - let fits_in_caption = transcript.len() - <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; - #[cfg(not(feature = "telegram"))] - let fits_in_caption = false; - - if fits_in_caption { - payload.text = transcript; - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - note_delivery_failed(&state, &activity_id).await; - } - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, &to, &logbook_html, None) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); - } - } else { - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - note_delivery_failed(&state, &activity_id).await; - } - let text_result = if logbook_html.is_empty() { - outbound - .send_text(&target.account_id, &to, &transcript, None) - .await - } else { - outbound - .send_text_with_suffix( - &target.account_id, - &to, - &transcript, - &logbook_html, - None, - ) - .await - }; - if let Err(e) = text_result { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send transcript follow-up: {e}" - ); - note_delivery_failed(&state, &activity_id).await; - } - } - } - }, - None => { - if !deliver_text_fallback( - outbound.as_ref(), - &target, - &to, - &text, - &logbook_html, - reply_to, - text_already_streamed, - ) - .await - { - note_delivery_failed(&state, &activity_id).await; - } - }, - }, - _ => match tts_payload { - Some(payload) => { - if let Err(e) = outbound - .send_media(&target.account_id, &to, &payload, reply_to) - .await - { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel voice reply: {e}" - ); - note_delivery_failed(&state, &activity_id).await; - } - }, - None => { - if !deliver_text_fallback( - outbound.as_ref(), - &target, - &to, - &text, - &logbook_html, - reply_to, - text_already_streamed, - ) - .await - { - note_delivery_failed(&state, &activity_id).await; - } - }, - }, - } + deliver_reply_to_target( + outbound.as_ref(), + &state, + &activity_id, + &session_key, + &target, + &to, + &text, + &logbook_html, + desired_reply_medium, + text_already_streamed, + ) + .await; }); } @@ -670,178 +575,190 @@ async fn deliver_channel_replies_to_targets( } } -async fn deliver_text_fallback( +/// Deliver the final reply to one channel target. +/// +/// A voice reply becomes a TTS media message; if TTS is unavailable, or the +/// medium is text, the reply falls back to plain text. Telegram is special-cased +/// because it can carry the transcript as a caption on the voice note, which +/// saves a second message when the transcript is short enough. +#[allow(clippy::too_many_arguments)] +async fn deliver_reply_to_target( outbound: &dyn moltis_channels::ChannelOutbound, + state: &Arc, + activity_id: &str, + session_key: &str, target: &moltis_channels::ChannelReplyTarget, to: &str, text: &str, logbook_html: &str, - reply_to: Option<&str>, + desired_reply_medium: ReplyMedium, text_already_streamed: bool, -) -> bool { - if text_already_streamed { - if !logbook_html.is_empty() - && let Err(e) = outbound - .send_html(&target.account_id, to, logbook_html, None) - .await +) { + let tts_payload = match desired_reply_medium { + ReplyMedium::Voice => build_tts_payload(state, session_key, target, text).await, + ReplyMedium::Text => None, + }; + let reply_to = target.message_id.as_deref(); + + // TTS disabled or failed β€” deliver the text instead. + let Some(mut payload) = tts_payload else { + if !deliver_text_fallback( + outbound, + target, + to, + text, + logbook_html, + reply_to, + text_already_streamed, + ) + .await { - warn!( - account_id = target.account_id, - chat_id = target.chat_id, - thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send logbook follow-up: {e}" - ); + note_delivery_failed(state, activity_id).await; } - return true; + return; + }; + + if target.channel_type != moltis_channels::ChannelType::Telegram { + send_voice_reply(outbound, state, activity_id, target, to, &payload, reply_to).await; + return; } - let result = if logbook_html.is_empty() { + // Telegram can attach the transcript as a caption, so take it off the + // payload and decide below whether it goes on the voice note or follows it. + let transcript = std::mem::take(&mut payload.text); + + // Text was already delivered via edit-in-place streaming β€” skip the + // caption/follow-up and only send the TTS voice audio. + if text_already_streamed { + send_voice_reply(outbound, state, activity_id, target, to, &payload, reply_to).await; + send_logbook_follow_up(outbound, target, to, logbook_html).await; + return; + } + + // Check if the transcript fits as a Telegram caption (when the feature is + // enabled). With the telegram feature off this is always false, so the + // voice note is followed by the text as a separate message. + #[cfg(feature = "telegram")] + let fits_in_caption = transcript.len() <= moltis_telegram::markdown::TELEGRAM_CAPTION_LIMIT; + #[cfg(not(feature = "telegram"))] + let fits_in_caption = false; + + // Short transcript fits as a caption on the voice message. + if fits_in_caption { + payload.text = transcript; + send_voice_reply(outbound, state, activity_id, target, to, &payload, reply_to).await; + send_logbook_follow_up(outbound, target, to, logbook_html).await; + return; + } + + // Transcript too long for a caption β€” send voice without a caption, then + // the full text as a follow-up. + send_voice_reply(outbound, state, activity_id, target, to, &payload, reply_to).await; + let text_result = if logbook_html.is_empty() { outbound - .send_text(&target.account_id, to, text, reply_to) + .send_text(&target.account_id, to, &transcript, None) .await } else { outbound - .send_text_with_suffix(&target.account_id, to, text, logbook_html, reply_to) + .send_text_with_suffix(&target.account_id, to, &transcript, logbook_html, None) .await }; - if let Err(e) = result { + if let Err(e) = text_result { warn!( account_id = target.account_id, chat_id = target.chat_id, thread_id = target.thread_id.as_deref().unwrap_or("-"), - "failed to send channel reply: {e}" + "failed to send transcript follow-up: {e}" ); - return false; + note_delivery_failed(state, activity_id).await; } - true } -#[derive(Debug, Deserialize)] -struct TtsStatusResponse { - enabled: bool, -} - -#[derive(Debug, Serialize)] -struct TtsConvertRequest<'a> { - text: &'a str, - format: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - provider: Option, - #[serde(skip_serializing_if = "Option::is_none", rename = "voiceId")] - voice_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - model: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct TtsConvertResponse { - audio: String, - #[serde(default)] - mime_type: Option, -} - -pub(crate) async fn generate_tts_audio( +/// Send a TTS voice message, reporting a delivery failure if it does not land. +async fn send_voice_reply( + outbound: &dyn moltis_channels::ChannelOutbound, state: &Arc, - session_key: &str, - text: &str, -) -> error::Result> { - use base64::Engine; - - let tts_status = state - .tts_service() - .status() + activity_id: &str, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + payload: &moltis_common::types::ReplyPayload, + reply_to: Option<&str>, +) { + if let Err(e) = outbound + .send_media(&target.account_id, to, payload, reply_to) .await - .map_err(error::Error::message)?; - let status: TtsStatusResponse = serde_json::from_value(tts_status) - .map_err(|_| error::Error::message("invalid tts.status response"))?; - if !status.enabled { - return Err(error::Error::message("TTS is disabled or not configured")); + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel voice reply: {e}" + ); + note_delivery_failed(state, activity_id).await; } +} - let text = moltis_voice::tts::sanitize_text_for_tts(text); - let text = text.trim(); - if text.is_empty() { - return Err(error::Error::message("response has no speakable text")); +/// Send the buffered status logbook as a separate follow-up message. +/// +/// Best-effort: the reply itself already reached the user, so a lost logbook is +/// not a delivery failure and must not turn the acknowledgment into ❌. +async fn send_logbook_follow_up( + outbound: &dyn moltis_channels::ChannelOutbound, + target: &moltis_channels::ChannelReplyTarget, + to: &str, + logbook_html: &str, +) { + if logbook_html.is_empty() { + return; } - - let (_, session_override) = state.tts_overrides(session_key, "").await; - - let request = TtsConvertRequest { - text, - format: "ogg", - provider: session_override.as_ref().and_then(|o| o.provider.clone()), - voice_id: session_override.as_ref().and_then(|o| o.voice_id.clone()), - model: session_override.as_ref().and_then(|o| o.model.clone()), - }; - - let request_value = serde_json::to_value(request) - .map_err(|_| error::Error::message("failed to build tts.convert request"))?; - let tts_result = state - .tts_service() - .convert(request_value) + if let Err(e) = outbound + .send_html(&target.account_id, to, logbook_html, None) .await - .map_err(error::Error::message)?; - - let response: TtsConvertResponse = serde_json::from_value(tts_result) - .map_err(|_| error::Error::message("invalid tts.convert response"))?; - base64::engine::general_purpose::STANDARD - .decode(&response.audio) - .map_err(|_| error::Error::message("invalid base64 audio returned by TTS provider")) + { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send logbook follow-up: {e}" + ); + } } -async fn build_tts_payload( - state: &Arc, - session_key: &str, +/// Deliver the reply as plain text, or just the logbook when the text was +/// already streamed in place. Returns whether the user received the reply. +async fn deliver_text_fallback( + outbound: &dyn moltis_channels::ChannelOutbound, target: &moltis_channels::ChannelReplyTarget, + to: &str, text: &str, -) -> Option { - use moltis_common::types::{MediaAttachment, ReplyPayload}; - - let tts_status = state.tts_service().status().await.ok()?; - let status: TtsStatusResponse = serde_json::from_value(tts_status).ok()?; - if !status.enabled { - return None; + logbook_html: &str, + reply_to: Option<&str>, + text_already_streamed: bool, +) -> bool { + if text_already_streamed { + send_logbook_follow_up(outbound, target, to, logbook_html).await; + return true; } - // Strip markdown/URLs the LLM may have included β€” use sanitized text - // only for TTS conversion, but keep the original for the caption. - let sanitized = moltis_voice::tts::sanitize_text_for_tts(text); - - let channel_key = format!("{}:{}", target.channel_type.as_str(), target.account_id); - let (channel_override, session_override) = state.tts_overrides(session_key, &channel_key).await; - let resolved = channel_override.or(session_override); - - let request = TtsConvertRequest { - text: &sanitized, - format: "ogg", - provider: resolved.as_ref().and_then(|o| o.provider.clone()), - voice_id: resolved.as_ref().and_then(|o| o.voice_id.clone()), - model: resolved.as_ref().and_then(|o| o.model.clone()), + let result = if logbook_html.is_empty() { + outbound + .send_text(&target.account_id, to, text, reply_to) + .await + } else { + outbound + .send_text_with_suffix(&target.account_id, to, text, logbook_html, reply_to) + .await }; - - let tts_result = state - .tts_service() - .convert(serde_json::to_value(request).ok()?) - .await - .ok()?; - - let response: TtsConvertResponse = serde_json::from_value(tts_result).ok()?; - - let mime_type = response - .mime_type - .unwrap_or_else(|| "audio/ogg".to_string()); - - Some(ReplyPayload { - text: text.to_string(), - media: Some(MediaAttachment { - url: format!("data:{mime_type};base64,{}", response.audio), - mime_type, - filename: None, - }), - reply_to_id: None, - silent: false, - }) + if let Err(e) = result { + warn!( + account_id = target.account_id, + chat_id = target.chat_id, + thread_id = target.thread_id.as_deref().unwrap_or("-"), + "failed to send channel reply: {e}" + ); + return false; + } + true } /// Buffer a tool execution status into the channel status log for a session. @@ -1059,6 +976,8 @@ fn truncate_url(url: &str) -> String { } } +/// Send a screenshot to all pending channel targets for a session. +/// Uses `peek_channel_replies` so targets remain for the final text response. pub(crate) async fn send_screenshot_to_channels( state: &Arc, activity_id: &str, @@ -1134,6 +1053,8 @@ async fn dispatch_screenshot_to_targets( chat_id = target.chat_id, "failed to send screenshot to channel: {e}" ); + // Always tell the user something went wrong β€” a silently + // missing screenshot looks like the tool did nothing. let error_msg = format!("⚠️ Failed to send screenshot: {e}"); let _ = outbound .send_text(&target.account_id, &to, &error_msg, reply_to) @@ -1163,6 +1084,8 @@ async fn dispatch_screenshot_to_targets( delivered } +/// Send a document payload to all pending channel targets for a session. +/// Uses `peek_channel_replies` so targets remain for the final text response. pub(crate) async fn dispatch_document_to_channels( state: &Arc, activity_id: &str, @@ -1388,5 +1311,7 @@ pub(crate) async fn send_location_to_channels( } } +pub(crate) mod tts; + #[cfg(test)] mod tests; diff --git a/crates/chat/src/channels/tests.rs b/crates/chat/src/channels/tests.rs index b33e0a12b8..8301df504e 100644 --- a/crates/chat/src/channels/tests.rs +++ b/crates/chat/src/channels/tests.rs @@ -90,6 +90,7 @@ fn telegram_target() -> ChannelReplyTarget { #[test] fn push_notification_url_uses_chats_prefix_and_replaces_colons() { + // Must match frontend sessionPath(): `/chats/${key.replace(/:/g, "/")}` assert_eq!(push_notification_url("session:42"), "/chats/session/42"); } diff --git a/crates/chat/src/channels/tts.rs b/crates/chat/src/channels/tts.rs new file mode 100644 index 0000000000..86788ddc4c --- /dev/null +++ b/crates/chat/src/channels/tts.rs @@ -0,0 +1,153 @@ +//! Text-to-speech synthesis for channel and web UI replies. +//! +//! Both entry points go through the `tts.status` / `tts.convert` service pair: +//! [`generate_tts_audio`] returns raw bytes for the web UI, while +//! [`build_tts_payload`] wraps them in a channel [`ReplyPayload`] with the +//! original (unsanitized) text kept as the caption. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use crate::{error, runtime::ChatRuntime}; + +#[derive(Debug, Deserialize)] +struct TtsStatusResponse { + enabled: bool, +} + +#[derive(Debug, Serialize)] +struct TtsConvertRequest<'a> { + text: &'a str, + format: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + provider: Option, + #[serde(skip_serializing_if = "Option::is_none", rename = "voiceId")] + voice_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + model: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TtsConvertResponse { + audio: String, + #[serde(default)] + mime_type: Option, +} + +/// Generate TTS audio bytes for a web UI response. +/// +/// Uses the session-level TTS override if configured, otherwise the global TTS +/// config. Returns raw audio bytes (OGG format) on success, and an error if TTS +/// is disabled, the text has nothing speakable, or generation fails. +pub(crate) async fn generate_tts_audio( + state: &Arc, + session_key: &str, + text: &str, +) -> error::Result> { + use base64::Engine; + + let tts_status = state + .tts_service() + .status() + .await + .map_err(error::Error::message)?; + let status: TtsStatusResponse = serde_json::from_value(tts_status) + .map_err(|_| error::Error::message("invalid tts.status response"))?; + if !status.enabled { + return Err(error::Error::message("TTS is disabled or not configured")); + } + + // Layer 2: strip markdown/URLs the LLM may have included despite the prompt. + let text = moltis_voice::tts::sanitize_text_for_tts(text); + let text = text.trim(); + if text.is_empty() { + return Err(error::Error::message("response has no speakable text")); + } + + let (_, session_override) = state.tts_overrides(session_key, "").await; + + let request = TtsConvertRequest { + text, + format: "ogg", + provider: session_override.as_ref().and_then(|o| o.provider.clone()), + voice_id: session_override.as_ref().and_then(|o| o.voice_id.clone()), + model: session_override.as_ref().and_then(|o| o.model.clone()), + }; + + let request_value = serde_json::to_value(request) + .map_err(|_| error::Error::message("failed to build tts.convert request"))?; + let tts_result = state + .tts_service() + .convert(request_value) + .await + .map_err(error::Error::message)?; + + let response: TtsConvertResponse = serde_json::from_value(tts_result) + .map_err(|_| error::Error::message("invalid tts.convert response"))?; + base64::engine::general_purpose::STANDARD + .decode(&response.audio) + .map_err(|_| error::Error::message("invalid base64 audio returned by TTS provider")) +} + +/// Build a voice [`ReplyPayload`] for one channel target, or `None` when TTS is +/// disabled or any step of the conversion fails (the caller then falls back to +/// delivering plain text). +/// +/// The channel-level override wins over the session-level one, since a user who +/// configured a voice for, say, WhatsApp expects it regardless of which session +/// the reply belongs to. +pub(crate) async fn build_tts_payload( + state: &Arc, + session_key: &str, + target: &moltis_channels::ChannelReplyTarget, + text: &str, +) -> Option { + use moltis_common::types::{MediaAttachment, ReplyPayload}; + + let tts_status = state.tts_service().status().await.ok()?; + let status: TtsStatusResponse = serde_json::from_value(tts_status).ok()?; + if !status.enabled { + return None; + } + + // Strip markdown/URLs the LLM may have included β€” use sanitized text + // only for TTS conversion, but keep the original for the caption. + let sanitized = moltis_voice::tts::sanitize_text_for_tts(text); + + let channel_key = format!("{}:{}", target.channel_type.as_str(), target.account_id); + let (channel_override, session_override) = state.tts_overrides(session_key, &channel_key).await; + let resolved = channel_override.or(session_override); + + let request = TtsConvertRequest { + text: &sanitized, + format: "ogg", + provider: resolved.as_ref().and_then(|o| o.provider.clone()), + voice_id: resolved.as_ref().and_then(|o| o.voice_id.clone()), + model: resolved.as_ref().and_then(|o| o.model.clone()), + }; + + let tts_result = state + .tts_service() + .convert(serde_json::to_value(request).ok()?) + .await + .ok()?; + + let response: TtsConvertResponse = serde_json::from_value(tts_result).ok()?; + + let mime_type = response + .mime_type + .unwrap_or_else(|| "audio/ogg".to_string()); + + Some(ReplyPayload { + text: text.to_string(), + media: Some(MediaAttachment { + url: format!("data:{mime_type};base64,{}", response.audio), + mime_type, + filename: None, + }), + reply_to_id: None, + silent: false, + }) +} diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index 6a444f37c9..c1369aedef 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,5 +1,5 @@ mod agent_loop; -pub(crate) mod channel_acks; +pub mod channel_acks; mod channels; mod compaction; mod compaction_run; diff --git a/crates/chat/src/runtime.rs b/crates/chat/src/runtime.rs index 4d05480f7a..c76ca8782d 100644 --- a/crates/chat/src/runtime.rs +++ b/crates/chat/src/runtime.rs @@ -152,9 +152,7 @@ pub trait ChatRuntime: Send + Sync { ) { } - /// Finalize acknowledgment keys directly, for paths where no run executes - /// (rejected hook, early error) and nothing else will signal a terminal. - /// Finalize whichever turn is currently active for this session. + /// Finalize the turn identified by `activity_id`. /// /// For abort, where the run future is killed and cannot emit its own /// terminal. Turn-addressed, so it cannot resolve a turn that starts later. @@ -165,6 +163,8 @@ pub trait ChatRuntime: Send + Sync { ) { } + /// Finalize acknowledgment keys directly, for paths where no run executes + /// (rejected hook, early error) and nothing else will signal a terminal. async fn finalize_channel_acks( &self, _ack_keys: Vec, diff --git a/crates/chat/src/service/chat_impl/queue_drain.rs b/crates/chat/src/service/chat_impl/queue_drain.rs index 1e6f65a59c..ce16ee4ae6 100644 --- a/crates/chat/src/service/chat_impl/queue_drain.rs +++ b/crates/chat/src/service/chat_impl/queue_drain.rs @@ -97,43 +97,48 @@ async fn prepare_collected( queued: Vec, ) -> (Option, Vec) { let multimodal = queued.iter().any(|m| m.params.get("content").is_some()); - if multimodal { - let blocks = merge_content_blocks(&queued); - if blocks.is_empty() { - abandon(state, &queued).await; - return (None, Vec::new()); - } - let Some(last) = queued.last() else { - abandon(state, &queued).await; - return (None, Vec::new()); - }; - info!(session = %session_key, count = queued.len(), "replaying collected messages (multimodal)"); - let mut merged = last.params.clone(); - merged["content"] = Value::Array(blocks); - if let Some(obj) = merged.as_object_mut() { - obj.remove("text"); - obj.remove("message"); - } - merged["_queued_replay"] = Value::Bool(true); - merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( - crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) - ); - let keys = crate::channel_acks::ack_keys_from_params(&merged); - return (Some(merged), keys); - } - let Some(mut merged) = merge_collected_text(&queued) else { + let merged = if multimodal { + merge_collected_content(&queued) + } else { + merge_collected_text(&queued) + }; + let Some(mut merged) = merged else { abandon(state, &queued).await; return (None, Vec::new()); }; - info!(session = %session_key, count = queued.len(), "replaying collected messages"); - merged["_queued_replay"] = Value::Bool(true); - merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!( - crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)) + info!( + session = %session_key, + count = queued.len(), + multimodal, + "replaying collected messages" ); - let keys = crate::channel_acks::ack_keys_from_params(&merged); + + merged["_queued_replay"] = Value::Bool(true); + // One run now answers every collected message, so it owns all of their + // acknowledgments and each inbound message gets the terminal reaction. + let keys = crate::channel_acks::merged_ack_keys(queued.iter().map(|m| &m.params)); + merged[crate::channel_acks::ACK_KEYS_PARAM] = serde_json::json!(keys); (Some(merged), keys) } +/// Merge queued messages into a single multimodal `content` array, keeping the +/// last message's other params (model, session, channel target) as the base. +fn merge_collected_content(queued: &[QueuedMessage]) -> Option { + let blocks = merge_content_blocks(queued); + if blocks.is_empty() { + return None; + } + let mut merged = queued.last()?.params.clone(); + merged["content"] = Value::Array(blocks); + if let Some(object) = merged.as_object_mut() { + // The text is now carried as a content block; leaving either alias in + // place would send it twice. + object.remove("text"); + object.remove("message"); + } + Some(merged) +} + fn message_text(params: &Value) -> Option<&str> { params .get("text") diff --git a/crates/gateway/src/channel_events.rs b/crates/gateway/src/channel_events.rs index 288dbbd6aa..c0c85e3763 100644 --- a/crates/gateway/src/channel_events.rs +++ b/crates/gateway/src/channel_events.rs @@ -334,15 +334,17 @@ fn start_channel_typing_loop( Some(done_tx) } -/// Emoji shortcodes for acknowledgment reactions (Slack-style names). Only -/// channels that populate [`ChannelReplyTarget::ack_message_id`] receive these; -/// today that is Slack, whose Web API expects shortcodes (not raw glyphs). /// Create and register a per-turn acknowledgment reaction controller, keyed by -/// session key. The controller immediately adds πŸ‘€ to the exact inbound message -/// and is later driven through phase emojis and finalized by the agent run. +/// the inbound message's own ack key. The controller immediately adds πŸ‘€ to that +/// exact message and is later driven through phase emojis and finalized by +/// whichever agent run claims it. /// /// No-op unless the channel populated `ack_message_id` (bot directly addressed -/// and reactions enabled) and an outbound implementation is available. +/// and reactions enabled) and an outbound implementation is available. Today +/// that is Slack; other channels leave `ack_message_id` unset and get no +/// reactions. +/// +/// Returns the ack key so the caller can carry it into `chat.send`. async fn register_channel_reaction_controller( state: &Arc, reply_to: &ChannelReplyTarget, diff --git a/crates/gateway/src/channel_events/dispatch.rs b/crates/gateway/src/channel_events/dispatch.rs index 6f4d36be66..fae784c5c2 100644 --- a/crates/gateway/src/channel_events/dispatch.rs +++ b/crates/gateway/src/channel_events/dispatch.rs @@ -146,7 +146,7 @@ pub(in crate::channel_events) async fn dispatch_to_chat( // reaction follows the message itself β€” through queueing and replay β€” // rather than whatever else shares the session. if let Some(ref key) = ack_key { - params["_ack_keys"] = serde_json::json!([key]); + params[moltis_chat::channel_acks::ACK_KEYS_PARAM] = serde_json::json!([key]); } // Attach thread context if available. diff --git a/crates/httpd/src/server/gateway.rs b/crates/httpd/src/server/gateway.rs index 97cc914d13..68ca4caf78 100644 --- a/crates/httpd/src/server/gateway.rs +++ b/crates/httpd/src/server/gateway.rs @@ -24,11 +24,6 @@ use super::{ PreparedGateway, RouteEnhancer, builder::finalize_gateway_app, runtime::FinalizeGatewayArgs, }; -#[cfg(feature = "slack")] -fn slack_json_response(status: StatusCode, body: serde_json::Value) -> axum::response::Response { - (status, Json(body)).into_response() -} - #[cfg(feature = "tailscale")] use super::TailscaleOpts; @@ -383,217 +378,49 @@ pub async fn prepare_gateway( } #[cfg(feature = "slack")] { - let slack_callback_dispatcher = super::slack_callbacks::SlackCallbackDispatcher::start( - Arc::clone(&slack_webhook_plugin), - ); - let slack_events_plugin = Arc::clone(&slack_webhook_plugin); - let state_for_slack_events = Arc::clone(&state); - let dispatcher_for_slack_events = Arc::clone(&slack_callback_dispatcher); - app = app.route( - "/api/channels/slack/{account_id}/events", - axum::routing::post( - move |axum::extract::Path(account_id): axum::extract::Path, - headers: axum::http::HeaderMap, - body: axum::body::Bytes| { - let plugin = Arc::clone(&slack_events_plugin); - let gw_state = Arc::clone(&state_for_slack_events); - let dispatcher = Arc::clone(&dispatcher_for_slack_events); - async move { - let verifier = { - let p = plugin.read().await; - p.channel_webhook_verifier(&account_id) - }; - let Some(verifier) = verifier else { - return slack_json_response( - StatusCode::NOT_FOUND, - serde_json::json!({ "ok": false, "error": "unknown Slack account" }), - ); - }; - match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( - verifier.as_ref(), - &gw_state.channel_webhook_rate_limiter, - &account_id, - "events", - &headers, - &body, - ) { - Err(rejection) => { - crate::channel_webhook_middleware::rejection_into_response( - rejection, - ) - }, - Ok(verified) => { - let payload: serde_json::Value = - match serde_json::from_slice(&verified.body) { - Ok(payload) => payload, - Err(error) => { - return slack_json_response( - StatusCode::BAD_REQUEST, - serde_json::json!({ "ok": false, "error": error.to_string() }), - ); - }, - }; - if payload.get("type").and_then(|value| value.as_str()) - == Some("url_verification") - { - return match payload - .get("challenge") - .and_then(|value| value.as_str()) - { - Some(challenge) => slack_json_response( - StatusCode::OK, - serde_json::json!({ "challenge": challenge }), - ), - None => slack_json_response( - StatusCode::BAD_REQUEST, - serde_json::json!({ "ok": false, "error": "missing Slack challenge" }), - ), - }; - } - match dispatcher.admit( - &gw_state.channel_webhook_dedup, - account_id, - verified.idempotency_key.as_deref(), - super::slack_callbacks::SlackCallbackKind::Event, - verified.body, - ) { - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate => slack_json_response( - StatusCode::OK, - serde_json::json!({ "ok": true, "deduplicated": true }), - ), - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => slack_json_response( - StatusCode::OK, - serde_json::json!({ "ok": true }), - ), - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( - StatusCode::SERVICE_UNAVAILABLE, - serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), - ), - } - }, - } - } - }, + use super::slack_callbacks::{ + SlackCallbackDispatcher, SlackCallbackKind, handle_slack_callback, + }; + + // One dispatcher (bounded queue + fair worker pool) shared by all three + // endpoints, so a burst on one cannot starve the others. + let dispatcher = SlackCallbackDispatcher::start(Arc::clone(&slack_webhook_plugin)); + for (path, kind) in [ + ( + "/api/channels/slack/{account_id}/events", + SlackCallbackKind::Event, ), - ); - let slack_interact_plugin = Arc::clone(&slack_webhook_plugin); - let state_for_slack_interact = Arc::clone(&state); - let dispatcher_for_slack_interact = Arc::clone(&slack_callback_dispatcher); - app = app.route( - "/api/channels/slack/{account_id}/interactions", - axum::routing::post( - move |axum::extract::Path(account_id): axum::extract::Path, - headers: axum::http::HeaderMap, - body: axum::body::Bytes| { - let plugin = Arc::clone(&slack_interact_plugin); - let gw_state = Arc::clone(&state_for_slack_interact); - let dispatcher = Arc::clone(&dispatcher_for_slack_interact); - async move { - let verifier = { - let p = plugin.read().await; - p.channel_webhook_verifier(&account_id) - }; - let Some(verifier) = verifier else { - return slack_json_response( - StatusCode::NOT_FOUND, - serde_json::json!({ "ok": false, "error": "unknown Slack account" }), - ); - }; - match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( - verifier.as_ref(), - &gw_state.channel_webhook_rate_limiter, - &account_id, - "interactions", - &headers, - &body, - ) { - Err(rejection) => { - crate::channel_webhook_middleware::rejection_into_response( - rejection, - ) - }, - Ok(verified) => match dispatcher.admit( - &gw_state.channel_webhook_dedup, - account_id, - verified.idempotency_key.as_deref(), - super::slack_callbacks::SlackCallbackKind::Interaction, - verified.body, - ) { - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate => slack_json_response( - StatusCode::OK, - serde_json::json!({ "ok": true, "deduplicated": true }), - ), - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => slack_json_response( - StatusCode::OK, - serde_json::json!({ "ok": true }), - ), - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( - StatusCode::SERVICE_UNAVAILABLE, - serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), - ), - }, - } - } - }, + ( + "/api/channels/slack/{account_id}/interactions", + SlackCallbackKind::Interaction, ), - ); - let slack_cmd_plugin = Arc::clone(&slack_webhook_plugin); - let state_for_slack_cmd = Arc::clone(&state); - let dispatcher_for_slack_cmd = Arc::clone(&slack_callback_dispatcher); - app = app.route( - "/api/channels/slack/{account_id}/commands", - axum::routing::post( - move |axum::extract::Path(account_id): axum::extract::Path, - headers: axum::http::HeaderMap, - body: axum::body::Bytes| { - let plugin = Arc::clone(&slack_cmd_plugin); - let gw_state = Arc::clone(&state_for_slack_cmd); - let dispatcher = Arc::clone(&dispatcher_for_slack_cmd); - async move { - let verifier = { - let p = plugin.read().await; - p.channel_webhook_verifier(&account_id) - }; - let Some(verifier) = verifier else { - return slack_json_response( - StatusCode::NOT_FOUND, - serde_json::json!({ "ok": false, "error": "unknown Slack account" }), - ); - }; - match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( - verifier.as_ref(), - &gw_state.channel_webhook_rate_limiter, - &account_id, - "commands", - &headers, - &body, - ) { - Err(rejection) => { - crate::channel_webhook_middleware::rejection_into_response( - rejection, - ) - }, - Ok(verified) => match dispatcher.admit( - &gw_state.channel_webhook_dedup, - account_id, - verified.idempotency_key.as_deref(), - super::slack_callbacks::SlackCallbackKind::Command, - verified.body, - ) { - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Duplicate - | moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Admitted => { - StatusCode::OK.into_response() - }, - moltis_gateway::channel_webhook_dedup::ChannelWebhookAdmission::Rejected => slack_json_response( - StatusCode::SERVICE_UNAVAILABLE, - serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), - ), - }, - } - } - }, + ( + "/api/channels/slack/{account_id}/commands", + SlackCallbackKind::Command, ), - ); + ] { + let plugin = Arc::clone(&slack_webhook_plugin); + let gateway_state = Arc::clone(&state); + let dispatcher = Arc::clone(&dispatcher); + app = app.route( + path, + axum::routing::post( + move |axum::extract::Path(account_id): axum::extract::Path, + headers: axum::http::HeaderMap, + body: axum::body::Bytes| { + handle_slack_callback( + Arc::clone(&plugin), + Arc::clone(&dispatcher), + Arc::clone(&gateway_state), + kind, + account_id, + headers, + body, + ) + }, + ), + ); + } } #[cfg(feature = "telephony")] { diff --git a/crates/httpd/src/server/slack_callbacks.rs b/crates/httpd/src/server/slack_callbacks.rs index 1b100fb50c..3985983e86 100644 --- a/crates/httpd/src/server/slack_callbacks.rs +++ b/crates/httpd/src/server/slack_callbacks.rs @@ -6,9 +6,16 @@ use std::{ }; use { + axum::{ + http::StatusCode, + response::{IntoResponse, Json, Response}, + }, bytes::Bytes, futures::FutureExt, - moltis_gateway::channel_webhook_dedup::{ChannelWebhookAdmission, ChannelWebhookDedupeStore}, + moltis_gateway::{ + channel_webhook_dedup::{ChannelWebhookAdmission, ChannelWebhookDedupeStore}, + state::GatewayState, + }, tokio::{ sync::{OwnedSemaphorePermit, Semaphore, mpsc}, task::JoinSet, @@ -19,6 +26,11 @@ const CALLBACK_QUEUE_CAPACITY: usize = 256; const CALLBACK_WORKER_LIMIT: usize = 16; const CALLBACK_ACCOUNT_CAPACITY: usize = CALLBACK_QUEUE_CAPACITY / CALLBACK_WORKER_LIMIT; +/// Slack callback endpoints served under `/api/channels/slack/{account_id}/`. +/// +/// All three share the same admission pipeline (verify β†’ rate limit β†’ dedupe β†’ +/// bounded queue) and differ only in how they acknowledge, so the variant is +/// what parameterizes [`handle_slack_callback`]. #[derive(Clone, Copy)] pub(super) enum SlackCallbackKind { Event, @@ -34,6 +46,127 @@ impl SlackCallbackKind { Self::Command => "commands", } } + + /// Acknowledge an admission decision. + /// + /// Slash commands render their response body in the Slack client, so an + /// accepted (or deduped) command answers with a bare 200 rather than JSON + /// the user would see. Events and interactions are machine-read, so they + /// report the decision as JSON. + fn ack_response(self, admission: ChannelWebhookAdmission) -> Response { + match (self, admission) { + ( + Self::Command, + ChannelWebhookAdmission::Admitted | ChannelWebhookAdmission::Duplicate, + ) => StatusCode::OK.into_response(), + (_, ChannelWebhookAdmission::Admitted) => { + slack_json_response(StatusCode::OK, serde_json::json!({ "ok": true })) + }, + (_, ChannelWebhookAdmission::Duplicate) => slack_json_response( + StatusCode::OK, + serde_json::json!({ "ok": true, "deduplicated": true }), + ), + (_, ChannelWebhookAdmission::Rejected) => slack_json_response( + StatusCode::SERVICE_UNAVAILABLE, + serde_json::json!({ "ok": false, "error": "Slack callback queue unavailable" }), + ), + } + } +} + +fn slack_json_response(status: StatusCode, body: serde_json::Value) -> Response { + (status, Json(body)).into_response() +} + +/// Verify a Slack callback and hand it to the bounded queue. +/// +/// Slack expects an acknowledgement within three seconds, so processing never +/// happens inline: the request is verified (HMAC, timestamp, rate limit), +/// deduplicated, queued, and acknowledged, and the queue's workers run the +/// actual ingest afterwards. A queue that cannot accept the callback answers +/// 503 so Slack retries rather than dropping the event silently. +pub(super) async fn handle_slack_callback( + plugin: Arc>, + dispatcher: Arc, + gateway_state: Arc, + kind: SlackCallbackKind, + account_id: String, + headers: axum::http::HeaderMap, + body: Bytes, +) -> Response { + use moltis_channels::ChannelPlugin; + + let verifier = { + let plugin = plugin.read().await; + plugin.channel_webhook_verifier(&account_id) + }; + let Some(verifier) = verifier else { + return slack_json_response( + StatusCode::NOT_FOUND, + serde_json::json!({ "ok": false, "error": "unknown Slack account" }), + ); + }; + + let verified = match moltis_gateway::channel_webhook_middleware::verify_channel_webhook( + verifier.as_ref(), + &gateway_state.channel_webhook_rate_limiter, + &account_id, + kind.endpoint(), + &headers, + &body, + ) { + Ok(verified) => verified, + Err(rejection) => { + return crate::channel_webhook_middleware::rejection_into_response(rejection); + }, + }; + + if matches!(kind, SlackCallbackKind::Event) + && let Some(response) = event_preflight(&verified.body) + { + return response; + } + + kind.ack_response(dispatcher.admit( + &gateway_state.channel_webhook_dedup, + account_id, + verified.idempotency_key.as_deref(), + kind, + verified.body, + )) +} + +/// Handle the two events-endpoint cases that must answer synchronously instead +/// of being queued: Slack's one-time `url_verification` handshake, which has to +/// echo the challenge back in the response body, and a malformed payload, which +/// is rejected rather than queued for a worker that could only discard it. +/// +/// Returns `None` when the body is a normal event envelope. +fn event_preflight(body: &[u8]) -> Option { + let payload: serde_json::Value = match serde_json::from_slice(body) { + Ok(payload) => payload, + Err(error) => { + return Some(slack_json_response( + StatusCode::BAD_REQUEST, + serde_json::json!({ "ok": false, "error": error.to_string() }), + )); + }, + }; + if payload.get("type").and_then(serde_json::Value::as_str) != Some("url_verification") { + return None; + } + Some( + match payload.get("challenge").and_then(serde_json::Value::as_str) { + Some(challenge) => slack_json_response( + StatusCode::OK, + serde_json::json!({ "challenge": challenge }), + ), + None => slack_json_response( + StatusCode::BAD_REQUEST, + serde_json::json!({ "ok": false, "error": "missing Slack challenge" }), + ), + }, + ) } struct SlackCallbackJob { @@ -440,6 +573,85 @@ mod tests { ) } + /// Status of a preflight that must answer synchronously rather than queue. + fn preflight_status(body: &[u8]) -> Option { + event_preflight(body).map(|response| response.status()) + } + + #[test] + fn url_verification_echoes_the_challenge_before_queueing() { + assert_eq!( + preflight_status(br#"{"type":"url_verification","challenge":"abc123"}"#), + Some(StatusCode::OK) + ); + } + + #[test] + fn url_verification_without_a_challenge_is_rejected() { + assert_eq!( + preflight_status(br#"{"type":"url_verification"}"#), + Some(StatusCode::BAD_REQUEST) + ); + } + + #[test] + fn malformed_event_body_is_rejected_instead_of_queued() { + assert_eq!(preflight_status(b"not json"), Some(StatusCode::BAD_REQUEST)); + } + + #[test] + fn a_normal_event_envelope_falls_through_to_the_queue() { + assert!( + event_preflight(br#"{"type":"event_callback","event":{"type":"message"}}"#).is_none() + ); + } + + #[test] + fn slash_commands_acknowledge_with_an_empty_body() { + // A JSON body here would be rendered to the user in the Slack client. + for admission in [ + ChannelWebhookAdmission::Admitted, + ChannelWebhookAdmission::Duplicate, + ] { + let response = SlackCallbackKind::Command.ack_response(admission); + assert_eq!(response.status(), StatusCode::OK); + assert!( + response + .headers() + .get(axum::http::header::CONTENT_TYPE) + .is_none() + ); + } + } + + #[test] + fn machine_read_endpoints_acknowledge_with_json() { + for kind in [SlackCallbackKind::Event, SlackCallbackKind::Interaction] { + let response = kind.ack_response(ChannelWebhookAdmission::Admitted); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + response.headers().get(axum::http::header::CONTENT_TYPE), + Some(&axum::http::HeaderValue::from_static("application/json")) + ); + } + } + + #[test] + fn a_saturated_queue_asks_slack_to_retry_on_every_endpoint() { + // 503 (not 200) so Slack redelivers instead of dropping the callback. + for kind in [ + SlackCallbackKind::Event, + SlackCallbackKind::Interaction, + SlackCallbackKind::Command, + ] { + assert_eq!( + kind.ack_response(ChannelWebhookAdmission::Rejected) + .status(), + StatusCode::SERVICE_UNAVAILABLE + ); + } + } + fn prior_worker_index(account_id: &str) -> usize { use std::hash::{DefaultHasher, Hash, Hasher}; diff --git a/crates/slack/Cargo.toml b/crates/slack/Cargo.toml index 5c0e6bfe44..d7d45a72c0 100644 --- a/crates/slack/Cargo.toml +++ b/crates/slack/Cargo.toml @@ -7,6 +7,7 @@ version.workspace = true async-trait = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } +form_urlencoded = { workspace = true } hmac = { workspace = true } http = { workspace = true } moltis-channels = { workspace = true } diff --git a/crates/slack/src/channel_webhook_verifier.rs b/crates/slack/src/channel_webhook_verifier.rs index 6d6f968590..172b1a1d25 100644 --- a/crates/slack/src/channel_webhook_verifier.rs +++ b/crates/slack/src/channel_webhook_verifier.rs @@ -31,31 +31,38 @@ impl SlackChannelWebhookVerifier { } } -fn decode_form_value(value: &str) -> Option { - let mut decoded = Vec::with_capacity(value.len()); - let mut bytes = value.bytes(); - while let Some(byte) = bytes.next() { - match byte { - b'%' => { - let pair = [bytes.next()?, bytes.next()?]; - let hex = std::str::from_utf8(&pair).ok()?; - decoded.push(u8::from_str_radix(hex, 16).ok()?); - }, - b'+' => decoded.push(b' '), - other => decoded.push(other), - } - } - String::from_utf8(decoded).ok() +/// Read one field out of an `application/x-www-form-urlencoded` body. +/// +/// Slack posts slash commands and interactions as form bodies, so `+` means +/// space and values are percent-encoded β€” decoding is delegated to +/// `form_urlencoded` rather than hand-rolled. +fn form_value(body: &[u8], name: &str) -> Option { + form_urlencoded::parse(body) + .find(|(key, _)| key == name) + .map(|(_, value)| value.into_owned()) } -fn form_value(body: &[u8], name: &str) -> Option { - let body = std::str::from_utf8(body).ok()?; - body.split('&').find_map(|pair| { - let (key, value) = pair.split_once('=')?; - (key == name).then(|| decode_form_value(value)).flatten() - }) +/// Whether a decoded value is usable as a Slack trigger id. +/// +/// `form_urlencoded` decodes leniently, so a body that did not decode cleanly +/// still yields a value: a malformed escape like `%GG` survives verbatim, and +/// invalid UTF-8 becomes U+FFFD. Real trigger ids are dot-separated +/// alphanumerics (`13345224609.738474920.8088930838d88f008e0`) and contain +/// neither, so their presence means the value is decode garbage. Admitting +/// garbage as a dedupe key would let unrelated callbacks collide on it and be +/// dropped as duplicates; rejecting it only means this callback is never +/// deduplicated, which is the safe direction. +fn is_trigger_id(value: &str) -> bool { + !value.is_empty() && !value.contains(['%', char::REPLACEMENT_CHARACTER]) } +/// Derive a dedupe key for a Slack callback. +/// +/// Each endpoint carries its own unique id, in decreasing order of specificity: +/// Events API envelopes have `event_id` (JSON), slash commands have a top-level +/// `trigger_id` (form), and interactions nest `trigger_id` inside the JSON +/// `payload` field (form). Returning `None` means the callback cannot be +/// deduplicated and will be admitted on every delivery. fn idempotency_key(body: &[u8]) -> Option { if let Some(event_id) = serde_json::from_slice::(body) .ok() @@ -64,14 +71,14 @@ fn idempotency_key(body: &[u8]) -> Option { return Some(event_id); } - if let Some(trigger_id) = form_value(body, "trigger_id").filter(|value| !value.is_empty()) { + if let Some(trigger_id) = form_value(body, "trigger_id").filter(|value| is_trigger_id(value)) { return Some(trigger_id); } form_value(body, "payload") .and_then(|payload| serde_json::from_str::(&payload).ok()) .and_then(|value| value["trigger_id"].as_str().map(ToOwned::to_owned)) - .filter(|value| !value.is_empty()) + .filter(|value| is_trigger_id(value)) } impl ChannelWebhookVerifier for SlackChannelWebhookVerifier { @@ -271,6 +278,17 @@ mod tests { assert!(envelope.idempotency_key.is_none()); } + #[test] + fn form_values_decode_plus_as_space_and_percent_escapes() { + // Slack posts commands as application/x-www-form-urlencoded, where a + // space is `+` β€” not `%20`. Treating `+` literally would corrupt every + // multi-word slash command argument. + let body = b"command=%2Fmoltis&text=hello+world%21&trigger_id=1.2.abc"; + assert_eq!(form_value(body, "command").as_deref(), Some("/moltis")); + assert_eq!(form_value(body, "text").as_deref(), Some("hello world!")); + assert_eq!(form_value(body, "missing"), None); + } + #[test] fn channel_type_is_slack() { let verifier = SlackChannelWebhookVerifier::new(Secret::new(TEST_SECRET.into())); From 88c48906e41c6feb5e787ff716368aaf16b7512b Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 15:29:54 -0700 Subject: [PATCH 21/23] fix(slack): stop client-supplied ack keys, isolate callback panics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects in the reaction/callback work on this branch. `_ack_keys` was trusted from the wire. The `chat.send` RPC handler cloned client params and only *added* `_conn_id`/`_accept_language`/ `_remote_ip`/`_timezone`, so every other `_`-prefixed param a WebSocket client sent survived into the chat service β€” including the ack keys the channel dispatch path mints to name an inbound Slack message's acknowledgment-reaction slot. One authenticated user could therefore drive the πŸ‘€/βœ…/❌ reactions on an unrelated message in any workspace the bot is in. The strip is targeted rather than a blanket removal of `_`-prefixed keys, because that prefix means "internal plumbing", not "server-only": the web UI legitimately sends `_session_key`, `_audio_filename` and `_document_files`. Only `_ack_keys` confers authority a client cannot hold, and only the channel dispatch path needs it β€” that path calls the chat service directly rather than through this registry, so removing it here cannot affect it. Folded the duplicated context injection from `chat.send` and `chat.send_sync` into one helper while doing it, and locked in that `_conn_id` is server-set with a test. (`_channel_reply_target` is spoofable the same way and is the more serious of the two β€” it would let a client address an outbound message to any chat the bot can reach. It predates this branch, so it is not fixed here; it needs its own change and a decision about whether web-originated sends may ever name a channel target.) A panicking Socket Mode callback silenced the bot permanently. There is a single worker draining that queue and `process(job).await` was unguarded, so one panic killed the task and every later callback with it β€” the bot would go quiet with only the panic in the log to explain why. The sibling HTTP callback workers already catch this; Socket Mode now matches, running each job on its own task so the unwind is localized while awaiting it immediately keeps callbacks strictly ordered. Block Kit headings leaked their markdown syntax. A `header` block's text is a `plain_text` field that Slack renders verbatim, but `heading_text` handed it the raw heading, so `## Install \`foo\`` reached users with the backticks visible β€” and LLM headings carry inline formatting constantly. Only unambiguously paired constructs are unwrapped (links keep their label; inline code, bold and strikethrough lose their delimiters). Single `*`/`_` are left alone on purpose: in a heading they are far likelier to be part of an identifier (`API_KEY`, `snake_case`) than an emphasis marker, so stripping them would corrupt the text instead of tidying it. --- crates/gateway/src/methods/services/core.rs | 152 +++++++++++++++----- crates/slack/src/blocks.rs | 90 +++++++++++- crates/slack/src/callback_worker.rs | 59 +++++++- 3 files changed, 261 insertions(+), 40 deletions(-) diff --git a/crates/gateway/src/methods/services/core.rs b/crates/gateway/src/methods/services/core.rs index 95213dcf98..97fa27f8fe 100644 --- a/crates/gateway/src/methods/services/core.rs +++ b/crates/gateway/src/methods/services/core.rs @@ -1,5 +1,40 @@ use super::*; +/// Prepare client-supplied `chat.send`/`chat.send_sync` params for the chat +/// service: drop anything the client must not be able to claim, then inject the +/// connection's own context. +/// +/// The `_`-prefixed params are internal plumbing, and some of them are set +/// legitimately by the web UI (`_session_key`, `_audio_filename`, +/// `_document_files`), so this is a targeted removal rather than a blanket +/// strip: `_ack_keys` names a *channel message's* acknowledgment-reaction slot +/// and is only ever minted by the channel dispatch path (which calls the chat +/// service directly, not through this registry). Letting a WebSocket client +/// send it would let one authenticated user drive the βœ…/❌ reaction on an +/// unrelated inbound Slack message. +async fn prepare_chat_send_params(ctx: &MethodContext) -> serde_json::Value { + let mut params = ctx.params.clone(); + if let Some(object) = params.as_object_mut() { + object.remove(moltis_chat::channel_acks::ACK_KEYS_PARAM); + } + params["_conn_id"] = serde_json::json!(ctx.client_conn_id); + + // Forward client Accept-Language, public remote IP, and timezone. + let registry = ctx.state.client_registry.read().await; + if let Some(client) = registry.clients.get(&ctx.client_conn_id) { + if let Some(ref lang) = client.accept_language { + params["_accept_language"] = serde_json::json!(lang); + } + if let Some(ref ip) = client.remote_ip { + params["_remote_ip"] = serde_json::json!(ip); + } + if let Some(ref tz) = client.timezone { + params["_timezone"] = serde_json::json!(tz); + } + } + params +} + pub(super) fn register(reg: &mut MethodRegistry) { // Config reg.register( @@ -533,30 +568,13 @@ pub(super) fn register(reg: &mut MethodRegistry) { }), ); - // Chat (uses chat_override if set, otherwise falls back to services.chat) - // Inject _conn_id and _accept_language so the chat service can resolve - // the active session and forward the user's locale to web tools. + // Chat (uses chat_override if set, otherwise falls back to services.chat). + // See `prepare_chat_send_params` for what is stripped and injected. reg.register( "chat.send", Box::new(|ctx| { Box::pin(async move { - let mut params = ctx.params.clone(); - params["_conn_id"] = serde_json::json!(ctx.client_conn_id); - // Forward client Accept-Language, public remote IP, and timezone. - { - let registry = ctx.state.client_registry.read().await; - if let Some(client) = registry.clients.get(&ctx.client_conn_id) { - if let Some(ref lang) = client.accept_language { - params["_accept_language"] = serde_json::json!(lang); - } - if let Some(ref ip) = client.remote_ip { - params["_remote_ip"] = serde_json::json!(ip); - } - if let Some(ref tz) = client.timezone { - params["_timezone"] = serde_json::json!(tz); - } - } - } + let params = prepare_chat_send_params(&ctx).await; ctx.state .chat() .send(params) @@ -569,22 +587,7 @@ pub(super) fn register(reg: &mut MethodRegistry) { "chat.send_sync", Box::new(|ctx| { Box::pin(async move { - let mut params = ctx.params.clone(); - params["_conn_id"] = serde_json::json!(ctx.client_conn_id); - { - let registry = ctx.state.client_registry.read().await; - if let Some(client) = registry.clients.get(&ctx.client_conn_id) { - if let Some(ref lang) = client.accept_language { - params["_accept_language"] = serde_json::json!(lang); - } - if let Some(ref ip) = client.remote_ip { - params["_remote_ip"] = serde_json::json!(ip); - } - if let Some(ref tz) = client.timezone { - params["_timezone"] = serde_json::json!(tz); - } - } - } + let params = prepare_chat_send_params(&ctx).await; ctx.state .chat() .send_sync(params) @@ -1308,3 +1311,80 @@ pub(super) fn register(reg: &mut MethodRegistry) { ); } } + +#[cfg(test)] +mod tests { + use super::*; + + use crate::{ + auth::{AuthMode, ResolvedAuth}, + services::GatewayServices, + state::GatewayState, + }; + + fn context(params: serde_json::Value) -> MethodContext { + MethodContext { + request_id: "test".into(), + method: "chat.send".into(), + params, + client_conn_id: "conn-1".into(), + client_role: "operator".into(), + client_scopes: vec!["operator.write".to_string()], + state: GatewayState::new( + ResolvedAuth { + mode: AuthMode::Token, + token: None, + password: None, + }, + GatewayServices::noop(), + ), + channel: None, + } + } + + #[tokio::test] + async fn client_supplied_ack_keys_are_stripped() { + // A WebSocket client must not be able to claim the acknowledgment slot + // of an inbound channel message it has nothing to do with. + let params = prepare_chat_send_params(&context(serde_json::json!({ + "text": "hi", + moltis_chat::channel_acks::ACK_KEYS_PARAM: ["slack-acct:C123:1700000000.1"], + }))) + .await; + + assert!( + params + .get(moltis_chat::channel_acks::ACK_KEYS_PARAM) + .is_none() + ); + assert_eq!(params["text"], "hi"); + } + + #[tokio::test] + async fn connection_context_is_injected_and_not_client_controlled() { + let params = prepare_chat_send_params(&context(serde_json::json!({ + "text": "hi", + "_conn_id": "someone-elses-connection", + }))) + .await; + + assert_eq!(params["_conn_id"], "conn-1"); + } + + #[tokio::test] + async fn legitimate_internal_params_survive() { + // The web UI sets these itself; stripping them would break uploads and + // explicit session targeting. + let params = prepare_chat_send_params(&context(serde_json::json!({ + "text": "hi", + "_session_key": "session:42", + "_audio_filename": "voice.ogg", + "_document_files": [{ "name": "a.pdf" }], + }))) + .await; + + assert_eq!(params["_session_key"], "session:42"); + assert_eq!(params["_audio_filename"], "voice.ogg"); + assert_eq!(params["_document_files"][0]["name"], "a.pdf"); + } +} diff --git a/crates/slack/src/blocks.rs b/crates/slack/src/blocks.rs index 86c53b81a8..9e822172db 100644 --- a/crates/slack/src/blocks.rs +++ b/crates/slack/src/blocks.rs @@ -83,7 +83,7 @@ pub fn markdown_to_blocks(markdown: &str) -> Option> { blocks.push(section_block(&chunk)); } } else { - blocks.push(header_block(&text)); + blocks.push(header_block(&strip_inline_markdown(&text))); } continue; } @@ -138,6 +138,63 @@ fn heading_text(line: &str) -> Option { } } +/// Reduce a heading to plain text for a `header` block. +/// +/// A header's text is a `plain_text` field, which Slack renders verbatim β€” it +/// does not interpret mrkdwn. Headings from an LLM routinely carry inline +/// formatting (`## Install \`foo\``, `## **Important**`), and passing that +/// through unchanged shows the raw syntax to the user. +/// +/// Only unambiguously paired constructs are unwrapped: links keep their label, +/// and inline code, bold and strikethrough lose their delimiters. Single `*`/`_` +/// are deliberately left alone β€” in a heading they are far more likely to be +/// part of an identifier (`API_KEY`, `snake_case`) than an emphasis marker, and +/// stripping them would corrupt the text rather than tidy it. +fn strip_inline_markdown(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + + while let Some(index) = rest.find(['`', '[', '*', '~']) { + let (head, tail) = rest.split_at(index); + out.push_str(head); + + // `label` of a `[label](url)` link, dropping the target. + if let Some(after) = tail.strip_prefix('[') + && let Some((label, remainder)) = after.split_once("](") + && let Some((_url, remainder)) = remainder.split_once(')') + { + out.push_str(&strip_inline_markdown(label)); + rest = remainder; + continue; + } + + // Paired `code`, **bold**, ~~strike~~ β€” delimiters removed, body kept. + let delimiter = match tail.as_bytes() { + [b'`', ..] => "`", + [b'*', b'*', ..] => "**", + [b'~', b'~', ..] => "~~", + _ => "", + }; + if !delimiter.is_empty() + && let Some(after) = tail.strip_prefix(delimiter) + && let Some((body, remainder)) = after.split_once(delimiter) + { + out.push_str(&strip_inline_markdown(body)); + rest = remainder; + continue; + } + + // Unpaired marker: keep it verbatim and move past it. + let mut chars = tail.chars(); + if let Some(ch) = chars.next() { + out.push(ch); + } + rest = chars.as_str(); + } + out.push_str(rest); + out +} + /// Build a header block. Callers must ensure `text` fits [`MAX_HEADER_CHARS`]; /// longer headings are rendered as sections so nothing is lost. fn header_block(text: &str) -> Value { @@ -290,6 +347,37 @@ mod tests { ); } + #[test] + fn heading_inline_formatting_is_not_shown_as_raw_syntax() { + // A header block's text is `plain_text`: Slack shows it verbatim, so + // markdown delimiters must not reach it. + let blocks = markdown_to_blocks("## Install `foo` and **restart**").unwrap(); + assert_eq!(blocks[0]["type"], "header"); + assert_eq!(blocks[0]["text"]["text"], "Install foo and restart"); + } + + #[test] + fn heading_link_keeps_its_label_and_drops_the_target() { + let blocks = markdown_to_blocks("# See [the docs](https://example.com)").unwrap(); + assert_eq!(blocks[0]["text"]["text"], "See the docs"); + } + + #[test] + fn heading_identifiers_with_underscores_and_stars_survive_intact() { + // Single `*`/`_` in a heading are far more likely to be part of an + // identifier than an emphasis marker; stripping them would corrupt it. + for heading in ["API_KEY setup", "glob a*b", "snake_case_name"] { + let blocks = markdown_to_blocks(&format!("# {heading}")).unwrap(); + assert_eq!(blocks[0]["text"]["text"], heading, "{heading}"); + } + } + + #[test] + fn unpaired_heading_delimiters_are_left_verbatim() { + let blocks = markdown_to_blocks("# 50% off `unclosed").unwrap(); + assert_eq!(blocks[0]["text"]["text"], "50% off `unclosed"); + } + #[test] fn over_limit_falls_back_to_none() { // Many dividers exceed the 50-block cap β†’ fall back to plain text. diff --git a/crates/slack/src/callback_worker.rs b/crates/slack/src/callback_worker.rs index d60600657f..dfd027b471 100644 --- a/crates/slack/src/callback_worker.rs +++ b/crates/slack/src/callback_worker.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{future::Future, sync::Arc}; use { moltis_channels::plugin::{ChannelEventSink, ChannelReplyTarget}, @@ -121,9 +121,11 @@ async fn run_worker( loop { let job = tokio::select! { () = cancel.cancelled() => { + // Drain what was already admitted before shutting down: those + // callbacks were acknowledged to Slack and will not be redelivered. receiver.close(); while let Some(job) = receiver.recv().await { - process(job).await; + process_isolated(job).await; } break; }, @@ -132,7 +134,28 @@ async fn run_worker( None => break, }, }; - process(job).await; + process_isolated(job).await; + } +} + +async fn process_isolated(job: CallbackJob) { + run_isolated(process(job)).await; +} + +/// Run one callback so a panic inside it cannot take the worker down. +/// +/// There is a single worker behind this queue, so an unguarded panic would stop +/// *all* subsequent Socket Mode callbacks for the life of the process β€” the bot +/// would go quiet with nothing but the panic in the log to say why. Running on a +/// dedicated task localizes the unwind, and awaiting it immediately keeps +/// callbacks strictly ordered. Mirrors the panic isolation the HTTP callback +/// workers already have. +async fn run_isolated(future: F) +where + F: Future + Send + 'static, +{ + if let Err(error) = tokio::spawn(future).await { + warn!("Slack callback job panicked: {error}"); } } @@ -229,4 +252,34 @@ mod tests { ); assert!(!dedup.contains(DedupKind::Event, "canceled")); } + + #[tokio::test] + async fn a_panicking_job_does_not_stop_later_jobs() { + // There is one worker behind this queue, so an unguarded panic would + // silence every subsequent Slack callback for the life of the process. + let ran = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let panicking = { + let ran = Arc::clone(&ran); + async move { + ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + panic!("callback exploded"); + } + }; + run_isolated(panicking).await; + + let following = { + let ran = Arc::clone(&ran); + async move { + ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + }; + run_isolated(following).await; + + assert_eq!( + ran.load(std::sync::atomic::Ordering::SeqCst), + 2, + "the job after the panicking one never ran" + ); + } } From 790fe330825278338fe005fb99ed39aa53f049fb Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 16:06:39 -0700 Subject: [PATCH 22/23] fix(channels): reject spoofed reply targets, share the fair callback queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the two follow-ups filed against the reaction/callback work. ## moltis-32nj: chat.send trusted _channel_reply_target from the wire The open question was whether a web-originated send ever legitimately names a channel target. It does not: `chat_impl::send` derives the target itself from the session's persisted `channel_binding`, and only when that session is still the active one for the chat. A client-supplied value was pure spoofing surface β€” set `_session_key` and the server's own injection is skipped (`is_web_message` requires its absence), so the client's value reached `deferred_channel_target` unchallenged and addressed an outbound message to any chat the bot can reach. So it joins `_ack_keys` in the set stripped at the RPC boundary. Rather than a second ad-hoc removal, the keys that cross the crate boundary now have one home in `moltis_chat::params`, with `SERVER_ONLY` naming the subset a client may never supply and `strip_server_only` applying it. The underscore prefix alone is not the criterion β€” the web UI legitimately sets `_session_key`, `_audio_filename` and `_document_files` β€” so the list is explicit and documents why each entry is on it. The literals at the set/read/strip sites are gone, so those three can no longer drift. ## moltis-9d0k: two callback paths, two concurrency models HTTP callbacks had a 16-worker account-fair pool; Socket Mode had a single worker for every account, so a slow workspace head-of-line blocked all the others. Both now use one scheduler, extracted to `moltis_channels::fair_queue` β€” the lower-level crate both depend on, per the DRY rule in CLAUDE.md. The extraction is generic over a job type that reports its account, and keeps the properties the HTTP path had: per-account ordering (an account is never active on two workers), round-robin fairness with a per-account share of the queue, non-blocking admission that rejects rather than awaits, and a permit carried by the job so capacity bounds queued *and* in-flight work. Panic isolation moved in too, so it now covers Socket Mode, and completion is reported even after a panic β€” otherwise the account would stay marked active and its remaining callbacks would never be served. Shutdown is one place where the shared queue is deliberately not a straight lift: it watches a *child* of the caller's token, so dropping a queue cannot cancel the token that also governs the connection feeding it, while a cancelled parent still drains the jobs already admitted (those were acknowledged to the platform and will not be redelivered). Scheduler tests moved to `fair_queue` alongside the code, covering ordering, cross-account independence, the per-account cap, capacity return, panic isolation and drain-on-cancel. Each call site keeps the tests for what it still owns: dedupe-commits-only-on-admission on both sides, and the HTTP response shapes. Both suites also gained a direct regression test for the head-of-line blocking this fixes. Note for reviewers: the `blocking_queue` test helpers gate on a `CancellationToken`, not a `Notify`. A token stays signalled once cancelled, so a job reaching its await after the release still proceeds; `notify_waiters` drops that wakeup and hangs the test. --- Cargo.lock | 1 + crates/channels/Cargo.toml | 1 + crates/channels/src/fair_queue.rs | 664 ++++++++++++++++++ crates/channels/src/lib.rs | 1 + crates/chat/src/channel_acks.rs | 5 +- crates/chat/src/lib.rs | 3 +- crates/chat/src/params.rs | 77 ++ crates/chat/src/service/chat_impl/send.rs | 70 +- .../channel_events/commands/attachments.rs | 2 +- crates/gateway/src/channel_events/dispatch.rs | 4 +- crates/gateway/src/methods/services/core.rs | 35 +- crates/httpd/src/server/slack_callbacks.rs | 639 +++-------------- crates/slack/src/callback_worker.rs | 255 ++++--- crates/slack/src/socket/callbacks.rs | 1 + 14 files changed, 1056 insertions(+), 702 deletions(-) create mode 100644 crates/channels/src/fair_queue.rs create mode 100644 crates/chat/src/params.rs diff --git a/Cargo.lock b/Cargo.lock index ac1d83791c..4afd746b99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7310,6 +7310,7 @@ dependencies = [ "serde_json", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", "url", ] diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.toml index d61581fa23..a5908b517b 100644 --- a/crates/channels/Cargo.toml +++ b/crates/channels/Cargo.toml @@ -15,6 +15,7 @@ serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } +tokio-util = { workspace = true } tracing = { workspace = true } url = { workspace = true } diff --git a/crates/channels/src/fair_queue.rs b/crates/channels/src/fair_queue.rs new file mode 100644 index 0000000000..2088128aeb --- /dev/null +++ b/crates/channels/src/fair_queue.rs @@ -0,0 +1,664 @@ +//! Bounded, account-fair work queue for inbound channel callbacks. +//! +//! Channel platforms expect a callback to be acknowledged within a few seconds, +//! far less than an agent turn takes, so callbacks are admitted synchronously +//! and processed afterwards. That makes admission the interesting part, and it +//! has three requirements that a plain `mpsc` channel does not meet: +//! +//! - **Per-account ordering.** An account is never processed on more than one +//! worker at a time, so its callbacks are handled in the order they arrived. +//! - **Fairness across accounts.** Ready accounts are served round-robin and +//! each is capped, so one busy workspace cannot head-of-line block the others +//! or consume the whole queue. +//! - **Bounded memory with an explicit rejection.** A full queue reports +//! [`Admission::Rejected`] so the caller can ask the platform to redeliver, +//! rather than dropping the callback silently. +//! +//! [`FairQueue::admit`] is deliberately non-blocking: it is called from request +//! handlers and socket callbacks that must not await. It never blocks on a lock +//! and never waits for capacity β€” it rejects instead. + +use std::{ + collections::{HashMap, VecDeque}, + future::Future, + sync::{Arc, Mutex, TryLockError}, +}; + +use { + tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc}, + tokio_util::sync::CancellationToken, + tracing::{error, warn}, +}; + +/// A unit of work that belongs to one channel account. +pub trait FairQueueJob: Send + 'static { + /// Account this job belongs to. Jobs sharing an account are processed in + /// admission order, one at a time. + fn account_id(&self) -> &str; +} + +/// Outcome of offering a job to the queue. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Admission { + /// Accepted; a worker will process it. + Admitted, + /// Refused because the queue (or this account's share of it) is full, or + /// because the queue is shutting down. The caller should ask the platform to + /// redeliver rather than treat the callback as handled. + Rejected, +} + +/// Sizing for a [`FairQueue`]. +#[derive(Clone, Copy, Debug)] +pub struct FairQueueConfig { + /// Total jobs that may be queued or in flight across all accounts. + pub capacity: usize, + /// Concurrent workers, and so the maximum number of accounts processed at + /// once. + pub workers: usize, + /// Cap on one account's queued-plus-active jobs. Keeps a single busy account + /// from consuming the whole queue. + pub per_account_capacity: usize, +} + +impl FairQueueConfig { + /// Config for `capacity` total jobs across `workers` workers, giving each + /// account an equal share of the queue. + #[must_use] + pub fn new(capacity: usize, workers: usize) -> Self { + Self { + capacity, + workers: workers.max(1), + per_account_capacity: (capacity / workers.max(1)).max(1), + } + } +} + +/// A queued job together with the capacity permit it holds. +/// +/// The permit lives as long as the job does β€” through queueing *and* +/// processing β€” and is released when the worker drops the slot, so +/// [`FairQueueConfig::capacity`] bounds total outstanding work rather than just +/// queue depth. +struct Slot { + job: J, + _permit: OwnedSemaphorePermit, +} + +impl FairQueueJob for Slot { + fn account_id(&self) -> &str { + self.job.account_id() + } +} + +struct AccountQueue { + jobs: VecDeque, + active: bool, +} + +impl Default for AccountQueue { + fn default() -> Self { + Self { + jobs: VecDeque::new(), + active: false, + } + } +} + +struct Queues { + accounts: HashMap>, + /// Accounts with work and no worker currently on them, in service order. + ready: VecDeque, + per_account_capacity: usize, +} + +impl Queues { + fn new(per_account_capacity: usize) -> Self { + Self { + accounts: HashMap::new(), + ready: VecDeque::new(), + per_account_capacity, + } + } + + fn has_capacity(&self, account_id: &str) -> bool { + self.accounts.get(account_id).is_none_or(|account| { + account.jobs.len() + usize::from(account.active) < self.per_account_capacity + }) + } + + fn try_enqueue(&mut self, job: J) -> bool { + let account_id = job.account_id().to_string(); + if !self.has_capacity(&account_id) { + return false; + } + let account = self.accounts.entry(account_id.clone()).or_default(); + // Only becomes newly servable if nothing is queued and no worker holds + // it; otherwise it is already in `ready` or will be re-added on + // completion. + let became_ready = !account.active && account.jobs.is_empty(); + account.jobs.push_back(job); + if became_ready { + self.ready.push_back(account_id); + } + true + } + + /// Undo the most recent `try_enqueue` for an account, for when a later step + /// of admission fails and the job must not stay queued. + fn rollback_last(&mut self, account_id: &str) { + let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { + let _ = account.jobs.pop_back(); + !account.active && account.jobs.is_empty() + } else { + false + }; + if remove_account { + self.ready.retain(|ready| ready != account_id); + self.accounts.remove(account_id); + } + } + + /// Next job in round-robin account order, marking its account active so no + /// second worker can pick the same account up. + fn take_next(&mut self) -> Option { + while let Some(account_id) = self.ready.pop_front() { + let Some(account) = self.accounts.get_mut(&account_id) else { + continue; + }; + let Some(job) = account.jobs.pop_front() else { + continue; + }; + account.active = true; + return Some(job); + } + None + } + + /// Release an account, re-queueing it at the back if it still has work. + fn complete(&mut self, account_id: &str) { + let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { + account.active = false; + if account.jobs.is_empty() { + true + } else { + self.ready.push_back(account_id.to_string()); + false + } + } else { + false + }; + if remove_account { + self.accounts.remove(account_id); + } + } + + fn is_empty(&self) -> bool { + self.accounts.is_empty() + } +} + +/// Handle to a running fair queue. +/// +/// The workers stop when this handle is dropped or the shutdown token fires, +/// after draining whatever was already admitted. Wrap it in an `Arc` to share. +pub struct FairQueue { + queues: Arc>>>, + wake: mpsc::Sender<()>, + permits: Arc, + capacity: usize, + cancel: CancellationToken, +} + +impl FairQueue { + /// Start the worker pool. + /// + /// `process` runs one job to completion. It may panic without taking the + /// pool down. Cancelling `shutdown` drains the jobs already admitted β€” they + /// were acknowledged to the platform and will not be redelivered β€” and then + /// stops the workers. + /// + /// The queue watches a *child* of `shutdown`, so dropping the queue stops + /// only its own workers and leaves the caller's token β€” which usually also + /// governs the connection that feeds it β€” untouched. + pub fn start(config: FairQueueConfig, shutdown: &CancellationToken, process: F) -> Self + where + F: Fn(J) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static, + { + let cancel = shutdown.child_token(); + let queues = Arc::new(Mutex::new(Queues::new(config.per_account_capacity))); + let (wake, wake_receiver) = mpsc::channel(1); + tokio::spawn(run_supervisor( + Arc::clone(&queues), + wake_receiver, + cancel.clone(), + config.workers, + process, + )); + Self { + queues, + wake, + permits: Arc::new(Semaphore::new(config.capacity)), + capacity: config.capacity, + cancel, + } + } + + /// Offer a job without blocking or awaiting. + /// + /// Rejects rather than waiting when the queue is shutting down, this + /// account's share is full, global capacity is exhausted, or the internal + /// lock is momentarily contended. + pub fn admit(&self, job: J) -> Admission { + if self.cancel.is_cancelled() { + return Admission::Rejected; + } + // `try_lock`, not `lock`: admission runs on request-handling tasks that + // must not block. Contention is brief, and the platform will redeliver. + let mut queues = match self.queues.try_lock() { + Ok(queues) => queues, + Err(TryLockError::Poisoned(error)) => error.into_inner(), + Err(TryLockError::WouldBlock) => return Admission::Rejected, + }; + let account_id = job.account_id().to_string(); + if !queues.has_capacity(&account_id) { + return Admission::Rejected; + } + // Taken before enqueueing and carried by the slot until the job has been + // processed, so capacity covers queued *and* in-flight work. Dropping + // the slot returns it. + let Ok(permit) = Arc::clone(&self.permits).try_acquire_owned() else { + return Admission::Rejected; + }; + if !queues.try_enqueue(Slot { + job, + _permit: permit, + }) { + return Admission::Rejected; + } + + match self.wake.try_send(()) { + // A pending wake is as good as a new one: the supervisor re-checks + // the queue after every job. + Ok(()) | Err(mpsc::error::TrySendError::Full(())) => Admission::Admitted, + Err(mpsc::error::TrySendError::Closed(())) => { + // The supervisor is gone and would never pick this up. Undoing + // the enqueue drops the slot, which returns its permit. + queues.rollback_last(&account_id); + Admission::Rejected + }, + } + } + + /// Jobs currently queued or being processed. Test and diagnostic use. + #[must_use] + pub fn in_flight(&self) -> usize { + self.capacity + .saturating_sub(self.permits.available_permits()) + } +} + +impl Drop for FairQueue { + fn drop(&mut self) { + // Stop this queue's workers when the handle goes away, after they finish + // the jobs already admitted. Cancels only the child token, so the + // caller's shutdown token is unaffected. + self.cancel.cancel(); + } +} + +async fn run_supervisor( + queues: Arc>>>, + mut wake_receiver: mpsc::Receiver<()>, + cancel: CancellationToken, + workers: usize, + process: F, +) where + J: FairQueueJob, + F: Fn(J) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static, +{ + let mut worker_senders = Vec::with_capacity(workers); + let mut idle = (0..workers).collect::>(); + let (completion_sender, mut completions) = mpsc::channel(workers); + let mut pool = tokio::task::JoinSet::new(); + for index in 0..workers { + let (sender, receiver) = mpsc::channel(1); + worker_senders.push(sender); + pool.spawn(run_worker( + index, + receiver, + completion_sender.clone(), + process.clone(), + )); + } + drop(completion_sender); + + let mut accepting = true; + loop { + // Hand work to every idle worker we can fill. + while let Some(index) = idle.pop_front() { + let job = lock(&queues).take_next(); + let Some(job) = job else { + idle.push_front(index); + break; + }; + if worker_senders[index].try_send(job).is_err() { + error!( + worker = index, + "channel callback worker stopped accepting work" + ); + return; + } + } + + // Nothing left to admit and nothing left to drain. + if !accepting && lock(&queues).is_empty() { + break; + } + + tokio::select! { + () = cancel.cancelled(), if accepting => { + // Stop taking new work but finish what was already admitted. + accepting = false; + wake_receiver.close(); + }, + wake = wake_receiver.recv(), if accepting => { + if wake.is_none() { + accepting = false; + } + }, + completion = completions.recv() => { + let Some((index, account_id)) = completion else { + error!("all channel callback workers stopped unexpectedly"); + return; + }; + lock(&queues).complete(&account_id); + idle.push_back(index); + }, + result = pool.join_next(), if !pool.is_empty() => { + if let Some(Err(error)) = result { + error!("channel callback worker failed: {error}"); + } + error!("channel callback worker stopped unexpectedly"); + return; + }, + } + } + + drop(worker_senders); + while let Some(result) = pool.join_next().await { + if let Some(error) = result.err() { + error!("channel callback worker failed: {error}"); + } + } +} + +fn lock(queues: &Arc>>) -> std::sync::MutexGuard<'_, Queues> { + queues.lock().unwrap_or_else(|error| error.into_inner()) +} + +async fn run_worker( + index: usize, + mut receiver: mpsc::Receiver>, + completions: mpsc::Sender<(usize, String)>, + process: F, +) where + J: FairQueueJob, + F: Fn(J) -> Fut, + Fut: Future + Send + 'static, +{ + while let Some(slot) = receiver.recv().await { + let account_id = slot.account_id().to_string(); + { + // Scoped so the permit is released as soon as the job is done, + // before completion is reported. + let Slot { job, _permit } = slot; + // Run on a dedicated task so a panic inside one callback cannot take + // the worker β€” and with it every later callback β€” down. Awaited + // immediately, so the account's ordering is preserved. + if let Err(error) = tokio::spawn(process(job)).await { + warn!(worker = index, %account_id, "channel callback panicked: {error}"); + } + } + // Report completion even after a panic, or the account would stay marked + // active and its remaining callbacks would never be served. + if completions.send((index, account_id)).await.is_err() { + return; + } + } +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + struct TestJob { + account_id: String, + label: String, + } + + impl FairQueueJob for TestJob { + fn account_id(&self) -> &str { + &self.account_id + } + } + + fn job(account_id: &str, label: &str) -> TestJob { + TestJob { + account_id: account_id.to_string(), + label: label.to_string(), + } + } + + #[test] + fn per_account_capacity_divides_the_queue() { + let config = FairQueueConfig::new(256, 16); + assert_eq!(config.per_account_capacity, 16); + // Never zero, however small the queue relative to the pool. + assert_eq!(FairQueueConfig::new(4, 16).per_account_capacity, 1); + assert_eq!(FairQueueConfig::new(8, 0).workers, 1); + } + + #[tokio::test] + async fn one_accounts_jobs_run_in_admission_order() { + let order = Arc::new(tokio::sync::Mutex::new(Vec::new())); + // Several workers, but a per-account share wide enough that the cap is + // not what this test is exercising. + let config = FairQueueConfig { + capacity: 16, + workers: 4, + per_account_capacity: 16, + }; + let queue = { + let order = Arc::clone(&order); + FairQueue::start(config, &CancellationToken::new(), move |job: TestJob| { + let order = Arc::clone(&order); + async move { + // Yield so a scheduler that ran an account concurrently + // would interleave here. + tokio::task::yield_now().await; + order.lock().await.push(job.label); + } + }) + }; + + for index in 0..6 { + assert_eq!( + queue.admit(job("acct", &format!("job-{index}"))), + Admission::Admitted + ); + } + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } + + assert_eq!( + *order.lock().await, + (0..6).map(|i| format!("job-{i}")).collect::>() + ); + } + + #[tokio::test] + async fn a_blocked_account_does_not_stall_other_accounts() { + let release = Arc::new(tokio::sync::Notify::new()); + let started = Arc::new(tokio::sync::Notify::new()); + let other_ran = Arc::new(tokio::sync::Notify::new()); + let queue = { + let (release, started, other_ran) = ( + Arc::clone(&release), + Arc::clone(&started), + Arc::clone(&other_ran), + ); + FairQueue::start( + FairQueueConfig::new(16, 4), + &CancellationToken::new(), + move |job: TestJob| { + let (release, started, other_ran) = ( + Arc::clone(&release), + Arc::clone(&started), + Arc::clone(&other_ran), + ); + async move { + if job.account_id == "busy" { + started.notify_one(); + release.notified().await; + } else { + other_ran.notify_one(); + } + } + }, + ) + }; + + assert_eq!(queue.admit(job("busy", "blocking")), Admission::Admitted); + started.notified().await; + assert_eq!( + queue.admit(job("quiet", "independent")), + Admission::Admitted + ); + + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), other_ran.notified()) + .await + .is_ok(), + "a busy account head-of-line blocked an unrelated one" + ); + release.notify_one(); + } + + #[tokio::test] + async fn one_account_cannot_consume_the_whole_queue() { + let release = Arc::new(tokio::sync::Notify::new()); + let queue = { + let release = Arc::clone(&release); + FairQueue::start( + FairQueueConfig::new(8, 4), + &CancellationToken::new(), + move |_job: TestJob| { + let release = Arc::clone(&release); + async move { release.notified().await } + }, + ) + }; + + // per_account_capacity == 2, so the third is refused while global + // capacity (8) is still available to other accounts. + assert_eq!(queue.admit(job("hot", "a")), Admission::Admitted); + assert_eq!(queue.admit(job("hot", "b")), Admission::Admitted); + assert_eq!(queue.admit(job("hot", "c")), Admission::Rejected); + assert_eq!(queue.admit(job("cold", "a")), Admission::Admitted); + + release.notify_waiters(); + } + + #[tokio::test] + async fn a_panicking_job_does_not_stop_the_account_or_the_pool() { + let ran = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let queue = { + let ran = Arc::clone(&ran); + FairQueue::start( + FairQueueConfig::new(16, 2), + &CancellationToken::new(), + move |job: TestJob| { + let ran = Arc::clone(&ran); + async move { + ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + assert_ne!(job.label, "boom", "intentional test panic"); + } + }, + ) + }; + + assert_eq!(queue.admit(job("acct", "boom")), Admission::Admitted); + assert_eq!(queue.admit(job("acct", "after")), Admission::Admitted); + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } + + assert_eq!( + ran.load(std::sync::atomic::Ordering::SeqCst), + 2, + "the job after the panicking one never ran" + ); + } + + #[tokio::test] + async fn a_completed_job_returns_its_capacity() { + let queue = FairQueue::start( + FairQueueConfig::new(2, 1), + &CancellationToken::new(), + move |_job: TestJob| async move {}, + ); + + for index in 0..6 { + // Capacity is 2, so this only keeps succeeding if permits come back. + let label = format!("job-{index}"); + while queue.admit(job("acct", &label)) == Admission::Rejected { + tokio::task::yield_now().await; + } + } + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } + assert_eq!(queue.in_flight(), 0); + } + + #[tokio::test] + async fn cancellation_drains_already_admitted_jobs() { + // Those callbacks were acknowledged to the platform and will not be + // redelivered, so shutdown must finish them rather than drop them. + let ran = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let cancel = CancellationToken::new(); + let queue = { + let ran = Arc::clone(&ran); + FairQueue::start( + FairQueueConfig::new(16, 1), + &cancel, + move |_job: TestJob| { + let ran = Arc::clone(&ran); + async move { + tokio::task::yield_now().await; + ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + }, + ) + }; + + for index in 0..4 { + assert_eq!( + queue.admit(job("acct", &format!("job-{index}"))), + Admission::Admitted + ); + } + cancel.cancel(); + + assert_eq!(queue.admit(job("acct", "too-late")), Admission::Rejected); + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } + assert_eq!(ran.load(std::sync::atomic::Ordering::SeqCst), 4); + } +} diff --git a/crates/channels/src/lib.rs b/crates/channels/src/lib.rs index 58c0a4d712..ec178e7c3f 100644 --- a/crates/channels/src/lib.rs +++ b/crates/channels/src/lib.rs @@ -10,6 +10,7 @@ pub mod commands; pub mod config_view; pub mod contract; pub mod error; +pub mod fair_queue; pub mod gating; pub mod media_download; pub mod message_log; diff --git a/crates/chat/src/channel_acks.rs b/crates/chat/src/channel_acks.rs index 27d909a508..31c2020e1b 100644 --- a/crates/chat/src/channel_acks.rs +++ b/crates/chat/src/channel_acks.rs @@ -11,10 +11,7 @@ use serde_json::Value; use crate::runtime::ChatRuntime; /// Params key carrying the acknowledgment identities a call is responsible for. -/// -/// Exported so the channel dispatch layer sets the same key this module reads, -/// instead of both sides hardcoding the string. -pub const ACK_KEYS_PARAM: &str = "_ack_keys"; +pub(crate) use crate::params::ACK_KEYS as ACK_KEYS_PARAM; /// Read the acknowledgment identities from `chat.send` params. pub(crate) fn ack_keys_from_params(params: &Value) -> Vec { diff --git a/crates/chat/src/lib.rs b/crates/chat/src/lib.rs index ec50561167..ef6ec92a86 100644 --- a/crates/chat/src/lib.rs +++ b/crates/chat/src/lib.rs @@ -1,5 +1,5 @@ mod agent_loop; -pub mod channel_acks; +pub(crate) mod channel_acks; #[cfg(any(feature = "push-notifications", test))] mod channel_push; mod channels; @@ -8,6 +8,7 @@ mod compaction_run; mod memory_tools; mod message; mod models; +pub mod params; mod prompt; mod run_with_tools; mod service; diff --git a/crates/chat/src/params.rs b/crates/chat/src/params.rs new file mode 100644 index 0000000000..a5d464010b --- /dev/null +++ b/crates/chat/src/params.rs @@ -0,0 +1,77 @@ +//! Keys for the `_`-prefixed internal params carried alongside a `chat.send`. +//! +//! The underscore prefix marks a param as plumbing rather than user input, but +//! it does **not** mean "server-only": the web UI legitimately sets +//! `_session_key`, `_audio_filename` and `_document_files` on its own calls. +//! +//! [`SERVER_ONLY`] is the subset that a client must never be able to supply, +//! because each one names a resource the caller would otherwise be asserting +//! authority over. The RPC boundary strips these from client params; the server +//! re-derives them from state it trusts. Defined here β€” rather than as string +//! literals at each call site β€” so the code that sets a key, the code that +//! reads it, and the code that strips it cannot drift apart. + +/// Acknowledgment identities a call is responsible for. +/// +/// Minted by the channel dispatch path, which mints one per inbound message and +/// carries it through queueing and replay so the reaction follows the message +/// itself. A client-supplied value would let one user drive the βœ…/❌ reaction on +/// an unrelated inbound channel message. +pub const ACK_KEYS: &str = "_ack_keys"; + +/// Channel destination a reply should be echoed to. +/// +/// Derived server-side in `chat_impl::send` from the session's persisted +/// `channel_binding`, and only when that session is still the active one for +/// the chat. A client-supplied value would let one user address an outbound +/// message to any chat the bot can reach. +pub const CHANNEL_REPLY_TARGET: &str = "_channel_reply_target"; + +/// Params that only the server may set, stripped from client-supplied params at +/// the RPC boundary. +/// +/// Add a key here when it confers authority the caller does not inherently +/// have β€” naming another message, another chat, or another session's resources. +pub const SERVER_ONLY: &[&str] = &[ACK_KEYS, CHANNEL_REPLY_TARGET]; + +/// Remove every [`SERVER_ONLY`] key from client-supplied params. +/// +/// A non-object value has no params to strip and is left alone; the chat +/// service rejects it later on its own terms. +pub fn strip_server_only(params: &mut serde_json::Value) { + let Some(object) = params.as_object_mut() else { + return; + }; + for key in SERVER_ONLY { + object.remove(*key); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strips_every_server_only_key_and_keeps_the_rest() { + let mut params = serde_json::json!({ + "text": "hi", + "_session_key": "session:42", + ACK_KEYS: ["acct:C1:1.0"], + CHANNEL_REPLY_TARGET: { "chat_id": "C999" }, + }); + strip_server_only(&mut params); + + assert_eq!(params["text"], "hi"); + assert_eq!(params["_session_key"], "session:42"); + for key in SERVER_ONLY { + assert!(params.get(*key).is_none(), "{key} survived"); + } + } + + #[test] + fn non_object_params_are_left_alone() { + let mut params = serde_json::json!("just a string"); + strip_server_only(&mut params); + assert_eq!(params, serde_json::json!("just a string")); + } +} diff --git a/crates/chat/src/service/chat_impl/send.rs b/crates/chat/src/service/chat_impl/send.rs index 23eaf109d7..aafd0cceee 100644 --- a/crates/chat/src/service/chat_impl/send.rs +++ b/crates/chat/src/service/chat_impl/send.rs @@ -333,7 +333,7 @@ impl LiveChatService { if is_active { match serde_json::to_value(&target) { Ok(target_val) => { - params["_channel_reply_target"] = target_val; + params[crate::params::CHANNEL_REPLY_TARGET] = target_val; }, Err(e) => { warn!( @@ -346,23 +346,22 @@ impl LiveChatService { } } - let deferred_channel_target = - params - .get("_channel_reply_target") - .cloned() - .and_then(|value| { - match serde_json::from_value::(value) { - Ok(target) => Some(target), - Err(e) => { - warn!( - session = %session_key, - error = %e, - "ignoring invalid _channel_reply_target for /sh" - ); - None - }, - } - }); + let deferred_channel_target = params + .get(crate::params::CHANNEL_REPLY_TARGET) + .cloned() + .and_then(|value| { + match serde_json::from_value::(value) { + Ok(target) => Some(target), + Err(e) => { + warn!( + session = %session_key, + error = %e, + "ignoring invalid _channel_reply_target for /sh" + ); + None + }, + } + }); info!( run_id = %run_id, @@ -678,7 +677,7 @@ impl LiveChatService { if is_active { match serde_json::to_value(&target) { Ok(target_val) => { - params["_channel_reply_target"] = target_val; + params[crate::params::CHANNEL_REPLY_TARGET] = target_val; }, Err(e) => { warn!( @@ -691,23 +690,22 @@ impl LiveChatService { } } - let deferred_channel_target = - params - .get("_channel_reply_target") - .cloned() - .and_then(|value| { - match serde_json::from_value::(value) { - Ok(target) => Some(target), - Err(e) => { - warn!( - session = %session_key, - error = %e, - "ignoring invalid _channel_reply_target" - ); - None - }, - } - }); + let deferred_channel_target = params + .get(crate::params::CHANNEL_REPLY_TARGET) + .cloned() + .and_then(|value| { + match serde_json::from_value::(value) { + Ok(target) => Some(target), + Err(e) => { + warn!( + session = %session_key, + error = %e, + "ignoring invalid _channel_reply_target" + ); + None + }, + } + }); // Dispatch the `MessageReceived` hook before the turn starts. The // hook can: diff --git a/crates/gateway/src/channel_events/commands/attachments.rs b/crates/gateway/src/channel_events/commands/attachments.rs index 3a67b511dc..f720dd5943 100644 --- a/crates/gateway/src/channel_events/commands/attachments.rs +++ b/crates/gateway/src/channel_events/commands/attachments.rs @@ -139,7 +139,7 @@ pub(in crate::channel_events) async fn dispatch_to_chat_with_attachments( "_session_key": &session_key, // Defer reply-target registration until chat.send() actually // starts executing this message (after semaphore acquire). - "_channel_reply_target": &reply_to, + moltis_chat::params::CHANNEL_REPLY_TARGET: &reply_to, }); if let Some(ref documents) = meta.documents { params["_document_files"] = serde_json::json!(documents); diff --git a/crates/gateway/src/channel_events/dispatch.rs b/crates/gateway/src/channel_events/dispatch.rs index fae784c5c2..311fa7cb22 100644 --- a/crates/gateway/src/channel_events/dispatch.rs +++ b/crates/gateway/src/channel_events/dispatch.rs @@ -139,14 +139,14 @@ pub(in crate::channel_events) async fn dispatch_to_chat( "_session_key": &session_key, // Defer reply-target registration until chat.send() actually // starts executing this message (after semaphore acquire). - "_channel_reply_target": &reply_to, + moltis_chat::params::CHANNEL_REPLY_TARGET: &reply_to, }); // Carry this message's acknowledgment identity into the run so the // reaction follows the message itself β€” through queueing and replay β€” // rather than whatever else shares the session. if let Some(ref key) = ack_key { - params[moltis_chat::channel_acks::ACK_KEYS_PARAM] = serde_json::json!([key]); + params[moltis_chat::params::ACK_KEYS] = serde_json::json!([key]); } // Attach thread context if available. diff --git a/crates/gateway/src/methods/services/core.rs b/crates/gateway/src/methods/services/core.rs index 97fa27f8fe..f696718450 100644 --- a/crates/gateway/src/methods/services/core.rs +++ b/crates/gateway/src/methods/services/core.rs @@ -7,16 +7,15 @@ use super::*; /// The `_`-prefixed params are internal plumbing, and some of them are set /// legitimately by the web UI (`_session_key`, `_audio_filename`, /// `_document_files`), so this is a targeted removal rather than a blanket -/// strip: `_ack_keys` names a *channel message's* acknowledgment-reaction slot -/// and is only ever minted by the channel dispatch path (which calls the chat -/// service directly, not through this registry). Letting a WebSocket client -/// send it would let one authenticated user drive the βœ…/❌ reaction on an -/// unrelated inbound Slack message. +/// strip. See [`moltis_chat::params::SERVER_ONLY`] for which keys are dropped +/// and why; the server re-derives each of them from state it trusts. +/// +/// Only the channel dispatch path sets those keys, and it calls the chat service +/// directly rather than through this registry, so stripping here cannot affect +/// it. async fn prepare_chat_send_params(ctx: &MethodContext) -> serde_json::Value { let mut params = ctx.params.clone(); - if let Some(object) = params.as_object_mut() { - object.remove(moltis_chat::channel_acks::ACK_KEYS_PARAM); - } + moltis_chat::params::strip_server_only(&mut params); params["_conn_id"] = serde_json::json!(ctx.client_conn_id); // Forward client Accept-Language, public remote IP, and timezone. @@ -1343,20 +1342,24 @@ mod tests { } #[tokio::test] - async fn client_supplied_ack_keys_are_stripped() { + async fn client_supplied_server_only_params_are_stripped() { // A WebSocket client must not be able to claim the acknowledgment slot - // of an inbound channel message it has nothing to do with. + // of an inbound channel message, nor name the chat a reply is sent to. + // Both are re-derived server-side from state the server trusts. let params = prepare_chat_send_params(&context(serde_json::json!({ "text": "hi", - moltis_chat::channel_acks::ACK_KEYS_PARAM: ["slack-acct:C123:1700000000.1"], + moltis_chat::params::ACK_KEYS: ["slack-acct:C123:1700000000.1"], + moltis_chat::params::CHANNEL_REPLY_TARGET: { + "channel_type": "slack", + "account_id": "victim-workspace", + "chat_id": "C999", + }, }))) .await; - assert!( - params - .get(moltis_chat::channel_acks::ACK_KEYS_PARAM) - .is_none() - ); + for key in moltis_chat::params::SERVER_ONLY { + assert!(params.get(*key).is_none(), "{key} was not stripped"); + } assert_eq!(params["text"], "hi"); } diff --git a/crates/httpd/src/server/slack_callbacks.rs b/crates/httpd/src/server/slack_callbacks.rs index 3985983e86..857911e739 100644 --- a/crates/httpd/src/server/slack_callbacks.rs +++ b/crates/httpd/src/server/slack_callbacks.rs @@ -1,9 +1,4 @@ -use std::{ - collections::{HashMap, VecDeque}, - future::Future, - panic::AssertUnwindSafe, - sync::{Arc, Mutex, RwLock, TryLockError}, -}; +use std::sync::{Arc, RwLock}; use { axum::{ @@ -11,20 +6,18 @@ use { response::{IntoResponse, Json, Response}, }, bytes::Bytes, - futures::FutureExt, + moltis_channels::fair_queue::{Admission, FairQueue, FairQueueConfig, FairQueueJob}, moltis_gateway::{ channel_webhook_dedup::{ChannelWebhookAdmission, ChannelWebhookDedupeStore}, state::GatewayState, }, - tokio::{ - sync::{OwnedSemaphorePermit, Semaphore, mpsc}, - task::JoinSet, - }, + tokio_util::sync::CancellationToken, }; +/// Total Slack callbacks that may be queued or in flight, across every account. const CALLBACK_QUEUE_CAPACITY: usize = 256; +/// Callbacks processed concurrently, and so the number of accounts served at once. const CALLBACK_WORKER_LIMIT: usize = 16; -const CALLBACK_ACCOUNT_CAPACITY: usize = CALLBACK_QUEUE_CAPACITY / CALLBACK_WORKER_LIMIT; /// Slack callback endpoints served under `/api/channels/slack/{account_id}/`. /// @@ -169,116 +162,35 @@ fn event_preflight(body: &[u8]) -> Option { ) } +/// One verified Slack callback waiting to be ingested. struct SlackCallbackJob { account_id: String, body: Bytes, kind: SlackCallbackKind, - _permit: OwnedSemaphorePermit, -} - -#[derive(Default)] -struct AccountQueue { - jobs: VecDeque, - active: bool, -} - -#[derive(Default)] -struct CallbackQueues { - accounts: HashMap, - ready_accounts: VecDeque, } -impl CallbackQueues { - fn account_has_capacity(&self, account_id: &str) -> bool { - self.accounts.get(account_id).is_none_or(|account| { - account.jobs.len() + usize::from(account.active) < CALLBACK_ACCOUNT_CAPACITY - }) - } - - fn try_enqueue(&mut self, job: SlackCallbackJob) -> bool { - let account_id = job.account_id.clone(); - if !self.account_has_capacity(&account_id) { - return false; - } - - let account = self.accounts.entry(account_id.clone()).or_default(); - let became_ready = !account.active && account.jobs.is_empty(); - account.jobs.push_back(job); - if became_ready { - self.ready_accounts.push_back(account_id); - } - true - } - - fn rollback_last(&mut self, account_id: &str) { - let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { - let _ = account.jobs.pop_back(); - !account.active && account.jobs.is_empty() - } else { - false - }; - if remove_account { - self.ready_accounts.retain(|ready| ready != account_id); - self.accounts.remove(account_id); - } - } - - fn take_next(&mut self) -> Option { - while let Some(account_id) = self.ready_accounts.pop_front() { - let Some(account) = self.accounts.get_mut(&account_id) else { - continue; - }; - let Some(job) = account.jobs.pop_front() else { - continue; - }; - account.active = true; - return Some(job); - } - None - } - - fn complete(&mut self, account_id: &str) { - let remove_account = if let Some(account) = self.accounts.get_mut(account_id) { - account.active = false; - if account.jobs.is_empty() { - true - } else { - self.ready_accounts.push_back(account_id.to_string()); - false - } - } else { - false - }; - if remove_account { - self.accounts.remove(account_id); - } - } - - fn is_empty(&self) -> bool { - self.accounts.is_empty() +impl FairQueueJob for SlackCallbackJob { + fn account_id(&self) -> &str { + &self.account_id } } -/// Bounded callback queue with an owned supervisor for all processing tasks. +/// Bounded, account-fair callback queue. +/// +/// Owns the worker pool; dropping it stops the workers after they finish the +/// callbacks already admitted (those were acknowledged to Slack and will not be +/// redelivered). pub(super) struct SlackCallbackDispatcher { - queues: Arc>, - wake_sender: mpsc::Sender<()>, - permits: Arc, - supervisor: tokio::task::JoinHandle<()>, + queue: FairQueue, } impl SlackCallbackDispatcher { pub(super) fn start(plugin: Arc>) -> Arc { - let queues = Arc::new(Mutex::new(CallbackQueues::default())); - let (wake_sender, wake_receiver) = mpsc::channel(1); - let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); - let supervisor = tokio::spawn(run_supervisor(Arc::clone(&queues), wake_receiver, plugin)); - Arc::new(Self { - queues, - wake_sender, - permits, - supervisor, - }) + let config = FairQueueConfig::new(CALLBACK_QUEUE_CAPACITY, CALLBACK_WORKER_LIMIT); + let queue = FairQueue::start(config, &CancellationToken::new(), move |job| { + process_callback(Arc::clone(&plugin), job) + }); + Arc::new(Self { queue }) } pub(super) fn admit( @@ -290,9 +202,7 @@ impl SlackCallbackDispatcher { body: Bytes, ) -> ChannelWebhookAdmission { admit_callback( - &self.queues, - &self.wake_sender, - &self.permits, + &self.queue, dedup_store, account_id, idempotency_key, @@ -302,16 +212,13 @@ impl SlackCallbackDispatcher { } } -impl Drop for SlackCallbackDispatcher { - fn drop(&mut self) { - self.supervisor.abort(); - } -} - +/// Deduplicate, then queue. +/// +/// The dedupe entry is committed only if the queue actually accepted the job, so +/// a callback rejected for capacity is not remembered as "seen" β€” Slack's retry +/// of it must be admitted rather than silently swallowed as a duplicate. fn admit_callback( - queues: &Arc>, - wake_sender: &mpsc::Sender<()>, - permits: &Arc, + queue: &FairQueue, dedup_store: &RwLock, account_id: String, idempotency_key: Option<&str>, @@ -319,8 +226,16 @@ fn admit_callback( body: Bytes, ) -> ChannelWebhookAdmission { let endpoint = kind.endpoint(); + let enqueue = |account_id: String| { + queue.admit(SlackCallbackJob { + account_id, + body, + kind, + }) == Admission::Admitted + }; + let Some(key) = idempotency_key else { - return if try_enqueue_callback(queues, wake_sender, permits, account_id, kind, body) { + return if enqueue(account_id) { ChannelWebhookAdmission::Admitted } else { ChannelWebhookAdmission::Rejected @@ -332,181 +247,10 @@ fn admit_callback( .write() .unwrap_or_else(|error| error.into_inner()) .admit_scoped("slack", &account_id, endpoint, key, || { - try_enqueue_callback(queues, wake_sender, permits, job_account_id, kind, body) + enqueue(job_account_id) }) } -fn try_enqueue_callback( - queues: &Arc>, - wake_sender: &mpsc::Sender<()>, - permits: &Arc, - account_id: String, - kind: SlackCallbackKind, - body: Bytes, -) -> bool { - let mut queues = match queues.try_lock() { - Ok(queues) => queues, - Err(TryLockError::Poisoned(error)) => error.into_inner(), - Err(TryLockError::WouldBlock) => return false, - }; - if !queues.account_has_capacity(&account_id) { - return false; - } - let Ok(permit) = Arc::clone(permits).try_acquire_owned() else { - return false; - }; - let job = SlackCallbackJob { - account_id: account_id.clone(), - body, - kind, - _permit: permit, - }; - if !queues.try_enqueue(job) { - return false; - } - - match wake_sender.try_send(()) { - Ok(()) | Err(mpsc::error::TrySendError::Full(())) => true, - Err(mpsc::error::TrySendError::Closed(())) => { - queues.rollback_last(&account_id); - false - }, - } -} - -async fn run_supervisor( - queues: Arc>, - wake_receiver: mpsc::Receiver<()>, - plugin: Arc>, -) { - run_fair_workers(queues, wake_receiver, move |job| { - process_callback(Arc::clone(&plugin), job) - }) - .await; -} - -/// Dispatch one callback per ready account in round-robin order. An account is -/// never active on more than one worker, preserving its admission order. -async fn run_fair_workers( - queues: Arc>, - mut wake_receiver: mpsc::Receiver<()>, - process: F, -) where - F: Fn(SlackCallbackJob) -> Fut + Clone + Send + Sync + 'static, - Fut: Future + Send + 'static, -{ - let mut workers = JoinSet::new(); - let mut worker_senders = Vec::with_capacity(CALLBACK_WORKER_LIMIT); - let mut idle_workers = (0..CALLBACK_WORKER_LIMIT).collect::>(); - let (completion_sender, mut completion_receiver) = mpsc::channel(CALLBACK_WORKER_LIMIT); - for worker_index in 0..CALLBACK_WORKER_LIMIT { - let (sender, worker_receiver) = mpsc::channel(1); - worker_senders.push(sender); - workers.spawn(run_serial_worker( - worker_index, - worker_receiver, - completion_sender.clone(), - process.clone(), - )); - } - drop(completion_sender); - - let mut wake_open = true; - loop { - while let Some(worker_index) = idle_workers.pop_front() { - let job = { - queues - .lock() - .unwrap_or_else(|error| error.into_inner()) - .take_next() - }; - let Some(job) = job else { - idle_workers.push_front(worker_index); - break; - }; - if let Err(error) = worker_senders[worker_index].try_send(job) { - tracing::error!(worker_index, %error, "Slack callback worker stopped accepting work"); - return; - } - } - - let queues_empty = queues - .lock() - .unwrap_or_else(|error| error.into_inner()) - .is_empty(); - if !wake_open && queues_empty { - break; - } - - tokio::select! { - wake = wake_receiver.recv(), if wake_open => { - if wake.is_none() { - wake_open = false; - } - }, - completion = completion_receiver.recv() => { - let Some((worker_index, account_id)) = completion else { - tracing::error!("all Slack callback workers stopped unexpectedly"); - return; - }; - queues - .lock() - .unwrap_or_else(|error| error.into_inner()) - .complete(&account_id); - idle_workers.push_back(worker_index); - }, - result = workers.join_next(), if !workers.is_empty() => { - log_worker_result(result); - tracing::error!("Slack callback worker stopped unexpectedly"); - return; - }, - } - } - - drop(worker_senders); - while !workers.is_empty() { - log_worker_result(workers.join_next().await); - } -} - -async fn run_serial_worker( - worker_index: usize, - mut receiver: mpsc::Receiver, - completion_sender: mpsc::Sender<(usize, String)>, - process: F, -) where - F: Fn(SlackCallbackJob) -> Fut, - Fut: Future, -{ - while let Some(job) = receiver.recv().await { - let account_id = job.account_id.clone(); - let process_result = std::panic::catch_unwind(AssertUnwindSafe(|| process(job))); - match process_result { - Ok(future) => { - if AssertUnwindSafe(future).catch_unwind().await.is_err() { - tracing::error!(worker_index, %account_id, "Slack callback worker panicked"); - } - }, - Err(_) => { - tracing::error!(worker_index, %account_id, "Slack callback worker panicked"); - }, - } - if completion_sender - .send((worker_index, account_id)) - .await - .is_err() - { - return; - } - } -} - -fn log_worker_result(result: Option>) { - if let Some(Err(error)) = result { - tracing::error!("Slack callback worker failed: {error}"); - } -} - async fn process_callback( plugin: Arc>, job: SlackCallbackJob, @@ -541,39 +285,6 @@ async fn process_callback( mod tests { use super::*; - fn account_job( - permits: &Arc, - account_id: &str, - body: &'static [u8], - ) -> SlackCallbackJob { - SlackCallbackJob { - account_id: account_id.to_string(), - body: Bytes::from_static(body), - kind: SlackCallbackKind::Event, - _permit: Arc::clone(permits).try_acquire_owned().unwrap(), - } - } - - fn admit_without_key( - queues: &Arc>, - wake_sender: &mpsc::Sender<()>, - permits: &Arc, - dedup: &RwLock, - account_id: &str, - ) -> ChannelWebhookAdmission { - admit_callback( - queues, - wake_sender, - permits, - dedup, - account_id.to_string(), - None, - SlackCallbackKind::Event, - Bytes::from_static(b"callback"), - ) - } - - /// Status of a preflight that must answer synchronously rather than queue. fn preflight_status(body: &[u8]) -> Option { event_preflight(body).map(|response| response.status()) } @@ -652,239 +363,91 @@ mod tests { } } - fn prior_worker_index(account_id: &str) -> usize { - use std::hash::{DefaultHasher, Hash, Hasher}; + /// Build a queue whose jobs block on `gate` until it is cancelled, so + /// admission can be driven to saturation deterministically. + /// + /// A `CancellationToken` rather than a `Notify`: it stays signalled once + /// cancelled, so a job that reaches its await *after* the release still + /// proceeds. `notify_waiters` would drop that wakeup and hang the test. + fn blocking_queue(capacity: usize, gate: CancellationToken) -> FairQueue { + FairQueue::start( + FairQueueConfig { + capacity, + workers: 1, + per_account_capacity: capacity, + }, + &CancellationToken::new(), + move |_job| { + let gate = gate.clone(); + async move { gate.cancelled().await } + }, + ) + } - let mut hasher = DefaultHasher::new(); - account_id.hash(&mut hasher); - hasher.finish() as usize % CALLBACK_WORKER_LIMIT + fn admit( + queue: &FairQueue, + dedup: &RwLock, + key: Option<&str>, + body: &'static [u8], + ) -> ChannelWebhookAdmission { + admit_callback( + queue, + dedup, + "account".to_string(), + key, + SlackCallbackKind::Event, + Bytes::from_static(body), + ) } - #[test] - fn failed_queue_admission_does_not_consume_dedup() { - let queues = Arc::new(Mutex::new(CallbackQueues::default())); - let (closed_sender, closed_receiver) = mpsc::channel(1); - drop(closed_receiver); - let permits = Arc::new(Semaphore::new(2)); + #[tokio::test] + async fn a_rejected_callback_is_not_remembered_as_seen() { + // Capacity rejection must not commit the dedupe entry: Slack retries a + // callback it got no 200 for, and that retry has to be admitted rather + // than swallowed as a duplicate. + let gate = CancellationToken::new(); + let queue = blocking_queue(1, gate.clone()); let dedup = RwLock::new(ChannelWebhookDedupeStore::new()); + // Fills the queue's single slot. assert_eq!( - admit_callback( - &queues, - &closed_sender, - &permits, - &dedup, - "account".to_string(), - Some("event-1"), - SlackCallbackKind::Event, - Bytes::from_static(b"first"), - ), + admit(&queue, &dedup, Some("event-1"), b"first"), + ChannelWebhookAdmission::Admitted + ); + // Saturated, so a different event is refused... + assert_eq!( + admit(&queue, &dedup, Some("event-2"), b"second"), ChannelWebhookAdmission::Rejected ); - - let (wake_sender, _wake_receiver) = mpsc::channel(1); + // ...and refusing it did not record it, so its retry gets in once + // capacity frees up. + gate.cancel(); + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } assert_eq!( - admit_callback( - &queues, - &wake_sender, - &permits, - &dedup, - "account".to_string(), - Some("event-1"), - SlackCallbackKind::Event, - Bytes::from_static(b"retry"), - ), + admit(&queue, &dedup, Some("event-2"), b"second-retry"), ChannelWebhookAdmission::Admitted ); - let _remaining_permit = Arc::clone(&permits).try_acquire_owned().unwrap(); + // A genuine redelivery of something already admitted is still deduped. assert_eq!( - admit_callback( - &queues, - &wake_sender, - &permits, - &dedup, - "account".to_string(), - Some("event-1"), - SlackCallbackKind::Event, - Bytes::from_static(b"duplicate"), - ), + admit(&queue, &dedup, Some("event-1"), b"duplicate"), ChannelWebhookAdmission::Duplicate ); } - #[test] - fn one_account_saturation_leaves_capacity_until_global_saturation() { - let queues = Arc::new(Mutex::new(CallbackQueues::default())); - let (wake_sender, _wake_receiver) = mpsc::channel(1); - let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); + #[tokio::test] + async fn a_callback_without_an_idempotency_key_is_always_admitted() { + let gate = CancellationToken::new(); + gate.cancel(); + let queue = blocking_queue(4, gate); let dedup = RwLock::new(ChannelWebhookDedupeStore::new()); - for _ in 0..CALLBACK_ACCOUNT_CAPACITY { + for _ in 0..2 { assert_eq!( - admit_without_key(&queues, &wake_sender, &permits, &dedup, "hot-account"), + admit(&queue, &dedup, None, b"no-key"), ChannelWebhookAdmission::Admitted ); } - assert_eq!( - admit_without_key(&queues, &wake_sender, &permits, &dedup, "hot-account"), - ChannelWebhookAdmission::Rejected - ); - assert_eq!( - permits.available_permits(), - CALLBACK_QUEUE_CAPACITY - CALLBACK_ACCOUNT_CAPACITY - ); - - assert_eq!( - admit_without_key(&queues, &wake_sender, &permits, &dedup, "cold-account"), - ChannelWebhookAdmission::Admitted - ); - for index in 0..(CALLBACK_QUEUE_CAPACITY - CALLBACK_ACCOUNT_CAPACITY - 1) { - assert_eq!( - admit_without_key( - &queues, - &wake_sender, - &permits, - &dedup, - &format!("other-account-{index}"), - ), - ChannelWebhookAdmission::Admitted - ); - } - assert_eq!(permits.available_permits(), 0); - assert_eq!( - admit_without_key(&queues, &wake_sender, &permits, &dedup, "overflow-account"), - ChannelWebhookAdmission::Rejected - ); - } - - #[tokio::test] - async fn callbacks_for_same_account_are_processed_in_admission_order() { - let queues = Arc::new(Mutex::new(CallbackQueues::default())); - let (wake_sender, wake_receiver) = mpsc::channel(1); - let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); - let first_started = Arc::new(tokio::sync::Notify::new()); - let release_first = Arc::new(tokio::sync::Notify::new()); - let second_started = Arc::new(tokio::sync::Notify::new()); - let processed = Arc::new(tokio::sync::Mutex::new(Vec::new())); - - { - let mut queues = queues.lock().unwrap(); - assert!(queues.try_enqueue(account_job(&permits, "account", b"first"))); - assert!(queues.try_enqueue(account_job(&permits, "account", b"second"))); - } - wake_sender.try_send(()).unwrap(); - - let supervisor = tokio::spawn(run_fair_workers(Arc::clone(&queues), wake_receiver, { - let first_started = Arc::clone(&first_started); - let release_first = Arc::clone(&release_first); - let second_started = Arc::clone(&second_started); - let processed = Arc::clone(&processed); - move |job: SlackCallbackJob| { - let first_started = Arc::clone(&first_started); - let release_first = Arc::clone(&release_first); - let second_started = Arc::clone(&second_started); - let processed = Arc::clone(&processed); - async move { - if job.body == Bytes::from_static(b"first") { - first_started.notify_one(); - release_first.notified().await; - processed.lock().await.push("first"); - } else { - second_started.notify_one(); - processed.lock().await.push("second"); - } - } - } - })); - - drop(wake_sender); - - first_started.notified().await; - assert!( - tokio::time::timeout( - std::time::Duration::from_millis(25), - second_started.notified() - ) - .await - .is_err(), - "the second callback started before the first completed" - ); - release_first.notify_one(); - supervisor.await.unwrap(); - - assert_eq!(*processed.lock().await, ["first", "second"]); - } - - #[tokio::test] - async fn accounts_colliding_under_prior_sharding_run_independently() { - let queues = Arc::new(Mutex::new(CallbackQueues::default())); - let (wake_sender, wake_receiver) = mpsc::channel(1); - let permits = Arc::new(Semaphore::new(CALLBACK_QUEUE_CAPACITY)); - let mut accounts_by_prior_worker = HashMap::new(); - let (hot_account, colliding_account) = (0..=CALLBACK_WORKER_LIMIT) - .find_map(|index| { - let account = format!("colliding-account-{index}"); - accounts_by_prior_worker - .insert(prior_worker_index(&account), account.clone()) - .map(|prior| (prior, account)) - }) - .unwrap(); - let hot_started = Arc::new(tokio::sync::Notify::new()); - let release_hot = Arc::new(tokio::sync::Notify::new()); - let colliding_started = Arc::new(tokio::sync::Notify::new()); - - assert!(queues.lock().unwrap().try_enqueue(account_job( - &permits, - &hot_account, - b"hot-blocking" - ))); - wake_sender.try_send(()).unwrap(); - - let supervisor = tokio::spawn(run_fair_workers(Arc::clone(&queues), wake_receiver, { - let hot_started = Arc::clone(&hot_started); - let release_hot = Arc::clone(&release_hot); - let colliding_started = Arc::clone(&colliding_started); - move |job: SlackCallbackJob| { - let hot_started = Arc::clone(&hot_started); - let release_hot = Arc::clone(&release_hot); - let colliding_started = Arc::clone(&colliding_started); - async move { - if job.body == Bytes::from_static(b"hot-blocking") { - hot_started.notify_one(); - release_hot.notified().await; - } else if job.body == Bytes::from_static(b"colliding") { - colliding_started.notify_one(); - } - } - } - })); - - hot_started.notified().await; - assert!(queues.lock().unwrap().try_enqueue(account_job( - &permits, - &colliding_account, - b"colliding" - ))); - assert!( - !matches!( - wake_sender.try_send(()), - Err(mpsc::error::TrySendError::Closed(())) - ), - "scheduler stopped before colliding account was admitted" - ); - - assert!( - tokio::time::timeout( - std::time::Duration::from_secs(1), - colliding_started.notified(), - ) - .await - .is_ok(), - "accounts colliding under the old shard hash blocked each other" - ); - - release_hot.notify_one(); - drop(wake_sender); - supervisor.await.unwrap(); } } diff --git a/crates/slack/src/callback_worker.rs b/crates/slack/src/callback_worker.rs index dfd027b471..b3570430d9 100644 --- a/crates/slack/src/callback_worker.rs +++ b/crates/slack/src/callback_worker.rs @@ -1,7 +1,10 @@ use std::{future::Future, sync::Arc}; use { - moltis_channels::plugin::{ChannelEventSink, ChannelReplyTarget}, + moltis_channels::{ + fair_queue::{Admission, FairQueue, FairQueueConfig, FairQueueJob}, + plugin::{ChannelEventSink, ChannelReplyTarget}, + }, slack_morphism::prelude::SlackPushEventCallback, tracing::warn, }; @@ -11,7 +14,12 @@ use crate::{ state::{AccountStateMap, DedupKind, EventDedup}, }; +/// Total Socket Mode callbacks that may be queued or in flight, across every +/// account. const CALLBACK_QUEUE_CAPACITY: usize = 256; +/// Callbacks processed concurrently, and so the number of accounts served at +/// once. Matches the HTTP callback path so both behave the same under load. +const CALLBACK_WORKER_LIMIT: usize = 16; pub(crate) enum CallbackJob { Push { @@ -33,14 +41,35 @@ pub(crate) enum CallbackJob { response_url: Option, }, ResponseUrl { + account_id: String, response_url: String, text: String, }, } +impl FairQueueJob for CallbackJob { + fn account_id(&self) -> &str { + match self { + Self::Push { account_id, .. } | Self::ResponseUrl { account_id, .. } => account_id, + // A command or interaction is addressed to the account its reply + // target names. + Self::Command { reply_to, .. } | Self::Interaction { reply_to, .. } => { + &reply_to.account_id + }, + } + } +} + +/// Bounded, account-fair queue for Socket Mode callbacks. +/// +/// Slack expects the socket callback to be acknowledged in seconds, so work is +/// admitted synchronously and processed afterwards. Accounts are served +/// round-robin with a capped share each, so one busy workspace cannot +/// head-of-line block the others β€” the same scheduler the HTTP callback path +/// uses. #[derive(Clone)] pub(crate) struct CallbackQueue { - sender: tokio::sync::mpsc::Sender, + queue: Arc>, cancel: tokio_util::sync::CancellationToken, } @@ -69,9 +98,29 @@ impl std::error::Error for CallbackAdmissionError {} impl CallbackQueue { pub(crate) fn start(cancel: tokio_util::sync::CancellationToken) -> Self { - let (sender, receiver) = bounded_channel(CALLBACK_QUEUE_CAPACITY); - tokio::spawn(run_worker(receiver, cancel.clone())); - Self { sender, cancel } + Self::start_with( + FairQueueConfig::new(CALLBACK_QUEUE_CAPACITY, CALLBACK_WORKER_LIMIT), + cancel, + process, + ) + } + + /// Start with an explicit config and processor, so tests can use a small + /// capacity and a processor that does not reach the network. + fn start_with( + config: FairQueueConfig, + cancel: tokio_util::sync::CancellationToken, + process: F, + ) -> Self + where + F: Fn(CallbackJob) -> Fut + Clone + Send + Sync + 'static, + Fut: Future + Send + 'static, + { + let queue = FairQueue::start(config, &cancel, process); + Self { + queue: Arc::new(queue), + cancel, + } } /// Admit work immediately, allowing the Socket Mode callback to ACK in time. @@ -79,18 +128,17 @@ impl CallbackQueue { if self.cancel.is_cancelled() { return Err(CallbackAdmissionError::Canceled); } - match self.sender.try_send(job) { - Ok(()) => Ok(()), - Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { - Err(CallbackAdmissionError::Full) - }, - Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => { - Err(CallbackAdmissionError::Canceled) - }, + match self.queue.admit(job) { + Admission::Admitted => Ok(()), + Admission::Rejected => Err(CallbackAdmissionError::Full), } } /// Atomically check deduplication and commit it only after queue admission. + /// + /// Committing only on success matters: a callback refused for capacity gets + /// no acknowledgement, so Slack redelivers it, and that retry must be + /// admitted rather than dropped as an already-seen duplicate. pub(crate) fn try_send_deduplicated( &self, dedup: &mut EventDedup, @@ -106,56 +154,11 @@ impl CallbackQueue { debug_assert!(inserted, "dedup changed while its lock was held"); Ok(CallbackAdmission::Queued) } -} - -fn bounded_channel( - capacity: usize, -) -> (tokio::sync::mpsc::Sender, tokio::sync::mpsc::Receiver) { - tokio::sync::mpsc::channel(capacity) -} -async fn run_worker( - mut receiver: tokio::sync::mpsc::Receiver, - cancel: tokio_util::sync::CancellationToken, -) { - loop { - let job = tokio::select! { - () = cancel.cancelled() => { - // Drain what was already admitted before shutting down: those - // callbacks were acknowledged to Slack and will not be redelivered. - receiver.close(); - while let Some(job) = receiver.recv().await { - process_isolated(job).await; - } - break; - }, - job = receiver.recv() => match job { - Some(job) => job, - None => break, - }, - }; - process_isolated(job).await; - } -} - -async fn process_isolated(job: CallbackJob) { - run_isolated(process(job)).await; -} - -/// Run one callback so a panic inside it cannot take the worker down. -/// -/// There is a single worker behind this queue, so an unguarded panic would stop -/// *all* subsequent Socket Mode callbacks for the life of the process β€” the bot -/// would go quiet with nothing but the panic in the log to say why. Running on a -/// dedicated task localizes the unwind, and awaiting it immediately keeps -/// callbacks strictly ordered. Mirrors the panic isolation the HTTP callback -/// workers already have. -async fn run_isolated(future: F) -where - F: Future + Send + 'static, -{ - if let Err(error) = tokio::spawn(future).await { - warn!("Slack callback job panicked: {error}"); + /// Callbacks queued or being processed. Test and diagnostic use. + #[cfg(test)] + fn in_flight(&self) -> usize { + self.queue.in_flight() } } @@ -197,7 +200,11 @@ async fn process(job: CallbackJob) { warn!("failed to send Slack interaction response: {error}"); } }, - CallbackJob::ResponseUrl { response_url, text } => { + CallbackJob::ResponseUrl { + response_url, + text, + account_id: _, + } => { if let Err(error) = post_response_url(&response_url, &text).await { warn!("failed to send Slack callback response: {error}"); } @@ -210,76 +217,116 @@ async fn process(job: CallbackJob) { mod tests { use super::*; + use tokio_util::sync::CancellationToken; + + fn job(account_id: &str) -> CallbackJob { + CallbackJob::ResponseUrl { + account_id: account_id.to_string(), + response_url: "https://hooks.slack.com/actions/test".to_string(), + text: "body".to_string(), + } + } + + /// A queue whose jobs block on `gate` until it is cancelled, so admission + /// can be driven to saturation without touching the network. + fn blocking_queue(capacity: usize, gate: CancellationToken) -> CallbackQueue { + CallbackQueue::start_with( + FairQueueConfig { + capacity, + workers: 1, + per_account_capacity: capacity, + }, + CancellationToken::new(), + move |_job| { + let gate = gate.clone(); + async move { gate.cancelled().await } + }, + ) + } + #[tokio::test] - async fn queue_full_and_cancellation_do_not_commit_dedup() { - let cancel = tokio_util::sync::CancellationToken::new(); - let (sender, mut receiver) = bounded_channel(1); - let queue = CallbackQueue { - sender, - cancel: cancel.clone(), - }; + async fn a_rejected_callback_does_not_commit_dedup() { + // Slack got no ACK for a refused callback and will redeliver it, so the + // retry must be admitted rather than dropped as already-seen. + let gate = CancellationToken::new(); + let queue = blocking_queue(1, gate.clone()); let mut dedup = EventDedup::default(); - let job = |text: &str| CallbackJob::ResponseUrl { - response_url: "https://hooks.slack.com/actions/test".to_string(), - text: text.to_string(), - }; assert_eq!( - queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("first")), + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("acct")), Ok(CallbackAdmission::Queued) ); assert_eq!( - queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("duplicate")), + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "first", job("acct")), Ok(CallbackAdmission::Duplicate) ); assert_eq!( - queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("full")), + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("acct")), Err(CallbackAdmissionError::Full) ); assert!(!dedup.contains(DedupKind::Event, "retry")); - assert!(receiver.recv().await.is_some()); + gate.cancel(); + while queue.in_flight() > 0 { + tokio::task::yield_now().await; + } assert_eq!( - queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("retry")), + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "retry", job("acct")), Ok(CallbackAdmission::Queued) ); - assert!(receiver.recv().await.is_some()); + } + #[tokio::test] + async fn a_canceled_queue_admits_nothing_and_commits_nothing() { + let cancel = CancellationToken::new(); + let queue = CallbackQueue::start_with( + FairQueueConfig::new(16, 1), + cancel.clone(), + |_job| async move {}, + ); + let mut dedup = EventDedup::default(); cancel.cancel(); + assert_eq!( - queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "canceled", job("canceled")), + queue.try_send_deduplicated(&mut dedup, DedupKind::Event, "canceled", job("acct")), Err(CallbackAdmissionError::Canceled) ); assert!(!dedup.contains(DedupKind::Event, "canceled")); } #[tokio::test] - async fn a_panicking_job_does_not_stop_later_jobs() { - // There is one worker behind this queue, so an unguarded panic would - // silence every subsequent Slack callback for the life of the process. - let ran = Arc::new(std::sync::atomic::AtomicUsize::new(0)); - - let panicking = { - let ran = Arc::clone(&ran); - async move { - ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - panic!("callback exploded"); - } + async fn a_busy_account_does_not_block_another_accounts_callbacks() { + // The regression this queue exists to prevent: before it, one worker + // served every account, so a slow workspace stalled all the others. + let gate = CancellationToken::new(); + let ran = Arc::new(tokio::sync::Notify::new()); + let queue = { + let (gate, ran) = (gate.clone(), Arc::clone(&ran)); + CallbackQueue::start_with( + FairQueueConfig::new(16, 4), + CancellationToken::new(), + move |job: CallbackJob| { + let (gate, ran) = (gate.clone(), Arc::clone(&ran)); + async move { + if job.account_id() == "busy" { + gate.cancelled().await; + } else { + ran.notify_one(); + } + } + }, + ) }; - run_isolated(panicking).await; - let following = { - let ran = Arc::clone(&ran); - async move { - ran.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } - }; - run_isolated(following).await; + assert!(queue.try_send(job("busy")).is_ok()); + assert!(queue.try_send(job("quiet")).is_ok()); - assert_eq!( - ran.load(std::sync::atomic::Ordering::SeqCst), - 2, - "the job after the panicking one never ran" + assert!( + tokio::time::timeout(std::time::Duration::from_secs(1), ran.notified()) + .await + .is_ok(), + "a busy Slack account head-of-line blocked an unrelated one" ); + gate.cancel(); } } diff --git a/crates/slack/src/socket/callbacks.rs b/crates/slack/src/socket/callbacks.rs index e7f61ec75f..8845931396 100644 --- a/crates/slack/src/socket/callbacks.rs +++ b/crates/slack/src/socket/callbacks.rs @@ -203,6 +203,7 @@ pub(super) async fn interaction_events_callback( DedupKind::Interaction, &trigger_id, CallbackJob::ResponseUrl { + account_id: account_id.to_string(), response_url, text: "Access denied.".to_string(), }, From b65f855cbdff42fe4465c5bdf5a24ec36169ef9e Mon Sep 17 00:00:00 2001 From: Fabien Penso Date: Wed, 29 Jul 2026 17:48:19 -0700 Subject: [PATCH 23/23] fix(httpd): gate Slack callback throttle state The Swift bridge intentionally builds moltis-httpd without the Slack feature, but the callback throttle variant and limits were still compiled and reported as dead code. Gate the Slack-only state and match paths while retaining them for tests, so both minimal and Slack-enabled builds remain warning-free. --- crates/httpd/src/request_throttle.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/httpd/src/request_throttle.rs b/crates/httpd/src/request_throttle.rs index e52fdaf177..7edff71bcc 100644 --- a/crates/httpd/src/request_throttle.rs +++ b/crates/httpd/src/request_throttle.rs @@ -37,6 +37,7 @@ enum ThrottleScope { Api, Share, Ws, + #[cfg(any(feature = "slack", test))] SlackCallbackPreauth, } @@ -65,7 +66,11 @@ impl ThrottleScope { } fn permits_authenticated_bypass(self) -> bool { - !matches!(self, Self::SlackCallbackPreauth) + #[cfg(any(feature = "slack", test))] + if self == Self::SlackCallbackPreauth { + return false; + } + true } } @@ -94,6 +99,7 @@ struct ThrottleLimits { api: RateLimit, share: RateLimit, ws: RateLimit, + #[cfg(any(feature = "slack", test))] callback_preauth: RateLimit, } @@ -125,6 +131,7 @@ impl Default for ThrottleLimits { max_requests: 30, window: Duration::from_secs(60), }, + #[cfg(any(feature = "slack", test))] // This IP-level ceiling runs before account lookup and signature // verification. It is intentionally above Slack's three callback // endpoints at their verified 600/minute + 200 burst policy. @@ -162,6 +169,7 @@ impl RequestThrottle { ThrottleScope::Api => self.limits.api, ThrottleScope::Share => self.limits.share, ThrottleScope::Ws => self.limits.ws, + #[cfg(any(feature = "slack", test))] ThrottleScope::SlackCallbackPreauth => self.limits.callback_preauth, } } @@ -220,17 +228,19 @@ impl RequestThrottle { } fn max_window(&self) -> Duration { - [ + let max_window = [ self.limits.login.window, self.limits.auth_api.window, self.limits.api.window, self.limits.share.window, self.limits.ws.window, - self.limits.callback_preauth.window, ] .into_iter() .max() - .unwrap_or(Duration::from_secs(60)) + .unwrap_or(Duration::from_secs(60)); + #[cfg(any(feature = "slack", test))] + let max_window = max_window.max(self.limits.callback_preauth.window); + max_window } }