diff --git a/crates/ctl/src/commands/chat.rs b/crates/ctl/src/commands/chat.rs new file mode 100644 index 0000000000..d87f26330f --- /dev/null +++ b/crates/ctl/src/commands/chat.rs @@ -0,0 +1,47 @@ +//! Chat subcommands. + +use { + crate::client::CtlClient, + serde_json::{Value, json}, +}; + +pub async fn send(client: &mut CtlClient, text: &str, session_key: &str) -> anyhow::Result { + client + .call( + "chat.send", + json!({ + "text": text, + "_session_key": session_key, + }), + ) + .await + .map_err(Into::into) +} + +pub async fn history(client: &mut CtlClient, session_key: &str) -> anyhow::Result { + client + .call( + "chat.history", + json!({ + "_session_key": session_key, + }), + ) + .await + .map_err(Into::into) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn send_params_preserve_terminal_bench_session() { + let params = json!({ + "text": "fix the task", + "_session_key": "terminal-bench:task-1", + }); + + assert_eq!(params["text"], "fix the task"); + assert_eq!(params["_session_key"], "terminal-bench:task-1"); + } +} diff --git a/crates/ctl/src/commands/mod.rs b/crates/ctl/src/commands/mod.rs index d0e6b84ab8..2522179fb4 100644 --- a/crates/ctl/src/commands/mod.rs +++ b/crates/ctl/src/commands/mod.rs @@ -1,5 +1,6 @@ //! CLI subcommand modules. +pub mod chat; pub mod config; pub mod health; pub mod mcp; diff --git a/crates/ctl/src/main.rs b/crates/ctl/src/main.rs index ae54b47758..22ce6a2f06 100644 --- a/crates/ctl/src/main.rs +++ b/crates/ctl/src/main.rs @@ -49,6 +49,23 @@ struct Cli { #[derive(Subcommand)] enum Command { + /// Send a message through the Moltis agent loop and wait for completion. + Chat { + /// Message to send to the agent. + text: String, + + /// Session key used to isolate this conversation. + #[arg(long, default_value = "terminal-bench")] + session_key: String, + }, + + /// Fetch persisted messages for a chat session. + ChatHistory { + /// Session key whose history should be returned. + #[arg(long, default_value = "terminal-bench")] + session_key: String, + }, + /// Check gateway health. Health, /// Show gateway status. @@ -90,6 +107,12 @@ async fn main() { }; let result = match cli.command { + Command::Chat { text, session_key } => { + commands::chat::send(&mut client, &text, &session_key).await + }, + Command::ChatHistory { session_key } => { + commands::chat::history(&mut client, &session_key).await + }, Command::Health => commands::health::health(&mut client).await, Command::Status => commands::health::status(&mut client).await, Command::Mcp(cmd) => commands::mcp::run(&mut client, cmd).await, diff --git a/tools/terminal_bench/README.md b/tools/terminal_bench/README.md new file mode 100644 index 0000000000..72012ee6b5 --- /dev/null +++ b/tools/terminal_bench/README.md @@ -0,0 +1,51 @@ +# Running Moltis in Terminal-Bench / Harbor + +`moltis-ctl chat` drives a task through the same authenticated WebSocket RPC and +agent loop as the web UI. The wrapper in this directory gives Harbor a simple +installed-agent entry point. + +## Build + +```sh +cargo build --release -p moltis-ctl +``` + +Make `target/release/moltis-ctl` available in the task container (or set +`MOLTIS_CTL_BIN` to its mounted path), then configure the installed-agent +command as: + +```sh +tools/terminal_bench/run_moltis_agent.sh +``` + +The wrapper accepts the task instruction as arguments, from stdin, or via +`HARBOR_TASK_INSTRUCTION`. + +## Required environment + +- `MOLTIS_API_KEY`: gateway API key (required; do not commit it) +- `MOLTIS_GATEWAY_URL`: gateway URL, default + `http://host.docker.internal:13131` +- `MOLTIS_CTL_BIN`: `moltis-ctl` path, default `moltis-ctl` +- `MOLTIS_SESSION_KEY`: optional stable per-task session key; defaults to + `terminal-bench:${HARBOR_TASK_ID:-task}` + +Each task must use a distinct session key so model/tool history cannot leak +between benchmark tasks. + +## Smoke test + +A representative end-to-end task can be run without Harbor orchestration: + +```sh +MOLTIS_API_KEY='...' \ +MOLTIS_GATEWAY_URL='http://127.0.0.1:13131' \ +tools/terminal_bench/run_moltis_agent.sh \ + 'Create /tmp/terminal-bench-smoke.txt containing exactly moltis-ok' + +test "$(cat /tmp/terminal-bench-smoke.txt)" = moltis-ok +``` + +The `chat` command does not return until the Moltis agent loop finishes. Its +JSON output includes the assistant text and token/round usage. Use +`moltis-ctl chat-history --session-key ` when debugging a failed run. diff --git a/tools/terminal_bench/run_moltis_agent.sh b/tools/terminal_bench/run_moltis_agent.sh new file mode 100755 index 0000000000..fe0c7cea45 --- /dev/null +++ b/tools/terminal_bench/run_moltis_agent.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${MOLTIS_API_KEY:?MOLTIS_API_KEY must be set}" + +MOLTIS_CTL_BIN="${MOLTIS_CTL_BIN:-moltis-ctl}" +MOLTIS_GATEWAY_URL="${MOLTIS_GATEWAY_URL:-http://host.docker.internal:13131}" +MOLTIS_SESSION_KEY="${MOLTIS_SESSION_KEY:-terminal-bench:${HARBOR_TASK_ID:-task}}" + +if (($#)); then + instruction="$*" +elif [[ -n "${HARBOR_TASK_INSTRUCTION:-}" ]]; then + instruction="$HARBOR_TASK_INSTRUCTION" +else + instruction="$(cat)" +fi + +if [[ -z "${instruction//[[:space:]]/}" ]]; then + echo "terminal-bench: missing task instruction" >&2 + exit 2 +fi + +exec "$MOLTIS_CTL_BIN" \ + --gateway-url "$MOLTIS_GATEWAY_URL" \ + --api-key "$MOLTIS_API_KEY" \ + chat \ + --session-key "$MOLTIS_SESSION_KEY" \ + "$instruction" diff --git a/tools/terminal_bench/test_run_moltis_agent.sh b/tools/terminal_bench/test_run_moltis_agent.sh new file mode 100755 index 0000000000..c468693f4e --- /dev/null +++ b/tools/terminal_bench/test_run_moltis_agent.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +cat >"$tmp/moltis-ctl" <<'MOCK' +#!/usr/bin/env bash +printf '%s\n' "$@" >"${CAPTURE_FILE:?}" +MOCK +chmod +x "$tmp/moltis-ctl" + +export CAPTURE_FILE="$tmp/args" +export MOLTIS_API_KEY="test-key" +export MOLTIS_CTL_BIN="$tmp/moltis-ctl" +export MOLTIS_GATEWAY_URL="http://gateway:13131" +export HARBOR_TASK_ID="representative-task" + +"$script_dir/run_moltis_agent.sh" "write" "the file" + +mapfile -t args <"$CAPTURE_FILE" +expected=( + --gateway-url http://gateway:13131 + --api-key test-key + chat + --session-key terminal-bench:representative-task + "write the file" +) +[[ "${args[*]}" == "${expected[*]}" ]] + +unset HARBOR_TASK_ID +export MOLTIS_SESSION_KEY="terminal-bench:stdin-task" +printf 'instruction from stdin\n' | "$script_dir/run_moltis_agent.sh" +mapfile -t args <"$CAPTURE_FILE" +[[ "${args[6]}" == "terminal-bench:stdin-task" ]] +[[ "${args[7]}" == "instruction from stdin" ]] + +if MOLTIS_API_KEY='' "$script_dir/run_moltis_agent.sh" test 2>/dev/null; then + echo "expected missing API key to fail" >&2 + exit 1 +fi + +echo "terminal-bench wrapper tests passed"