From 9f6c9cebf470e67b0798e899608f2a2a62ab8728 Mon Sep 17 00:00:00 2001 From: Sercan Degirmenci Date: Mon, 13 Jul 2026 03:18:02 +0300 Subject: [PATCH 1/4] feat(server): use shared model-agnostic canonical request/response types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce crates/spark-server/src/ir/ — a provider-free chat IR (ChatRequest, Message with content always a list of parts, ChatResponse, StreamDelta) — and migrate every API surface onto it: openai/to_ir.rs + anthropic/to_ir.rs adapt requests in; encode.rs / encode_stream.rs / translate.rs / responses_translate.rs serialize responses out. Deletes the Anthropic struct→JSON→struct round-trip, the Responses-API image drop, and the / [tool error] string prefixes (now real fields). Images ride every role uniformly, including tool results (issue #165). Also scopes the post-think EOS guard to force-closed spans (sticky ActiveSeq::think_force_closed): it previously fired on every tools-armed turn, eating the model's natural <|im_end|> after brief post-tool answers and causing scaffold garbage + spurious duplicate tool calls. Verified live on Qwen3.6-35B-A3B-NVFP4: turn-2 /v1/messages replays 4/4 clean end_turn; 2-turn tool+image agent loop 3/3 clean. Co-Authored-By: Claude Fable 5 --- crates/spark-server/src/anthropic/convert.rs | 41 -- crates/spark-server/src/anthropic/handlers.rs | 310 ++++--------- .../src/anthropic/handlers_stream.rs | 99 +--- crates/spark-server/src/anthropic/helpers.rs | 6 +- crates/spark-server/src/anthropic/mod.rs | 2 +- .../src/anthropic/tests/ir_carry.rs | 413 +++++++++++++++++ .../spark-server/src/anthropic/tests/mod.rs | 3 + .../src/anthropic/tests/translator_stream.rs | 232 ++++++++++ crates/spark-server/src/anthropic/to_ir.rs | 248 ++++++++++ .../spark-server/src/anthropic/translate.rs | 437 ++---------------- .../spark-server/src/anthropic/translator.rs | 389 +++++++--------- crates/spark-server/src/anthropic/types.rs | 37 ++ crates/spark-server/src/api/chat/echo.rs | 33 ++ .../spark-server/src/api/chat/loop_detect.rs | 42 +- crates/spark-server/src/api/chat/mod.rs | 161 +++---- crates/spark-server/src/api/chat/msg_entry.rs | 272 ++++++++--- crates/spark-server/src/api/chat/prepare.rs | 130 ++++++ .../src/api/chat/sampling_setup.rs | 95 ++-- crates/spark-server/src/api/chat/template.rs | 257 ++++++++-- crates/spark-server/src/api/chat/thinking.rs | 198 +++++++- crates/spark-server/src/api/chat_blocking.rs | 193 ++++---- crates/spark-server/src/api/chat_phases.rs | 18 +- .../spark-server/src/api/chat_stream/ctx.rs | 1 - .../src/api/chat_stream/handle_done.rs | 144 +++--- .../src/api/chat_stream/handle_error.rs | 13 +- .../src/api/chat_stream/handle_token.rs | 107 ++--- .../spark-server/src/api/chat_stream/mod.rs | 42 +- .../spark-server/src/api/chat_stream/state.rs | 4 +- .../src/api/chat_stream/tool_handlers.rs | 179 ++++--- .../src/api/chat_stream_dispatch.rs | 29 +- crates/spark-server/src/api/completions.rs | 4 +- .../spark-server/src/api/completions_exec.rs | 6 +- crates/spark-server/src/api/conversations.rs | 4 +- crates/spark-server/src/api/inference_impl.rs | 20 +- .../spark-server/src/api/inference_types.rs | 28 +- crates/spark-server/src/api/misc_handlers.rs | 4 +- crates/spark-server/src/api/mod.rs | 1 + crates/spark-server/src/api/responses.rs | 6 +- .../spark-server/src/api/responses_stream.rs | 286 ++++++------ .../src/api/responses_stream_finalize.rs | 2 +- .../src/api/responses_translate.rs | 164 ++----- crates/spark-server/src/api/stored.rs | 48 +- crates/spark-server/src/citation.rs | 424 +++++++++++++---- crates/spark-server/src/conversation_store.rs | 2 +- crates/spark-server/src/ids.rs | 58 +++ crates/spark-server/src/ir/message.rs | 192 ++++++++ crates/spark-server/src/ir/mod.rs | 21 + crates/spark-server/src/ir/request.rs | 122 +++++ crates/spark-server/src/ir/response.rs | 117 +++++ crates/spark-server/src/ir/stream.rs | 50 ++ crates/spark-server/src/ir/tests.rs | 80 ++++ crates/spark-server/src/main.rs | 2 + .../src/main_modules/app_state.rs | 7 +- .../src/main_modules/middleware.rs | 4 +- crates/spark-server/src/main_modules/serve.rs | 42 +- crates/spark-server/src/metrics.rs | 13 - crates/spark-server/src/openai/annotations.rs | 242 +--------- .../spark-server/src/openai/chat_message.rs | 105 +++-- .../spark-server/src/openai/chat_request.rs | 186 +++----- crates/spark-server/src/openai/completions.rs | 1 + crates/spark-server/src/openai/encode.rs | 146 ++++++ .../spark-server/src/openai/encode_stream.rs | 355 ++++++++++++++ crates/spark-server/src/openai/mod.rs | 60 +-- .../src/openai/responses_lowering.rs | 37 -- crates/spark-server/src/openai/tests.rs | 278 +++++++++-- crates/spark-server/src/openai/to_ir.rs | 369 +++++++++++++++ .../src/scheduler/decode_logits_step.rs | 9 +- .../spark-server/src/scheduler/emit_step.rs | 3 + crates/spark-server/src/scheduler/helpers.rs | 6 +- .../src/scheduler/helpers_tests.rs | 6 +- .../spark-server/src/scheduler/lifecycle.rs | 2 + .../src/scheduler/phase_promote_prefills.rs | 1 + .../src/scheduler/prefill_a_step.rs | 2 + .../src/scheduler/prefill_a_step_params.rs | 2 +- .../src/scheduler/prefill_b_step.rs | 2 + crates/spark-server/src/scheduler/types.rs | 10 +- scripts/test-qwen36-tool-image.sh | 402 ++++++++++++++++ 77 files changed, 5471 insertions(+), 2595 deletions(-) delete mode 100644 crates/spark-server/src/anthropic/convert.rs create mode 100644 crates/spark-server/src/anthropic/tests/ir_carry.rs create mode 100644 crates/spark-server/src/anthropic/tests/translator_stream.rs create mode 100644 crates/spark-server/src/anthropic/to_ir.rs create mode 100644 crates/spark-server/src/api/chat/echo.rs create mode 100644 crates/spark-server/src/api/chat/prepare.rs create mode 100644 crates/spark-server/src/ids.rs create mode 100644 crates/spark-server/src/ir/message.rs create mode 100644 crates/spark-server/src/ir/mod.rs create mode 100644 crates/spark-server/src/ir/request.rs create mode 100644 crates/spark-server/src/ir/response.rs create mode 100644 crates/spark-server/src/ir/stream.rs create mode 100644 crates/spark-server/src/ir/tests.rs create mode 100644 crates/spark-server/src/openai/encode.rs create mode 100644 crates/spark-server/src/openai/encode_stream.rs create mode 100644 crates/spark-server/src/openai/to_ir.rs create mode 100755 scripts/test-qwen36-tool-image.sh diff --git a/crates/spark-server/src/anthropic/convert.rs b/crates/spark-server/src/anthropic/convert.rs deleted file mode 100644 index 66c311984..000000000 --- a/crates/spark-server/src/anthropic/convert.rs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -use crate::tool_parser; - -use super::types::{AnthropicContent, ContentBlock}; - -/// Flatten Anthropic message content blocks into a single text string. -/// For assistant messages with tool_use blocks, also extracts tool calls -/// so they can be formatted by the tool parser for multi-turn. -pub(super) fn flatten_content( - content: &AnthropicContent, -) -> (String, Vec) { - match content { - AnthropicContent::Text(s) => (s.clone(), Vec::new()), - AnthropicContent::Blocks(blocks) => { - let mut text_parts = Vec::new(); - let mut tool_calls = Vec::new(); - for block in blocks { - match block { - ContentBlock::Text { text } => text_parts.push(text.clone()), - ContentBlock::ToolUse { id, name, input } => { - tool_calls.push(tool_parser::IncomingToolCall { - id: Some(id.clone()), - function: tool_parser::IncomingFunction { - name: name.clone(), - arguments: input.to_string(), - }, - }); - } - ContentBlock::ToolResult { content, .. } => { - if let Some(c) = content { - text_parts.push(c.to_text()); - } - } - ContentBlock::Thinking { .. } | ContentBlock::Unknown => {} - } - } - (text_parts.join(""), tool_calls) - } - } -} diff --git a/crates/spark-server/src/anthropic/handlers.rs b/crates/spark-server/src/anthropic/handlers.rs index 910c351e4..e6d5e2137 100644 --- a/crates/spark-server/src/anthropic/handlers.rs +++ b/crates/spark-server/src/anthropic/handlers.rs @@ -8,9 +8,7 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; use crate::AppState; -use crate::openai; -use super::convert::*; use super::handlers_stream::*; use super::helpers::*; use super::translate::*; @@ -20,12 +18,13 @@ use super::types::*; /// POST /v1/messages — Anthropic Messages API. /// -/// Translates the request into an OpenAI `ChatCompletionRequest`, dispatches -/// through `api::chat_completions_inner` (which runs every fix12-23 +/// Lowers the request directly into the canonical chat IR +/// (`MessagesRequest::into_ir`), dispatches through +/// `api::chat_completions_inner` (which runs every fix12-23 /// sanitization, salvage, watchdog, and dump path), and translates the -/// response back into Anthropic format. The Anthropic-specific surface is -/// strictly format conversion — no policy or sampling decisions are made -/// here. +/// response back into Anthropic format. The Anthropic-specific surface +/// is strictly format conversion — no policy or sampling decisions are +/// made here. pub async fn messages(State(state): State>, body: axum::body::Bytes) -> Response { // 1. Parse the Anthropic request. let req: MessagesRequest = match serde_json::from_slice(&body) { @@ -68,36 +67,40 @@ pub async fn messages(State(state): State>, body: axum::body::Byte } }); - // 3. Translate to a ChatCompletionRequest. - let chat_json = anthropic_to_chat_request_json(&req); - - // Translation-drift telemetry (P5.1, 2026-04-25). Detects whether - // structural information from the Anthropic request was preserved - // in the translated OpenAI shape — a count-level audit cheap - // enough to run on every request. The metric counts mismatches; - // verbose diff logging is gated behind ATLAS_DEBUG_TRANSLATION_DRIFT - // (an opt-in for forensics, not a hot-path log). - audit_translation_drift(&req, &chat_json); - - let chat_req: openai::ChatCompletionRequest = match serde_json::from_value(chat_json) { - Ok(r) => r, - Err(e) => { - return anthropic_error( - StatusCode::BAD_REQUEST, - "invalid_request_error", - format!("Translation error: {e}"), - ); + // 3. Lower the Anthropic wire request straight into the IR envelope + // (typed peer adapter — no OpenAI-wire-JSON intermediary). The + // old count-level translation-drift audit is gone with the JSON + // hop: a typed constructor makes that failure class + // unrepresentable at runtime; the adapter unit tests own it. + // + // 4. Run the shared pipeline. All sanitization, salvage, watchdog, + // sampling preset, and prompt mutation logic lives there. + // Anthropic has no echo-only wire fields (service_tier / store / + // stream_options), so the echo context stays default. + let outcome = + crate::api::chat_completions_inner(state.clone(), None, req.into_ir(), None).await; + + // 5. Encode back to Anthropic shape. Non-streaming success arrives + // as the response IR — no body re-parse; a malformed inner body + // is now structurally impossible. + let chat_resp = match outcome { + crate::api::ChatOutcome::Blocking(ir) => { + let messages_resp = ir_to_anthropic_response(*ir); + if let (Some(seq), Some(dump)) = (dump_seq, state.dump_writer.as_ref()) { + dump.dump_response("/v1/messages", seq, &messages_resp, false); + } + return Json(messages_resp).into_response(); } + crate::api::ChatOutcome::Streaming(deltas) => { + // Note: streaming dump from the Anthropic side is + // best-effort; Anthropic-shape response capture is a + // follow-up. + let _ = dump_seq; + return anthropic_sse_from_deltas(deltas, model_echo); + } + crate::api::ChatOutcome::Http(r) => r, }; - // Drop `req` here — anthropic_to_chat_request_json already cloned what - // it needed, and we've finished extracting `stream` + `model_echo`. - - // 4. Run the shared OpenAI pipeline. All sanitization, salvage, - // watchdog, sampling preset, and prompt mutation logic lives there. - let chat_resp = crate::api::chat_completions_inner(state.clone(), None, chat_req, None).await; - - // 5. Translate the response back to Anthropic shape. if !chat_resp.status().is_success() { // Forward the error envelope. Translate the JSON body into // Anthropic's error shape if it's an OpenAI-style envelope; else @@ -125,48 +128,11 @@ pub async fn messages(State(state): State>, body: axum::body::Byte return anthropic_error(parts.status, "api_error", err_msg); } - if stream { - let resp = wrap_chat_sse_for_anthropic(chat_resp, model_echo).await; - // Note: streaming dump from the Anthropic side is best-effort; - // chat_completions_inner already wrote a synthesized OpenAI-shape - // response dump entry under endpoint="/v1/chat/completions" if - // we'd passed a dump_seq. Anthropic-shape response capture is a - // follow-up. - let _ = dump_seq; - resp - } else { - let (parts, body) = chat_resp.into_parts(); - let body_bytes = match axum::body::to_bytes(body, usize::MAX).await { - Ok(b) => b, - Err(e) => { - return anthropic_error( - StatusCode::INTERNAL_SERVER_ERROR, - "api_error", - format!("Body collect error: {e}"), - ); - } - }; - let chat_value: serde_json::Value = match serde_json::from_slice(&body_bytes) { - Ok(v) => v, - Err(e) => { - return anthropic_error( - StatusCode::INTERNAL_SERVER_ERROR, - "api_error", - format!("Inner response decode error: {e}"), - ); - } - }; - let messages_resp = chat_to_anthropic_response(&chat_value, model_echo); - - if let (Some(seq), Some(dump)) = (dump_seq, state.dump_writer.as_ref()) { - dump.dump_response("/v1/messages", seq, &messages_resp, false); - } - - // Preserve status code/headers from chat_completions_inner and - // serialize the translated body. - let json_bytes = serde_json::to_vec(&messages_resp).unwrap_or_default(); - Response::from_parts(parts, axum::body::Body::from(json_bytes)) - } + // Unreachable in practice: Http outcomes are error envelopes (both + // success shapes returned above), and the error branch returned too. + // Fall back to forwarding whatever arrived. + let _ = stream; + chat_resp } // ── Count tokens endpoint ── @@ -189,160 +155,52 @@ pub async fn count_tokens( } }; - let tools_active = - state.tool_call_parser.is_some() && req.tools.as_ref().is_some_and(|t| !t.is_empty()); - let enable_thinking = if state.disable_thinking { - false - } else { - let from_body = req.thinking.as_ref().map(|t| t.thinking_type.as_str()); - let explicit = from_body.is_some(); - let base = match from_body { - Some("enabled") => true, - Some("disabled") => false, - _ => state.behavior.thinking_default, - }; - // MODEL.toml `thinking_in_tools=false` is the DEFAULT for tool-active - // turns — an explicit `thinking.type="enabled"` from the client still - // opts in. See api.rs for rationale. - if tools_active && !state.behavior.thinking_in_tools && !explicit { - false - } else { - base - } - }; - - struct CountMsgEntry { - role: String, - content: String, - tool_calls: Option>, - } - - let mut messages: Vec = Vec::with_capacity(req.messages.len() + 1); - if let Some(ref sys) = req.system { - let sys_text = match sys { - SystemContent::Blocks(blocks) => blocks - .iter() - .filter(|b| { - b.block_type == "text" - && !b.text.as_deref().unwrap_or("").starts_with("x-anthropic-") - }) - .filter_map(|b| b.text.clone()) - .collect::>() - .join("\n"), - SystemContent::Text(s) => s.clone(), - }; - messages.push(CountMsgEntry { - role: "system".into(), - content: sys_text, - tool_calls: None, - }); + // Count against the EXACT prompt the serving path renders: same + // adapter (into_ir), same tool-prompt injection / hint / cwd / + // thinking resolution / Jinja variant (prepare_chat_prompt). The + // old path was a third, divergent lowering — it dropped images and + // thinking blocks, force-prepended an empty `` wrapper, and + // rendered through the non-openai Jinja variant, so counts drifted + // from real usage. + let mut ir_req = req.into_ir(); + // Counting must not require the vision encoder: strip image parts + // (they contributed 0 tokens in the old count too — pad expansion + // needs real pixel grids, which a count endpoint can't produce). + for m in &mut ir_req.messages { + m.content + .retain(|p| !matches!(p, crate::ir::ContentPart::Image(_))); } - for m in &req.messages { - let role = match m.role.as_str() { - "user" => "user", - "assistant" => "assistant", - _ => "user", - }; - let (mut text, incoming_tool_calls) = flatten_content(&m.content); - - // Mirror the same transformations as the real /v1/messages path - // so the token count matches what actually gets sent to the model. - if role == "assistant" && state.tokenizer.supports_thinking() { - text = format!("\n\n\n\n{text}"); - } - let tool_calls_json = - if tools_active && role == "assistant" && !incoming_tool_calls.is_empty() { - let parsed: Vec = incoming_tool_calls - .iter() - .map(|tc| { - let args: serde_json::Value = serde_json::from_str(&tc.function.arguments) - .unwrap_or(serde_json::Value::Object(Default::default())); - serde_json::json!({ - "id": tc.id.as_deref().unwrap_or(""), - "type": "function", - "function": { "name": tc.function.name, "arguments": args } - }) - }) - .collect(); - Some(parsed) - } else { - None - }; - if tools_active - && role == "user" - && let AnthropicContent::Blocks(blocks) = &m.content - { - let has_tool_result = blocks - .iter() - .any(|b| matches!(b, ContentBlock::ToolResult { .. })); - if has_tool_result { - for block in blocks { - match block { - ContentBlock::ToolResult { content, .. } => { - let result_text = - content.as_ref().map(|c| c.to_text()).unwrap_or_default(); - messages.push(CountMsgEntry { - role: "tool".into(), - content: result_text, - tool_calls: None, - }); - } - ContentBlock::Text { text: t } if !t.is_empty() => { - messages.push(CountMsgEntry { - role: "user".into(), - content: t.clone(), - tool_calls: None, - }); - } - _ => {} - } - } - continue; - } - } - - messages.push(CountMsgEntry { - role: role.to_string(), - content: text, - tool_calls: tool_calls_json, - }); - } - - // Build JSON messages for token counting (mirrors real /v1/messages path). - let json_messages: Vec = messages - .iter() - .map(|m| { - let mut msg = serde_json::json!({"role": m.role, "content": m.content}); - if let Some(ref tcs) = m.tool_calls { - msg["tool_calls"] = serde_json::Value::Array(tcs.clone()); - } - msg - }) - .collect(); - let jinja_tools: Option> = if tools_active { - req.tools.as_ref().map(|ts| { - let oai = convert_tools(ts); - oai.iter().map(|t| serde_json::json!({ - "type": "function", - "function": { "name": t.function.name, "description": t.function.description, "parameters": t.function.parameters } - })).collect() - }) - } else { - None - }; - - let input_tokens = match state.tokenizer.apply_chat_template_jinja( - &json_messages, - jinja_tools.as_deref(), - enable_thinking, - state.behavior.disable_tool_steering, - ) { - Ok(t) => t.len(), - Err(_) => 0, + let prepared = match crate::api::chat::prepare::prepare_chat_prompt(&state, &mut ir_req) { + Ok(p) => p, + Err(resp) => return openai_error_to_anthropic(resp).await, }; let body = serde_json::json!({ - "input_tokens": input_tokens + "input_tokens": prepared.prompt_tokens.len() }); Json(body).into_response() } + +/// Re-shape an OpenAI-envelope error `Response` (produced by the shared +/// pipeline) into Anthropic's error body, preserving the status code. +async fn openai_error_to_anthropic(resp: Response) -> Response { + let (parts, body) = resp.into_parts(); + let message = match axum::body::to_bytes(body, usize::MAX).await { + Ok(bytes) => serde_json::from_slice::(&bytes) + .ok() + .and_then(|v| { + v.get("error") + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + .map(|s| s.to_string()) + }) + .unwrap_or_else(|| String::from_utf8_lossy(&bytes).into_owned()), + Err(e) => format!("Error body collect: {e}"), + }; + let error_type = if parts.status.is_client_error() { + "invalid_request_error" + } else { + "api_error" + }; + anthropic_error(parts.status, error_type, message) +} diff --git a/crates/spark-server/src/anthropic/handlers_stream.rs b/crates/spark-server/src/anthropic/handlers_stream.rs index 8da711c53..cc3a28771 100644 --- a/crates/spark-server/src/anthropic/handlers_stream.rs +++ b/crates/spark-server/src/anthropic/handlers_stream.rs @@ -5,102 +5,49 @@ use axum::response::{IntoResponse, Response, Sse}; use futures::StreamExt; use super::translator::*; -/// Translate an OpenAI SSE response into Anthropic's structured event -/// stream, **per-chunk**: each OpenAI `data: {…}` line that arrives -/// produces zero or more Anthropic events emitted to the client -/// before the next OpenAI chunk is processed. -/// -/// Implementation: spawn a tokio task that consumes the inner body's -/// `Bytes` stream, splits on `\n\n` event boundaries, parses the -/// `data:` payload as JSON, and feeds it through -/// `AnthropicTranslator::process_openai_chunk`. The translator's -/// emitted events are sent down an `mpsc` channel that's wrapped as -/// a `ReceiverStream` and handed to axum's `Sse::new` — the same -/// pattern api.rs uses for its own SSE response. -pub(super) async fn wrap_chat_sse_for_anthropic( - chat_resp: Response, + +/// Encode the pipeline's neutral delta stream as Anthropic's structured +/// SSE event stream, **per-delta**: each `ir::StreamDelta` produces zero +/// or more Anthropic events emitted to the client before the next delta +/// is processed. (This replaces the old wrapper that re-parsed the +/// serialized OpenAI SSE bytes chunk-by-chunk — one fewer serialize/ +/// parse round-trip and one fewer protocol to drift against.) +pub(super) fn anthropic_sse_from_deltas( + deltas: crate::ir::DeltaStream, req_model: String, ) -> Response { - let (parts, body) = chat_resp.into_parts(); - if !parts.status.is_success() { - // Forward error envelope verbatim, status preserved. - return Response::from_parts(parts, body); - } - - // Match the OpenAI chat_stream sizing — see chat_stream/mod.rs for rationale. + // Match the chat_stream sizing — see chat_stream/mod.rs for rationale. let (tx, rx) = tokio::sync::mpsc::channel::>(1024); tokio::spawn(async move { let mut translator = AnthropicTranslator::new(req_model); - let mut buf = String::new(); - let mut data_stream = body.into_data_stream(); - let mut pending: Vec = Vec::new(); + let mut pending: Vec = Vec::new(); + let mut deltas = deltas; - // Inner helper: drain `pending` into the channel. Returns - // `false` when the receiver has hung up — caller should - // abort. + // Drain `pending` into the channel, converting each typed + // `SseEvent` to an axum wire event. Returns `false` when the + // receiver has hung up — caller should abort. async fn flush( tx: &tokio::sync::mpsc::Sender>, - pending: &mut Vec, + pending: &mut Vec, ) -> bool { for ev in pending.drain(..) { - if tx.send(Ok(ev)).await.is_err() { + if tx.send(Ok(to_axum_event(&ev))).await.is_err() { return false; } } true } - while let Some(chunk_res) = data_stream.next().await { - let chunk = match chunk_res { - Ok(b) => b, - Err(e) => { - tracing::warn!("anthropic stream: inner body error: {e}"); - break; - } - }; - buf.push_str(&String::from_utf8_lossy(&chunk)); - - // Process any complete `data: {…}\n\n` records. - while let Some(boundary) = buf.find("\n\n") { - let raw_record: String = buf.drain(..boundary + 2).collect(); - for line in raw_record.lines() { - let payload = match line.strip_prefix("data:") { - Some(s) => s.trim(), - None => continue, - }; - if payload.is_empty() || payload == "[DONE]" { - continue; - } - let val: serde_json::Value = match serde_json::from_str(payload) { - Ok(v) => v, - Err(_) => continue, - }; - translator.process_openai_chunk(&val, &mut pending); - } - if !flush(&tx, &mut pending).await { - return; - } + while let Some(delta) = deltas.next().await { + translator.on_delta(&delta, &mut pending); + if !flush(&tx, &mut pending).await { + return; } } - // Final flush: drain any tail bytes (no trailing `\n\n`) and - // ensure message_stop fires even if upstream omitted - // finish_reason. - if !buf.is_empty() { - for line in buf.lines() { - let payload = match line.strip_prefix("data:") { - Some(s) => s.trim(), - None => continue, - }; - if payload.is_empty() || payload == "[DONE]" { - continue; - } - if let Ok(val) = serde_json::from_str::(payload) { - translator.process_openai_chunk(&val, &mut pending); - } - } - } + // Ensure message_stop fires even if the stream ended without a + // Finish delta. translator.finalize(&mut pending); if !flush(&tx, &mut pending).await { tracing::warn!("anthropic stream: final flush failed (receiver dropped)"); diff --git a/crates/spark-server/src/anthropic/helpers.rs b/crates/spark-server/src/anthropic/helpers.rs index c7c1202a1..006dae90a 100644 --- a/crates/spark-server/src/anthropic/helpers.rs +++ b/crates/spark-server/src/anthropic/helpers.rs @@ -54,12 +54,16 @@ pub(super) fn convert_tool_choice(tc: &AnthropicToolChoice) -> tool_parser::Tool } } -/// Convert Anthropic finish reason to stop_reason string. +/// Convert an OpenAI finish reason to Anthropic's stop_reason string. pub(super) fn convert_stop_reason(finish_reason: &str) -> &'static str { match finish_reason { "stop" => "end_turn", "tool_calls" => "tool_use", "length" => "max_tokens", + // Safety-filtered output maps to Anthropic's dedicated refusal + // stop reason (2025-05 API), not a normal end_turn — clients + // branch on this to avoid retrying verbatim. + "content_filter" => "refusal", _ => "end_turn", } } diff --git a/crates/spark-server/src/anthropic/mod.rs b/crates/spark-server/src/anthropic/mod.rs index 6424ebbd0..91eefc19a 100644 --- a/crates/spark-server/src/anthropic/mod.rs +++ b/crates/spark-server/src/anthropic/mod.rs @@ -6,10 +6,10 @@ //! reuses the existing scheduler pipeline, and converts the response back to //! Anthropic format. Supports both streaming (SSE) and non-streaming. -mod convert; mod handlers; mod handlers_stream; mod helpers; +mod to_ir; mod translate; mod translator; mod types; diff --git a/crates/spark-server/src/anthropic/tests/ir_carry.rs b/crates/spark-server/src/anthropic/tests/ir_carry.rs new file mode 100644 index 000000000..98619acb8 --- /dev/null +++ b/crates/spark-server/src/anthropic/tests/ir_carry.rs @@ -0,0 +1,413 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Anthropic request adapter: the wire request lowers DIRECTLY into the +// chat IR (no OpenAI-wire-JSON hop), carrying images, reasoning, tool +// calls, and the tool_result error flag as typed structure (issue +// #165 + IR migration). + +use super::super::translate::ir_to_anthropic_response; +use super::super::types::MessagesRequest; +use crate::ir::message::ImageSource; +use crate::ir::{ChatRequest, ContentPart, ImageData, Role, ThinkingDirective}; + +/// Lower an Anthropic request body into the IR envelope (the same path +/// `handlers::messages` takes). +fn lower(req_json: serde_json::Value) -> ChatRequest { + let req: MessagesRequest = serde_json::from_value(req_json).expect("MessagesRequest"); + req.into_ir() +} + +fn text_of(parts: &[ContentPart]) -> String { + parts + .iter() + .filter_map(|p| match p { + ContentPart::Text(t) => Some(t.as_str()), + _ => None, + }) + .collect() +} + +fn images_of(parts: &[ContentPart]) -> Vec<&ImageData> { + parts + .iter() + .filter_map(|p| match p { + ContentPart::Image(ImageSource { data }) => Some(data), + _ => None, + }) + .collect() +} + +#[test] +fn user_image_block_is_carried() { + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": [ + {"type": "text", "text": "what is this?"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA"}} + ]}] + })); + let user = ir + .messages + .iter() + .find(|m| m.role == Role::User) + .expect("user msg"); + assert_eq!( + images_of(&user.content), + vec![&ImageData::Base64("data:image/png;base64,AAA".into())] + ); + assert_eq!(text_of(&user.content), "what is this?"); + // Canonical part order: images first, then the joined text. + assert!(matches!(user.content[0], ContentPart::Image(_))); +} + +#[test] +fn tool_result_image_and_error_flag_are_carried() { + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "is_error": true, "content": [ + {"type": "text", "text": "Exit code 127"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": "BBB"}} + ]} + ]}] + })); + let tool = ir + .messages + .iter() + .find(|m| m.role == Role::Tool) + .expect("tool msg"); + assert_eq!(tool.tool_call_id.as_deref(), Some("t1")); + assert_eq!( + images_of(&tool.content), + vec![&ImageData::Base64("data:image/jpeg;base64,BBB".into())] + ); + // The error flag is structural now; the `[tool error]\n` marker is + // rendered by msg_entry, not baked into the text. + assert!(tool.tool_error); + assert_eq!(text_of(&tool.content), "Exit code 127"); +} + +#[test] +fn thinking_block_becomes_first_class_reasoning() { + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "let me reason"}, + {"type": "text", "text": "the answer is 4"} + ]}] + })); + let asst = ir + .messages + .iter() + .find(|m| m.role == Role::Assistant) + .expect("assistant msg"); + assert_eq!( + asst.reasoning.as_ref().map(|r| r.text.as_str()), + Some("let me reason") + ); + assert_eq!(text_of(&asst.content), "the answer is 4"); +} + +#[test] +fn url_image_source_becomes_url_variant() { + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": [ + {"type": "image", "source": {"type": "url", "url": "https://example.com/a.png"}} + ]}] + })); + let user = ir + .messages + .iter() + .find(|m| m.role == Role::User) + .expect("user msg"); + // Typed as Url — the pipeline rejects it with an explicit 400 + // instead of feeding a URL to the base64 decoder. + assert_eq!( + images_of(&user.content), + vec![&ImageData::Url("https://example.com/a.png".into())] + ); +} + +#[test] +fn tool_use_and_envelope_fields_are_carried() { + let ir = lower(serde_json::json!({ + "model": "claude-ish", "max_tokens": 99, "stream": true, + "temperature": 0.3, "top_k": 5, "top_p": 0.9, + "stop_sequences": ["STOP"], + "system": [{"type": "text", "text": "sys"}, {"type": "text", "text": "x-anthropic-billing"}], + "tools": [{"name": "get_weather", "description": "w", "input_schema": {"type": "object"}}], + "tool_choice": {"type": "any"}, + "thinking": {"type": "enabled", "budget_tokens": 512}, + "messages": [ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "c1", "name": "get_weather", "input": {"city": "SF"}} + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "c1", "content": "sunny"} + ]} + ] + })); + assert_eq!(ir.model, "claude-ish"); + assert_eq!(ir.max_tokens, 99); + assert!(ir.stream); + assert_eq!(ir.sampling.temperature, Some(0.3)); + assert_eq!(ir.sampling.top_k, Some(5)); + assert_eq!(ir.sampling.top_p, Some(0.9)); + assert_eq!(ir.stop, vec!["STOP".to_string()]); + assert_eq!(ir.n, 1); + assert_eq!(ir.thinking, ThinkingDirective::On { budget: Some(512) }); + + // System: billing block filtered, text kept. + assert_eq!(ir.messages[0].role, Role::System); + assert_eq!(text_of(&ir.messages[0].content), "sys"); + + // tool_use → structured tool call with parsed arguments. + let asst = &ir.messages[1]; + assert_eq!(asst.role, Role::Assistant); + assert_eq!(asst.tool_calls.len(), 1); + assert_eq!(asst.tool_calls[0].id, "c1"); + assert_eq!(asst.tool_calls[0].name, "get_weather"); + assert_eq!( + asst.tool_calls[0].arguments, + serde_json::json!({"city": "SF"}) + ); + + // tool_result → Tool message linked by id, no error flag. + let tool = &ir.messages[2]; + assert_eq!(tool.role, Role::Tool); + assert_eq!(tool.tool_call_id.as_deref(), Some("c1")); + assert!(!tool.tool_error); + assert_eq!(text_of(&tool.content), "sunny"); + + // Tools + tool_choice converted to the neutral definitions. + assert_eq!(ir.tools.len(), 1); + assert_eq!(ir.tools[0].function.name, "get_weather"); + match ir.tool_choice { + Some(crate::tool_parser::ToolChoice::Mode(ref m)) => assert_eq!(m, "required"), + ref other => panic!("expected Mode(required), got {other:?}"), + } +} + +#[test] +fn thinking_disabled_and_unspecified_directives() { + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "thinking": {"type": "disabled", "budget_tokens": 100}, + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(ir.thinking, ThinkingDirective::Off); + + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "thinking": {"type": "adaptive"}, + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(ir.thinking, ThinkingDirective::On { budget: None }); + + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}] + })); + assert_eq!(ir.thinking, ThinkingDirective::Unspecified); +} + +#[test] +fn rendered_prompt_json_matches_retired_json_hop_output() { + // GOLDEN captured from the retired anthropic_to_chat_request_json + // path (pre-deletion): the same fixture, run through the old + // wire→OpenAI-JSON→IncomingMessage→IR lowering, produced exactly + // this build_json_messages output. The direct adapter must render + // identical prompt bytes (kv-cache prefix stability across the + // migration). + let ir = lower(serde_json::json!({ + "model": "m", "max_tokens": 32, + "system": "Be helpful.", + "messages": [ + {"role": "user", "content": "check the weather"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "need the tool"}, + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "c1", "name": "get_weather", "input": {"city": "SF"}} + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "c1", "is_error": true, "content": "network down"} + ]}, + {"role": "user", "content": "try again"} + ] + })); + let out = crate::api::chat::test_build_msg_entries(&ir.messages, true).expect("builds"); + let json = crate::api::chat::test_build_json_messages(&out); + let expected = serde_json::json!([ + {"role": "system", "content": "Be helpful."}, + {"role": "user", "content": "check the weather"}, + { + "role": "assistant", + "content": "Checking.", + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "get_weather", "arguments": {"city": "SF"}}} + ], + "reasoning_content": "need the tool" + }, + {"role": "tool", "content": "[tool error]\nnetwork down"}, + {"role": "user", "content": "try again"} + ]); + assert_eq!(serde_json::Value::Array(json), expected); +} + +#[test] +fn wire_surfaces_lower_to_identical_ir_core() { + // The same conversation expressed in the Anthropic and OpenAI chat + // wire dialects must lower to the SAME IR messages/thinking/stop — + // the whole point of the narrow waist. (The Responses surface + // lowers through the chat wire, so chat parity covers it.) + let anth = lower(serde_json::json!({ + "model": "m", "max_tokens": 64, + "system": "Be helpful.", + "temperature": 0.5, "top_p": 0.9, "top_k": 4, + "stop_sequences": ["END"], + "thinking": {"type": "enabled", "budget_tokens": 512}, + "tools": [{"name": "get_weather", "description": "w", + "input_schema": {"type": "object"}}], + "messages": [ + {"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAA"}}, + {"type": "text", "text": "what is this?"} + ]}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "hmm"}, + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "c1", "name": "get_weather", "input": {"city": "SF"}} + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "c1", "content": "sunny"} + ]}, + ] + })); + + let chat: crate::openai::ChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "m", "max_tokens": 64, + "temperature": 0.5, "top_p": 0.9, "top_k": 4, + "stop": ["END"], + "thinking": {"type": "enabled", "budget_tokens": 512}, + "tools": [{"type": "function", "function": {"name": "get_weather", "description": "w", + "parameters": {"type": "object"}}}], + "messages": [ + {"role": "system", "content": "Be helpful."}, + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA"}}, + {"type": "text", "text": "what is this?"} + ]}, + {"role": "assistant", "content": "Checking.", "reasoning_content": "hmm", + "tool_calls": [{"id": "c1", "type": "function", + "function": {"name": "get_weather", "arguments": "{\"city\":\"SF\"}"}}]}, + {"role": "tool", "tool_call_id": "c1", "content": "sunny"}, + ] + })) + .expect("chat wire parses"); + let chat_ir = chat.into_ir(); + + assert_eq!(anth.messages, chat_ir.messages); + assert_eq!(anth.thinking, chat_ir.thinking); + assert_eq!(anth.stop, chat_ir.stop); + assert_eq!(anth.max_tokens, chat_ir.max_tokens); + assert_eq!(anth.sampling.temperature, chat_ir.sampling.temperature); + assert_eq!(anth.sampling.top_p, chat_ir.sampling.top_p); + assert_eq!(anth.sampling.top_k, chat_ir.sampling.top_k); + assert_eq!(anth.tools.len(), chat_ir.tools.len()); + assert_eq!(anth.tools[0].function.name, chat_ir.tools[0].function.name); + assert_eq!( + anth.tools[0].function.parameters, + chat_ir.tools[0].function.parameters + ); +} + +// ── response direction (ir_to_anthropic_response) ── + +fn ir_response(choice: crate::ir::Choice) -> crate::ir::ChatResponse { + crate::ir::ChatResponse { + id: "abc".into(), + model: "m".into(), + created: 1, + choices: vec![choice], + usage: crate::ir::Usage { + prompt_tokens: 10, + completion_tokens: 5, + cached_prompt_tokens: 3, + reasoning_tokens: 2, + time_to_first_token_ms: 0.0, + response_tokens_per_second: 0.0, + }, + } +} + +fn choice() -> crate::ir::Choice { + crate::ir::Choice { + index: 0, + content: Some("hello".into()), + reasoning: None, + tool_calls: Vec::new(), + refusal: None, + finish_reason: crate::ir::FinishReason::Stop, + matched_stop: None, + logprobs: None, + } +} + +#[test] +fn response_maps_reasoning_text_tools_and_usage() { + let mut c = choice(); + c.reasoning = Some("thinking".into()); + c.tool_calls = vec![crate::ir::message::ToolCall { + id: "c1".into(), + name: "f".into(), + arguments: serde_json::json!({"x": 1}), + }]; + c.finish_reason = crate::ir::FinishReason::ToolCalls; + let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + assert_eq!(json["id"], "msg_abc"); + assert_eq!(json["model"], "m"); + assert_eq!(json["stop_reason"], "tool_use"); + assert_eq!(json["usage"]["input_tokens"], 10); + assert_eq!(json["usage"]["output_tokens"], 5); + // B5: prefix-cache hits surface as Anthropic cache accounting. + assert_eq!(json["usage"]["cache_read_input_tokens"], 3); + assert_eq!(json["content"][0]["type"], "thinking"); + assert_eq!(json["content"][0]["thinking"], "thinking"); + assert_eq!(json["content"][1]["type"], "text"); + assert_eq!(json["content"][1]["text"], "hello"); + assert_eq!(json["content"][2]["type"], "tool_use"); + assert_eq!(json["content"][2]["name"], "f"); + assert_eq!(json["content"][2]["input"], serde_json::json!({"x": 1})); +} + +#[test] +fn response_stop_reason_mapping() { + // content_filter → refusal (Anthropic's dedicated stop reason). + let mut c = choice(); + c.finish_reason = crate::ir::FinishReason::ContentFilter; + let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + assert_eq!(json["stop_reason"], "refusal"); + + // length → max_tokens. + let mut c = choice(); + c.finish_reason = crate::ir::FinishReason::Length; + let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + assert_eq!(json["stop_reason"], "max_tokens"); + + // A client stop sequence gets stop_sequence + the matched echo. + let mut c = choice(); + c.matched_stop = Some("END".into()); + let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + assert_eq!(json["stop_reason"], "stop_sequence"); + assert_eq!(json["stop_sequence"], "END"); +} + +#[test] +fn response_empty_content_omits_text_block() { + let mut c = choice(); + c.content = Some(String::new()); + let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + assert_eq!(json["content"].as_array().unwrap().len(), 0); + assert_eq!(json["stop_reason"], "end_turn"); +} diff --git a/crates/spark-server/src/anthropic/tests/mod.rs b/crates/spark-server/src/anthropic/tests/mod.rs index a27e5f835..581785280 100644 --- a/crates/spark-server/src/anthropic/tests/mod.rs +++ b/crates/spark-server/src/anthropic/tests/mod.rs @@ -11,3 +11,6 @@ // mod translator_a; // mod translator_b; // mod types_convert; + +mod ir_carry; +mod translator_stream; diff --git a/crates/spark-server/src/anthropic/tests/translator_stream.rs b/crates/spark-server/src/anthropic/tests/translator_stream.rs new file mode 100644 index 000000000..1b45a44c7 --- /dev/null +++ b/crates/spark-server/src/anthropic/tests/translator_stream.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Golden tests for the Anthropic streaming translator's event framing. +// These characterize the wire output Claude Code depends on; the +// translator now consumes neutral `ir::StreamDelta`s instead of +// re-parsed OpenAI chunk JSON, but the emitted event sequences are +// unchanged. + +use super::super::translator::{AnthropicTranslator, SseEvent}; +use crate::ir::response::{FinishReason, Usage}; +use crate::ir::stream::StreamDelta; + +fn usage(prompt: usize, completion: usize) -> Usage { + Usage { + prompt_tokens: prompt, + completion_tokens: completion, + cached_prompt_tokens: 0, + reasoning_tokens: 0, + time_to_first_token_ms: 0.0, + response_tokens_per_second: 0.0, + } +} + +fn drive(deltas: Vec) -> Vec { + let mut t = AnthropicTranslator::new("m".to_string()); + let mut out = Vec::new(); + for d in &deltas { + t.on_delta(d, &mut out); + } + t.finalize(&mut out); + out +} + +fn names(evs: &[SseEvent]) -> Vec<&str> { + evs.iter().map(|e| e.event.as_str()).collect() +} + +#[test] +fn text_stream_framing() { + let evs = drive(vec![ + StreamDelta::Content { + text: "Hi".into(), + token_ids: Vec::new(), + }, + StreamDelta::Finish { + reason: FinishReason::Stop, + usage: usage(3, 1), + token_ids: Vec::new(), + }, + ]); + assert_eq!( + names(&evs), + vec![ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(evs[1].data["content_block"]["type"], "text"); + assert_eq!(evs[2].data["delta"]["type"], "text_delta"); + assert_eq!(evs[2].data["delta"]["text"], "Hi"); + assert_eq!(evs[4].data["delta"]["stop_reason"], "end_turn"); + assert_eq!(evs[4].data["usage"]["output_tokens"], 1); + // B1: the final message_delta patches input_tokens (usage arrives on + // the terminal delta, after message_start already reported 0). + assert_eq!(evs[4].data["usage"]["input_tokens"], 3); + assert_eq!(evs[4].data["usage"]["cache_read_input_tokens"], 0); + // message_start still opens with zero (usage unknown at that point) + // and a minted msg_ id. + assert_eq!(evs[0].data["message"]["usage"]["input_tokens"], 0); + let id = evs[0].data["message"]["id"].as_str().unwrap(); + assert!(id.starts_with("msg_"), "unexpected id shape: {id}"); +} + +#[test] +fn thinking_then_text_framing() { + let evs = drive(vec![ + StreamDelta::Reasoning { + text: "think".into(), + token_ids: Vec::new(), + }, + StreamDelta::Content { + text: "answer".into(), + token_ids: Vec::new(), + }, + StreamDelta::Finish { + reason: FinishReason::Stop, + usage: usage(0, 0), + token_ids: Vec::new(), + }, + ]); + assert_eq!( + names(&evs), + vec![ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(evs[1].data["content_block"]["type"], "thinking"); + assert_eq!(evs[2].data["delta"]["type"], "thinking_delta"); + assert_eq!(evs[2].data["delta"]["thinking"], "think"); + assert_eq!(evs[5].data["delta"]["type"], "text_delta"); + assert_eq!(evs[5].data["delta"]["text"], "answer"); +} + +#[test] +fn tool_call_stream_framing() { + let evs = drive(vec![ + StreamDelta::ToolCallStart { + index: 0, + id: "call_1".into(), + name: "get_weather".into(), + }, + StreamDelta::ToolCallArgs { + index: 0, + fragment: "{\"city\":\"SF\"}".into(), + token_ids: Vec::new(), + }, + StreamDelta::Finish { + reason: FinishReason::ToolCalls, + usage: usage(0, 0), + token_ids: Vec::new(), + }, + ]); + assert_eq!( + names(&evs), + vec![ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(evs[1].data["content_block"]["type"], "tool_use"); + assert_eq!(evs[1].data["content_block"]["name"], "get_weather"); + assert_eq!(evs[1].data["content_block"]["id"], "call_1"); + assert_eq!(evs[2].data["delta"]["type"], "input_json_delta"); + assert_eq!(evs[2].data["delta"]["partial_json"], "{\"city\":\"SF\"}"); + assert_eq!(evs[4].data["delta"]["stop_reason"], "tool_use"); +} + +#[test] +fn multi_tool_calls_close_and_reopen_blocks() { + // Two tool calls in one turn: the second start must close the first + // block and open a fresh one at the next index (Claude Code executes + // each block once). + let evs = drive(vec![ + StreamDelta::ToolCallStart { + index: 0, + id: "c1".into(), + name: "read".into(), + }, + StreamDelta::ToolCallArgs { + index: 0, + fragment: "{}".into(), + token_ids: Vec::new(), + }, + StreamDelta::ToolCallStart { + index: 1, + id: "c2".into(), + name: "bash".into(), + }, + StreamDelta::ToolCallArgs { + index: 1, + fragment: "{\"command\":\"ls\"}".into(), + token_ids: Vec::new(), + }, + StreamDelta::Finish { + reason: FinishReason::ToolCalls, + usage: usage(0, 2), + token_ids: Vec::new(), + }, + ]); + assert_eq!( + names(&evs), + vec![ + "message_start", + "content_block_start", // tool 0 + "content_block_delta", + "content_block_stop", // tool 0 closed by tool 1 start + "content_block_start", // tool 1 + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(evs[1].data["content_block"]["id"], "c1"); + assert_eq!(evs[1].data["index"], 0); + assert_eq!(evs[4].data["content_block"]["id"], "c2"); + assert_eq!(evs[4].data["index"], 1); +} + +#[test] +fn finalize_without_finish_emits_end_turn() { + // Upstream died before the Finish delta: finalize must still close + // the block and end the message coherently. + let mut t = AnthropicTranslator::new("m".to_string()); + let mut out = Vec::new(); + t.on_delta( + &StreamDelta::Content { + text: "partial".into(), + token_ids: Vec::new(), + }, + &mut out, + ); + t.finalize(&mut out); + assert_eq!( + names(&out), + vec![ + "message_start", + "content_block_start", + "content_block_delta", + "content_block_stop", + "message_delta", + "message_stop", + ] + ); + assert_eq!(out[4].data["delta"]["stop_reason"], "end_turn"); +} diff --git a/crates/spark-server/src/anthropic/to_ir.rs b/crates/spark-server/src/anthropic/to_ir.rs new file mode 100644 index 000000000..9d63c513d --- /dev/null +++ b/crates/spark-server/src/anthropic/to_ir.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Adapter: Anthropic Messages API wire request → canonical chat IR. +// Replaces the historical hand-built OpenAI-wire-JSON hop (the request +// used to be rewritten into chat-completions JSON, re-parsed as an +// OpenAI request, and only then lowered) — the Anthropic surface is +// now a peer adapter of the OpenAI edge, and structural information +// (tool_result `is_error`, image source kinds) survives as typed IR +// instead of text conventions. + +use crate::ir; +use crate::ir::message::{ImageSource, Reasoning, ToolCall}; +use crate::ir::{ContentPart, ImageData, Message, Role, ThinkingDirective}; + +use super::helpers::{convert_tool_choice, convert_tools}; +use super::types::{ + AnthropicContent, ContentBlock, MessagesRequest, SystemContent, ToolResultContent, +}; + +impl MessagesRequest { + /// Lower the parsed Anthropic wire request into the + /// provider-agnostic [`ir::ChatRequest`] envelope. + /// + /// Message mapping (mirrors the retired JSON-hop translation + /// byte-for-byte at the rendered-prompt level): + /// * `system` (string or `text` blocks, `x-anthropic-*` billing + /// blocks filtered) → one leading System message when non-empty. + /// * assistant blocks → ONE assistant message: `[image*, text]` + /// parts, `tool_use` → structured tool_calls (arguments stay + /// parsed JSON), `thinking` blocks → first-class reasoning + /// (joined with `\n`). + /// * user blocks → an optional user message (`[image*, text]`, + /// only when text/images exist) followed by one Tool message per + /// `tool_result` in block order. `is_error` travels as + /// `Message::tool_error` — the `[tool error]\n` marker is + /// rendered by the shared pipeline (`msg_entry`), not baked into + /// the text here. + /// * unknown roles collapse to user (Anthropic wire only defines + /// user/assistant). + pub fn into_ir(self) -> ir::ChatRequest { + let mut messages: Vec = Vec::with_capacity(self.messages.len() + 1); + + // System message (filter x-anthropic- billing/config blocks). + if let Some(sys) = &self.system { + let sys_text = match sys { + SystemContent::Text(s) => s.clone(), + SystemContent::Blocks(blocks) => blocks + .iter() + .filter(|b| { + b.block_type == "text" + && !b.text.as_deref().unwrap_or("").starts_with("x-anthropic-") + }) + .filter_map(|b| b.text.clone()) + .collect::>() + .join("\n"), + }; + if !sys_text.is_empty() { + messages.push(Message::synthetic_system(sys_text)); + } + } + + // Conversation history. + for m in self.messages { + let role = match m.role.as_str() { + "assistant" => Role::Assistant, + _ => Role::User, + }; + match m.content { + AnthropicContent::Text(s) => { + let content = if s.is_empty() { + Vec::new() + } else { + vec![ContentPart::Text(s)] + }; + messages.push(Message { + role, + content, + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + }); + } + AnthropicContent::Blocks(blocks) => { + let mut text_parts: Vec = Vec::new(); + let mut images: Vec = Vec::new(); + let mut reasoning_parts: Vec = Vec::new(); + let mut tool_calls: Vec = Vec::new(); + // (tool_use_id, text, images, is_error) + let mut tool_results: Vec<(String, String, Vec, bool)> = Vec::new(); + for b in blocks { + match b { + ContentBlock::Text { text } => text_parts.push(text), + ContentBlock::Image { source } => { + if let Some(uri) = source.to_image_uri() { + images.push(uri); + } + } + ContentBlock::ToolUse { id, name, input } => { + // `input` is already structured JSON — no + // stringify/re-parse roundtrip. + tool_calls.push(ToolCall { + id, + name, + arguments: input, + }); + } + ContentBlock::ToolResult { + tool_use_id, + content, + is_error, + } => { + let text = + content.as_ref().map(|c| c.to_text()).unwrap_or_default(); + // Carry images embedded in a tool result + // (e.g. a screenshot the tool returned) so + // they reach the vision encoder (issue #165). + let tr_images: Vec = match content { + Some(ToolResultContent::Blocks(inner)) => inner + .iter() + .filter_map(|ib| match ib { + ContentBlock::Image { source } => source.to_image_uri(), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + tool_results.push(( + tool_use_id, + text, + tr_images, + is_error.unwrap_or(false), + )); + } + ContentBlock::Thinking { thinking } => { + if let Some(t) = thinking + && !t.is_empty() + { + reasoning_parts.push(t); + } + } + ContentBlock::Unknown => {} + } + } + let text_content = text_parts.join(""); + if role == Role::Assistant { + messages.push(Message { + role: Role::Assistant, + content: parts_from(&images, &text_content), + tool_calls, + tool_call_id: None, + name: None, + reasoning: if reasoning_parts.is_empty() { + None + } else { + Some(Reasoning { + text: reasoning_parts.join("\n"), + }) + }, + tool_error: false, + }); + } else { + if !text_content.is_empty() || !images.is_empty() { + messages.push(Message { + role: Role::User, + content: parts_from(&images, &text_content), + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + }); + } + for (tool_use_id, text, tr_images, is_error) in tool_results { + messages.push(Message { + role: Role::Tool, + content: parts_from(&tr_images, &text), + tool_calls: Vec::new(), + tool_call_id: Some(tool_use_id), + name: None, + reasoning: None, + tool_error: is_error, + }); + } + } + } + } + } + + // Anthropic thinking config → neutral directive. Same rungs the + // OpenAI ladder applies to its `thinking` channel: disabled wins, + // then an explicit budget, then budget-less enabled/adaptive + // (think as long as needed — defer to the per-model cap). + let thinking = match &self.thinking { + None => ThinkingDirective::Unspecified, + Some(tc) if tc.thinking_type == "disabled" => ThinkingDirective::Off, + Some(tc) => match tc.budget_tokens { + Some(b) => ThinkingDirective::On { + budget: Some(u32::try_from(b).unwrap_or(u32::MAX)), + }, + None => ThinkingDirective::On { budget: None }, + }, + }; + + ir::ChatRequest { + model: self.model, + messages, + tools: self.tools.as_deref().map(convert_tools).unwrap_or_default(), + tool_choice: self.tool_choice.as_ref().map(convert_tool_choice), + sampling: ir::SamplingParams { + temperature: self.temperature, + top_k: self.top_k, + top_p: self.top_p, + ..Default::default() + }, + max_tokens: self.max_tokens, + min_tokens: 0, + stop: self.stop_sequences, + stream: self.stream, + n: 1, + response_format: None, + thinking, + repetition_detection: None, + logit_bias: Vec::new(), + top_logprobs: None, + seed: None, + timeout_secs: None, + return_token_ids: false, + } + } +} + +/// `[image*, text]` content parts — image order preserved, single +/// joined text part last, omitted when empty. Matches the OpenAI +/// adapter's shape so the template sees one canonical layout. +fn parts_from(images: &[String], text: &str) -> Vec { + let mut content: Vec = Vec::with_capacity(images.len() + 1); + for uri in images { + content.push(ContentPart::Image(ImageSource { + data: ImageData::from_uri(uri.clone()), + })); + } + if !text.is_empty() { + content.push(ContentPart::Text(text.to_string())); + } + content +} diff --git a/crates/spark-server/src/anthropic/translate.rs b/crates/spark-server/src/anthropic/translate.rs index 738e37e0f..7d1e5617e 100644 --- a/crates/spark-server/src/anthropic/translate.rs +++ b/crates/spark-server/src/anthropic/translate.rs @@ -1,405 +1,45 @@ // SPDX-License-Identifier: AGPL-3.0-only -use super::helpers::*; use super::types::*; -/// Audit the Anthropic→OpenAI translation for structural drift. -/// -/// We don't do a full inverse round-trip (that would be a separate -/// translator); instead we check three count-level invariants that -/// compound across long agent sessions when violated: -/// -/// 1. Every Anthropic user/assistant message produces at least one -/// OpenAI message (a user-with-tool_result splits into 1+ -/// `role="tool"` messages plus an optional `role="user"`). -/// 2. Every Anthropic `tool_use` content block becomes a -/// `tool_calls[i]` entry on its assistant message. -/// 3. The system field (text or blocks) becomes the first OpenAI -/// message with `role="system"`, whose text is non-empty -/// whenever the Anthropic system field had any non-billing text. -/// -/// On any mismatch: increment the drift metric, optionally log the -/// diff (gated by `ATLAS_DEBUG_TRANSLATION_DRIFT`). -pub(super) fn audit_translation_drift(req: &MessagesRequest, chat_json: &serde_json::Value) { - let mut anomalies: Vec = Vec::new(); +/// Serialize the canonical [`crate::ir::ChatResponse`] into Anthropic's +/// `MessagesResponse` wire shape (non-streaming). Reads the first +/// choice — the Anthropic adapter pins `n = 1`. +pub(super) fn ir_to_anthropic_response(ir: crate::ir::ChatResponse) -> MessagesResponse { + let id = format!("msg_{}", ir.id); + let choice = ir.choices.into_iter().next(); - // Count Anthropic-side input. - let anth_msgs = req.messages.len(); - let mut anth_tool_uses: usize = 0; - let mut anth_tool_results: usize = 0; - for m in &req.messages { - if let AnthropicContent::Blocks(blocks) = &m.content { - for b in blocks { - match b { - ContentBlock::ToolUse { .. } => anth_tool_uses += 1, - ContentBlock::ToolResult { .. } => anth_tool_results += 1, - _ => {} - } - } - } - } - let anth_system_nonempty = req - .system - .as_ref() - .map(|s| match s { - SystemContent::Text(t) => !t.trim().is_empty(), - SystemContent::Blocks(blocks) => blocks - .iter() - .filter_map(|b| b.text.as_ref()) - .any(|t| !t.starts_with("x-anthropic-") && !t.trim().is_empty()), - }) - .unwrap_or(false); - - // Count OpenAI-side output. - let openai_msgs = chat_json - .get("messages") - .and_then(|v| v.as_array()) - .map(|a| a.len()) - .unwrap_or(0); - let openai_tool_calls = chat_json - .get("messages") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter_map(|m| m.get("tool_calls").and_then(|t| t.as_array())) - .map(|tcs| tcs.len()) - .sum::() - }) - .unwrap_or(0); - let openai_role_tool = chat_json - .get("messages") - .and_then(|v| v.as_array()) - .map(|arr| { - arr.iter() - .filter(|m| m.get("role").and_then(|r| r.as_str()) == Some("tool")) - .count() - }) - .unwrap_or(0); - let openai_first_role = chat_json - .get("messages") - .and_then(|v| v.as_array()) - .and_then(|a| a.first()) - .and_then(|m| m.get("role")) - .and_then(|r| r.as_str()); - let openai_first_content = chat_json - .get("messages") - .and_then(|v| v.as_array()) - .and_then(|a| a.first()) - .and_then(|m| m.get("content")) - .and_then(|c| c.as_str()); - - // Invariant 1: Anthropic message count ≤ OpenAI message count - // (split-on-tool_result can only INCREASE). - if openai_msgs < anth_msgs { - anomalies.push(format!( - "message-count regressed: anthropic={anth_msgs} → openai={openai_msgs}" - )); - } - - // Invariant 2: tool_use block count == tool_calls count. - if anth_tool_uses != openai_tool_calls { - anomalies.push(format!( - "tool_use mismatch: anthropic blocks={anth_tool_uses}, openai tool_calls={openai_tool_calls}" - )); - } - - // Invariant 3: tool_result blocks ≤ role=tool messages. - if anth_tool_results > openai_role_tool { - anomalies.push(format!( - "tool_result lost: anthropic blocks={anth_tool_results}, openai role=tool msgs={openai_role_tool}" - )); - } - - // Invariant 4: non-empty system field → first openai message is system. - if anth_system_nonempty - && (openai_first_role != Some("system") - || openai_first_content.is_none_or(|s| s.trim().is_empty())) - { - anomalies.push( - "non-empty Anthropic system did NOT produce a non-empty system message".to_string(), - ); - } - - if anomalies.is_empty() { - return; - } - - crate::metrics::ANTHROPIC_TRANSLATION_DRIFTS.inc_by(anomalies.len() as u64); - if std::env::var("ATLAS_DEBUG_TRANSLATION_DRIFT").is_ok() { - for a in &anomalies { - tracing::warn!( - anth_msgs, - anth_tool_uses, - anth_tool_results, - openai_msgs, - openai_tool_calls, - openai_role_tool, - "anthropic translation drift: {a}" - ); - } - } -} - -pub(super) fn anthropic_to_chat_request_json(req: &MessagesRequest) -> serde_json::Value { - let mut messages: Vec = Vec::with_capacity(req.messages.len() + 1); - - // System message - if let Some(sys) = &req.system { - let sys_text = match sys { - SystemContent::Text(s) => s.clone(), - SystemContent::Blocks(blocks) => blocks - .iter() - .filter(|b| { - b.block_type == "text" - && !b.text.as_deref().unwrap_or("").starts_with("x-anthropic-") - }) - .filter_map(|b| b.text.clone()) - .collect::>() - .join("\n"), - }; - if !sys_text.is_empty() { - messages.push(serde_json::json!({ - "role": "system", - "content": sys_text, - })); + let mut content: Vec = Vec::new(); + let mut stop_reason = "end_turn"; + let mut stop_sequence: Option = None; + if let Some(c) = choice { + if let Some(r) = c.reasoning { + content.push(ResponseBlock::Thinking { thinking: r }); } - } - - // Conversation history - for m in &req.messages { - let role = match m.role.as_str() { - "assistant" => "assistant", - _ => "user", - }; - match &m.content { - AnthropicContent::Text(s) => { - messages.push(serde_json::json!({"role": role, "content": s})); - } - AnthropicContent::Blocks(blocks) => { - let mut text_parts: Vec = Vec::new(); - let mut tool_calls: Vec = Vec::new(); - let mut tool_results: Vec<(String, String)> = Vec::new(); - for b in blocks { - match b { - ContentBlock::Text { text } => text_parts.push(text.clone()), - ContentBlock::ToolUse { id, name, input } => { - tool_calls.push(serde_json::json!({ - "id": id, - "type": "function", - "function": { - "name": name, - "arguments": input.to_string(), - }, - })); - } - ContentBlock::ToolResult { - tool_use_id, - content, - is_error, - } => { - let text = content.as_ref().map(|c| c.to_text()).unwrap_or_default(); - // F6 (2026-04-26): when Anthropic's - // is_error flag is set, prepend an - // explicit `[tool error]\n` marker so the - // model has a structural signal that the - // tool call failed. Without this, the - // model has hallucinated success after - // `Exit code 127\ncargo: command not - // found` (observed in dump fix26 seq 27). - let prefixed = if is_error.unwrap_or(false) { - format!("[tool error]\n{text}") - } else { - text - }; - tool_results.push((tool_use_id.clone(), prefixed)); - } - ContentBlock::Thinking { .. } | ContentBlock::Unknown => {} - } - } - let text_content = text_parts.join(""); - if role == "assistant" { - let mut msg = serde_json::json!({ - "role": "assistant", - "content": text_content, - }); - if !tool_calls.is_empty() { - msg["tool_calls"] = serde_json::Value::Array(tool_calls); - } - messages.push(msg); - } else { - if !text_content.is_empty() { - messages.push(serde_json::json!({ - "role": "user", - "content": text_content, - })); - } - for (tool_use_id, text) in tool_results { - messages.push(serde_json::json!({ - "role": "tool", - "tool_call_id": tool_use_id, - "content": text, - })); - } - } - } + if let Some(text) = c.content + && !text.is_empty() + { + content.push(ResponseBlock::Text { text }); } - } - - // Tools array - let tools_json = req.tools.as_ref().map(|tools| { - tools - .iter() - .map(|t| { - serde_json::json!({ - "type": "function", - "function": { - "name": t.name, - "description": t.description, - "parameters": t.input_schema, - }, - }) - }) - .collect::>() - }); - - // tool_choice mapping - let tool_choice_json: Option = - req.tool_choice - .as_ref() - .map(|tc| match tc.choice_type.as_str() { - "any" => serde_json::json!("required"), - "auto" => serde_json::json!("auto"), - "none" => serde_json::json!("none"), - "tool" => { - if let Some(name) = &tc.name { - serde_json::json!({ - "type": "function", - "function": { "name": name }, - }) - } else { - serde_json::json!("auto") - } - } - _ => serde_json::json!("auto"), + for tc in c.tool_calls { + content.push(ResponseBlock::ToolUse { + id: tc.id, + name: tc.name, + input: tc.arguments, }); - - let mut chat = serde_json::json!({ - "model": req.model, - "messages": messages, - "max_tokens": req.max_tokens, - "stream": req.stream, - }); - if let Some(t) = req.temperature { - chat["temperature"] = serde_json::json!(t); - } - if let Some(k) = req.top_k { - chat["top_k"] = serde_json::json!(k); - } - if let Some(p) = req.top_p { - chat["top_p"] = serde_json::json!(p); - } - if !req.stop_sequences.is_empty() { - chat["stop"] = serde_json::json!(req.stop_sequences); - } - if let Some(tools) = tools_json { - chat["tools"] = serde_json::Value::Array(tools); - } - if let Some(tc) = tool_choice_json { - chat["tool_choice"] = tc; - } - - // Anthropic thinking config — preserve via the `thinking` field that - // ChatCompletionRequest already accepts (vLLM-compatible shape). - if let Some(thinking) = &req.thinking { - let mut t = serde_json::json!({"type": thinking.thinking_type}); - if let Some(b) = thinking.budget_tokens { - t["budget_tokens"] = serde_json::json!(b); - } - chat["thinking"] = t; - } - - chat -} - -/// Translate a non-streaming chat-completion JSON body into Anthropic's -/// `MessagesResponse` shape. Reads the body via untyped `serde_json::Value` -/// so we don't need ChatCompletionResponse to derive Deserialize. -pub(super) fn chat_to_anthropic_response( - chat_value: &serde_json::Value, - model: String, -) -> MessagesResponse { - let id = chat_value - .get("id") - .and_then(|v| v.as_str()) - .map(|s| format!("msg_{}", s.trim_start_matches("chatcmpl-"))) - .unwrap_or_else(|| "msg_unknown".to_string()); - - let usage = chat_value.get("usage").cloned().unwrap_or_default(); - let input_tokens = usage - .get("prompt_tokens") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - let output_tokens = usage - .get("completion_tokens") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize; - - let choice = chat_value - .get("choices") - .and_then(|c| c.get(0)) - .cloned() - .unwrap_or_default(); - let msg = choice.get("message").cloned().unwrap_or_default(); - let finish_reason = choice - .get("finish_reason") - .and_then(|v| v.as_str()) - .unwrap_or("stop"); - - let mut content: Vec = Vec::new(); - - // Reasoning → thinking block - let reasoning = msg - .get("reasoning_content") - .and_then(|v| v.as_str()) - .or_else(|| msg.get("reasoning").and_then(|v| v.as_str())) - .filter(|s| !s.is_empty()); - if let Some(r) = reasoning { - content.push(ResponseBlock::Thinking { - thinking: r.to_string(), - }); - } - - // Text content - if let Some(text) = msg.get("content").and_then(|v| v.as_str()) - && !text.is_empty() - { - content.push(ResponseBlock::Text { - text: text.to_string(), - }); - } - - // Tool calls - if let Some(tcs) = msg.get("tool_calls").and_then(|v| v.as_array()) { - for tc in tcs { - let id = tc - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let function = tc.get("function").cloned().unwrap_or_default(); - let name = function - .get("name") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let args_str = function - .get("arguments") - .and_then(|v| v.as_str()) - .unwrap_or("{}"); - let input: serde_json::Value = serde_json::from_str(args_str) - .unwrap_or(serde_json::Value::Object(Default::default())); - content.push(ResponseBlock::ToolUse { id, name, input }); } + stop_reason = match c.finish_reason { + // A client stop sequence gets its dedicated reason + echo. + crate::ir::FinishReason::Stop if c.matched_stop.is_some() => "stop_sequence", + crate::ir::FinishReason::Stop => "end_turn", + crate::ir::FinishReason::ToolCalls => "tool_use", + crate::ir::FinishReason::Length => "max_tokens", + // Safety-filtered output maps to Anthropic's dedicated + // refusal stop reason (2025-05 API), not a normal end_turn. + crate::ir::FinishReason::ContentFilter => "refusal", + crate::ir::FinishReason::Other(_) => "end_turn", + }; + stop_sequence = c.matched_stop; } MessagesResponse { @@ -407,12 +47,13 @@ pub(super) fn chat_to_anthropic_response( response_type: "message".to_string(), role: "assistant".to_string(), content, - model, - stop_reason: Some(convert_stop_reason(finish_reason).to_string()), - stop_sequence: None, + model: ir.model, + stop_reason: Some(stop_reason.to_string()), + stop_sequence, usage: AnthropicUsage { - input_tokens, - output_tokens, + input_tokens: ir.usage.prompt_tokens, + output_tokens: ir.usage.completion_tokens, + cache_read_input_tokens: ir.usage.cached_prompt_tokens, }, } } diff --git a/crates/spark-server/src/anthropic/translator.rs b/crates/spark-server/src/anthropic/translator.rs index 5b06aeffd..b6d323785 100644 --- a/crates/spark-server/src/anthropic/translator.rs +++ b/crates/spark-server/src/anthropic/translator.rs @@ -4,6 +4,22 @@ use axum::response::sse::Event; use super::helpers::*; +/// A typed Anthropic SSE event (event name + JSON data). Testable, unlike +/// axum's opaque `Event`; converted to an axum event at the handler edge +/// by [`to_axum_event`]. +#[derive(Debug, Clone, PartialEq)] +pub(super) struct SseEvent { + pub(super) event: String, + pub(super) data: serde_json::Value, +} + +/// Convert a typed [`SseEvent`] to an axum SSE event for the wire. +pub(super) fn to_axum_event(e: &SseEvent) -> Event { + Event::default() + .event(&e.event) + .data(serde_json::to_string(&e.data).unwrap_or_default()) +} + /// Open-block tracker for the streaming translator's state machine. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) enum OpenBlock { @@ -46,6 +62,8 @@ pub(super) enum OpenBlock { pub struct AnthropicTranslator { model: String, msg_started: bool, + /// Anthropic wire message id (`msg_`), minted at construction + /// — the delta stream is wire-id-free. msg_id: String, block_idx: u32, open_block: OpenBlock, @@ -57,6 +75,7 @@ pub struct AnthropicTranslator { tool_started: std::collections::HashMap, completion_tokens: usize, prompt_tokens: usize, + cached_prompt_tokens: usize, finished: bool, } @@ -65,19 +84,37 @@ impl AnthropicTranslator { Self { model, msg_started: false, - msg_id: String::new(), + msg_id: format!("msg_{}", crate::ids::uuid_v4()), block_idx: 0, open_block: OpenBlock::None, tool_started: std::collections::HashMap::new(), completion_tokens: 0, prompt_tokens: 0, + cached_prompt_tokens: 0, finished: false, } } - fn make_event(ev_type: &str, data: serde_json::Value) -> Event { - let json_str = serde_json::to_string(&data).unwrap_or_default(); - Event::default().event(ev_type).data(json_str) + /// Final `message_delta.usage`. Carries `input_tokens` (and cache + /// hits) as well as `output_tokens`: the upstream usage arrives on + /// the LAST chunk, after `message_start` already went out with + /// `input_tokens: 0` — without patching it here, streaming clients + /// billed every request at zero input tokens. Anthropic's + /// MessageDeltaUsage carries all three fields (2025-05 API), and + /// clients merge message_delta.usage over message_start's. + fn final_usage(&self) -> serde_json::Value { + serde_json::json!({ + "input_tokens": self.prompt_tokens, + "cache_read_input_tokens": self.cached_prompt_tokens, + "output_tokens": self.completion_tokens, + }) + } + + fn make_event(ev_type: &str, data: serde_json::Value) -> SseEvent { + SseEvent { + event: ev_type.to_string(), + data, + } } /// Close the currently open block (if any) and increment the @@ -93,7 +130,7 @@ impl AnthropicTranslator { /// a new index — Claude Code interprets each duplicate as a /// separate tool execution. Root-caused 2026-04-26 (8-agent /// sweep, F3). - pub(super) fn close_open_block(&mut self) -> Option { + pub(super) fn close_open_block(&mut self) -> Option { match self.open_block { OpenBlock::None => None, OpenBlock::Text | OpenBlock::Thinking => { @@ -124,15 +161,11 @@ impl AnthropicTranslator { } } - pub(super) fn ensure_message_start(&mut self, out: &mut Vec) { + pub(super) fn ensure_message_start(&mut self, out: &mut Vec) { if self.msg_started { return; } - let id = if self.msg_id.is_empty() { - "msg_unknown".to_string() - } else { - format!("msg_{}", self.msg_id.trim_start_matches("chatcmpl-")) - }; + let id = self.msg_id.clone(); out.push(Self::make_event( "message_start", serde_json::json!({ @@ -155,106 +188,40 @@ impl AnthropicTranslator { self.msg_started = true; } - /// Process one OpenAI chat-completion chunk JSON value, push - /// resulting Anthropic events into `out`. - pub(super) fn process_openai_chunk(&mut self, val: &serde_json::Value, out: &mut Vec) { - // Pick up the chat-completion id on the first chunk that - // carries it (typically the role chunk). - if self.msg_id.is_empty() - && let Some(s) = val.get("id").and_then(|v| v.as_str()) - { - self.msg_id = s.to_string(); - } - - let choice = match val.get("choices").and_then(|c| c.get(0)) { - Some(c) => c, - None => { - // Some OpenAI chunks (the separate-usage chunk) carry - // no `choices`. Pick up usage if present. - if let Some(u) = val.get("usage") { - if let Some(p) = u.get("prompt_tokens").and_then(|v| v.as_u64()) { - self.prompt_tokens = p as usize; - } - if let Some(c) = u.get("completion_tokens").and_then(|v| v.as_u64()) { - self.completion_tokens = c as usize; + /// Process one neutral streaming delta, pushing the resulting + /// Anthropic events into `out`. The block state machine is the same + /// one that used to reverse-engineer OpenAI chunk JSON — the inputs + /// are just typed now. + pub(super) fn on_delta(&mut self, d: &crate::ir::StreamDelta, out: &mut Vec) { + use crate::ir::StreamDelta; + match d { + StreamDelta::Reasoning { text, .. } if !text.is_empty() => { + self.ensure_message_start(out); + if !matches!(self.open_block, OpenBlock::Thinking) { + if let Some(stop) = self.close_open_block() { + out.push(stop); } - } - return; - } - }; - - let delta = choice.get("delta"); - - // Capture `usage` whenever it appears (the final chunk - // typically carries it). - if let Some(u) = val.get("usage") { - if let Some(p) = u.get("prompt_tokens").and_then(|v| v.as_u64()) { - self.prompt_tokens = p as usize; - } - if let Some(c) = u.get("completion_tokens").and_then(|v| v.as_u64()) { - self.completion_tokens = c as usize; - } - } - - // Lazy message_start. We delay it until we have something - // meaningful (the role chunk) so the emitted `prompt_tokens` - // reflect the chat-completion's reported input usage if the - // first chunk happens to carry it. Most servers send role + - // usage in the first / penultimate chunks respectively. - let has_delta_signal = delta - .and_then(|d| d.as_object()) - .map(|o| { - o.contains_key("role") - || o.contains_key("content") - || o.contains_key("tool_calls") - || o.contains_key("reasoning_content") - }) - .unwrap_or(false); - if has_delta_signal { - self.ensure_message_start(out); - } - - // Reasoning / thinking delta — Atlas emits these as - // `delta.reasoning_content` chunks during the model's - // thinking phase. Anthropic's streaming spec wraps thinking - // in its own content block (`content_block.type="thinking"`) - // with `delta.type="thinking_delta"`. Without this mapping - // Claude Code shows "Brewed for Ns" with no visible progress - // for the entire thinking phase, and users cancel thinking - // the server has stalled (2026-04-25 incident). - if let Some(d) = delta - && let Some(text) = d.get("reasoning_content").and_then(|v| v.as_str()) - && !text.is_empty() - { - if !matches!(self.open_block, OpenBlock::Thinking) { - if let Some(stop) = self.close_open_block() { - out.push(stop); + out.push(Self::make_event( + "content_block_start", + serde_json::json!({ + "type": "content_block_start", + "index": self.block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }), + )); + self.open_block = OpenBlock::Thinking; } out.push(Self::make_event( - "content_block_start", + "content_block_delta", serde_json::json!({ - "type": "content_block_start", + "type": "content_block_delta", "index": self.block_idx, - "content_block": {"type": "thinking", "thinking": ""}, + "delta": {"type": "thinking_delta", "thinking": text}, }), )); - self.open_block = OpenBlock::Thinking; } - out.push(Self::make_event( - "content_block_delta", - serde_json::json!({ - "type": "content_block_delta", - "index": self.block_idx, - "delta": {"type": "thinking_delta", "thinking": text}, - }), - )); - } - - // Text content delta. - if let Some(d) = delta { - if let Some(text) = d.get("content").and_then(|v| v.as_str()) - && !text.is_empty() - { + StreamDelta::Content { text, .. } if !text.is_empty() => { + self.ensure_message_start(out); if !matches!(self.open_block, OpenBlock::Text) { if let Some(stop) = self.close_open_block() { out.push(stop); @@ -278,136 +245,108 @@ impl AnthropicTranslator { }), )); } - - // Tool-call deltas. OpenAI emits these as an array; each - // entry has `index` (which OpenAI tool-call slot it - // refers to) plus optionally `id`, `function.name`, - // `function.arguments` (a string fragment). - if let Some(tcs) = d.get("tool_calls").and_then(|v| v.as_array()) { - for tc in tcs { - let oa_idx = tc.get("index").and_then(|v| v.as_u64()).unwrap_or(0) as usize; - let id = tc - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let function = tc.get("function"); - let name = function - .and_then(|f| f.get("name")) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let arguments_fragment = function - .and_then(|f| f.get("arguments")) - .and_then(|v| v.as_str()) - .unwrap_or(""); - - let need_start = !self.tool_started.contains_key(&oa_idx); - if need_start { - // Only open a new block once we have at least - // a name (the model's first emit per tool - // typically carries `id`+`name` together). - if name.is_empty() { - // Skip this fragment — wait for the start - // chunk to land. Subsequent argument - // fragments stay queued. - continue; - } - if !matches!(self.open_block, OpenBlock::ToolUse(idx) if idx == oa_idx) { - if let Some(stop) = self.close_open_block() { - out.push(stop); - } - out.push(Self::make_event( - "content_block_start", - serde_json::json!({ - "type": "content_block_start", - "index": self.block_idx, - "content_block": { - "type": "tool_use", - "id": id, - "name": name, - "input": {}, - }, - }), - )); - self.open_block = OpenBlock::ToolUse(oa_idx); - self.tool_started.insert(oa_idx, ()); - } - } else if !matches!(self.open_block, OpenBlock::ToolUse(idx) if idx == oa_idx) { - // Should be unreachable now that - // `close_open_block` clears `tool_started` - // for `ToolUse` (F3, 2026-04-26): if the - // block were closed, `need_start` would be - // true and we'd have taken the branch above. - // The only way to reach here is the upstream - // OpenAI stream interleaving args for tool - // index `oa_idx` while a *different* tool - // block is open without the prior block ever - // being closed — protocol-level upstream - // bug, not something we can synthesise a - // safe fix for. Drop the late fragment with - // a warning rather than emit a duplicate - // `tool_use` block (which Claude Code would - // execute twice). - tracing::warn!( - target: "anthropic_translator", - oa_idx, - current_block = ?self.open_block, - arg_fragment_len = arguments_fragment.len(), - "dropping interleaved tool-call argument fragment for already-open different tool block" - ); - continue; - } - - if !arguments_fragment.is_empty() { - out.push(Self::make_event( - "content_block_delta", - serde_json::json!({ - "type": "content_block_delta", - "index": self.block_idx, - "delta": { - "type": "input_json_delta", - "partial_json": arguments_fragment, - }, - }), - )); + StreamDelta::ToolCallStart { index, id, name } => { + self.ensure_message_start(out); + // The delta stream guarantees the name on the start event + // (the old chunk protocol could stream pre-name argument + // fragments, which had to be dropped). + let need_start = !self.tool_started.contains_key(index); + if need_start + && !matches!(self.open_block, OpenBlock::ToolUse(idx) if idx == *index) + { + if let Some(stop) = self.close_open_block() { + out.push(stop); } + out.push(Self::make_event( + "content_block_start", + serde_json::json!({ + "type": "content_block_start", + "index": self.block_idx, + "content_block": { + "type": "tool_use", + "id": id, + "name": name, + "input": {}, + }, + }), + )); + self.open_block = OpenBlock::ToolUse(*index); + self.tool_started.insert(*index, ()); } } - } - - // Finish reason: close the current block (if any) and emit - // message_delta + message_stop. The OpenAI stream may follow - // up with a [DONE] sentinel after this; we ignore it. - if let Some(fr) = choice.get("finish_reason").and_then(|v| v.as_str()) - && !self.finished - { - self.ensure_message_start(out); - if let Some(stop) = self.close_open_block() { - out.push(stop); + StreamDelta::ToolCallArgs { + index, fragment, .. + } => { + if fragment.is_empty() { + return; + } + self.ensure_message_start(out); + if !matches!(self.open_block, OpenBlock::ToolUse(idx) if idx == *index) { + // Argument fragment for a tool block that is not the + // open one — upstream protocol violation. Drop with a + // warning rather than emit a duplicate tool_use block + // (which Claude Code would execute twice). + tracing::warn!( + target: "anthropic_translator", + oa_idx = index, + current_block = ?self.open_block, + arg_fragment_len = fragment.len(), + "dropping tool-call argument fragment for non-open tool block" + ); + return; + } + out.push(Self::make_event( + "content_block_delta", + serde_json::json!({ + "type": "content_block_delta", + "index": self.block_idx, + "delta": { + "type": "input_json_delta", + "partial_json": fragment, + }, + }), + )); } - out.push(Self::make_event( - "message_delta", - serde_json::json!({ - "type": "message_delta", - "delta": { - "stop_reason": convert_stop_reason(fr), - "stop_sequence": serde_json::Value::Null, - }, - "usage": {"output_tokens": self.completion_tokens}, - }), - )); - out.push(Self::make_event( - "message_stop", - serde_json::json!({"type": "message_stop"}), - )); - self.finished = true; + StreamDelta::Finish { reason, usage, .. } => { + if self.finished { + return; + } + self.prompt_tokens = usage.prompt_tokens; + self.completion_tokens = usage.completion_tokens; + self.cached_prompt_tokens = usage.cached_prompt_tokens; + self.ensure_message_start(out); + if let Some(stop) = self.close_open_block() { + out.push(stop); + } + out.push(Self::make_event( + "message_delta", + serde_json::json!({ + "type": "message_delta", + "delta": { + "stop_reason": convert_stop_reason(reason.as_wire()), + "stop_sequence": serde_json::Value::Null, + }, + "usage": self.final_usage(), + }), + )); + out.push(Self::make_event( + "message_stop", + serde_json::json!({"type": "message_stop"}), + )); + self.finished = true; + } + // Refusals have no Anthropic streaming representation (the + // old translator ignored `delta.refusal` too); errors abort + // upstream and carry no renderable event here. + StreamDelta::Refusal { .. } | StreamDelta::Error { .. } => {} + // Empty text/reasoning fragments. + StreamDelta::Content { .. } | StreamDelta::Reasoning { .. } => {} } } /// Stream ended without an explicit `finish_reason`. Best-effort /// flush so the client sees a coherent end of message. - pub(super) fn finalize(&mut self, out: &mut Vec) { + pub(super) fn finalize(&mut self, out: &mut Vec) { if self.finished { return; } @@ -423,7 +362,7 @@ impl AnthropicTranslator { "stop_reason": "end_turn", "stop_sequence": serde_json::Value::Null, }, - "usage": {"output_tokens": self.completion_tokens}, + "usage": self.final_usage(), }), )); out.push(Self::make_event( diff --git a/crates/spark-server/src/anthropic/types.rs b/crates/spark-server/src/anthropic/types.rs index bf6153927..a4a514627 100644 --- a/crates/spark-server/src/anthropic/types.rs +++ b/crates/spark-server/src/anthropic/types.rs @@ -114,10 +114,43 @@ pub enum ContentBlock { #[serde(default)] thinking: Option, }, + #[serde(rename = "image")] + Image { source: ImageSourceBlock }, #[serde(other)] Unknown, } +/// Anthropic image source. `type:"base64"` carries `media_type` + `data`; +/// `type:"url"` carries `url`. +#[derive(Debug, Deserialize)] +pub struct ImageSourceBlock { + #[serde(rename = "type")] + pub source_type: String, + #[serde(default)] + pub media_type: Option, + #[serde(default)] + pub data: Option, + #[serde(default)] + pub url: Option, +} + +impl ImageSourceBlock { + /// Build the string the vision encoder consumes: a `data:` URI for + /// base64 sources, or the raw URL for url sources. `None` when the + /// required fields are missing. + pub(super) fn to_image_uri(&self) -> Option { + match self.source_type.as_str() { + "base64" => { + let data = self.data.as_ref()?; + let mt = self.media_type.as_deref().unwrap_or("image/png"); + Some(format!("data:{mt};base64,{data}")) + } + "url" => self.url.clone(), + _ => None, + } + } +} + /// Tool result content: string or nested blocks. #[derive(Debug, Deserialize)] #[serde(untagged)] @@ -201,4 +234,8 @@ pub enum ResponseBlock { pub struct AnthropicUsage { pub input_tokens: usize, pub output_tokens: usize, + /// Prompt tokens served from the prefix cache. Always emitted (0 + /// when the prefix cache missed) — Anthropic clients read it for + /// cache accounting. + pub cache_read_input_tokens: usize, } diff --git a/crates/spark-server/src/api/chat/echo.rs b/crates/spark-server/src/api/chat/echo.rs new file mode 100644 index 000000000..2dbf72a98 --- /dev/null +++ b/crates/spark-server/src/api/chat/echo.rs @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Echo-only response context. These wire fields never influence +// generation — they are read back verbatim when the response is +// encoded (service_tier/metadata echo, `store: true` persistence, +// include_usage stream framing). They ride beside the IR envelope +// instead of inside it so the IR carries exactly what the pipeline +// computes with (PCND). Absorbed into the per-surface encoders once +// the response direction is fully IR-based. + +/// Encode-time echoes captured from the surface's wire request. +#[derive(Debug, Clone, Default)] +pub(crate) struct ResponseEcho { + pub(crate) service_tier: Option, + pub(crate) metadata: Option>, + /// Persist the completion for later retrieval (`store: true`). + pub(crate) store: bool, + /// Emit the final usage-only chunk before `[DONE]` + /// (`stream_options.include_usage`). + pub(crate) include_usage: bool, +} + +impl ResponseEcho { + /// Capture the echo fields from a parsed OpenAI wire request. + pub(crate) fn from_wire(req: &crate::openai::ChatCompletionRequest) -> Self { + ResponseEcho { + service_tier: req.service_tier.clone(), + metadata: req.metadata.clone(), + store: req.store.unwrap_or(false), + include_usage: req.stream_options.map(|o| o.include_usage).unwrap_or(false), + } + } +} diff --git a/crates/spark-server/src/api/chat/loop_detect.rs b/crates/spark-server/src/api/chat/loop_detect.rs index b8768bcb5..e720ed2a2 100644 --- a/crates/spark-server/src/api/chat/loop_detect.rs +++ b/crates/spark-server/src/api/chat/loop_detect.rs @@ -8,7 +8,7 @@ // returned via `LoopDetectOut` so the orchestrator can wire them // into the downstream sampling-bias logic. -use crate::openai::ChatCompletionRequest; +use crate::ir::{Message, Role}; pub(super) struct LoopDetectOut { /// True when the verdict was Suppress OR spinning detection @@ -33,7 +33,7 @@ fn loop_suppress_disabled() -> bool { std::env::var("ATLAS_LOOP_NO_SUPPRESS").as_deref() == Ok("1") } -pub(super) fn check_loops(req: &ChatCompletionRequest, tools_active: bool) -> LoopDetectOut { +pub(super) fn check_loops(messages: &[Message], tools_active: bool) -> LoopDetectOut { let mut suppress_tool_call = false; let mut tool_call_repeat_count: usize = 0; @@ -44,20 +44,26 @@ pub(super) fn check_loops(req: &ChatCompletionRequest, tools_active: bool) -> Lo }; } - let signatures: Vec = req - .messages + // IR tool-call arguments are structured JSON; the signature hasher + // wants strings. Re-serialization is deterministic per turn, so + // turn-to-turn similarity (the only thing the detector compares) is + // unaffected by the wire's original whitespace. + let signatures: Vec = messages .iter() .rev() - .filter(|m| m.role == "assistant") + .filter(|m| m.role == Role::Assistant) .map(|m| { - let calls: Vec<(&str, &str)> = m + let text = m.text(); + let owned: Vec<(String, String)> = m .tool_calls - .as_deref() - .unwrap_or(&[]) .iter() - .map(|tc| (tc.function.name.as_str(), tc.function.arguments.as_str())) + .map(|tc| (tc.name.clone(), tc.arguments.to_string())) .collect(); - crate::loop_detector::Signature::build(&m.content.text, calls) + let calls: Vec<(&str, &str)> = owned + .iter() + .map(|(n, a)| (n.as_str(), a.as_str())) + .collect(); + crate::loop_detector::Signature::build(&text, calls) }) .take(8) .collect(); @@ -67,13 +73,15 @@ pub(super) fn check_loops(req: &ChatCompletionRequest, tools_active: bool) -> Lo // produced ≥5 consecutive short, low-content responses, // something is structurally wrong even if no two are similar. let mut recent_short: usize = 0; - for m in req.messages.iter().rev() { - if m.role != "assistant" { + for m in messages.iter().rev() { + if m.role != Role::Assistant { continue; } - let tool_args_len: usize = m.tool_calls.as_ref().map_or(0, |tcs| { - tcs.iter().map(|tc| tc.function.arguments.len()).sum() - }); + let tool_args_len: usize = m + .tool_calls + .iter() + .map(|tc| tc.arguments.to_string().len()) + .sum(); // A turn that issued ANY tool call is taking an action (progress) — it // is NOT spinning, even when the args are short. In an agentic coding // loop the verify cycle (`bash cargo build`, `bash cargo run`, @@ -85,8 +93,8 @@ pub(super) fn check_loops(req: &ChatCompletionRequest, tools_active: bool) -> Lo // Genuine repeated-tool-call loops are caught separately by // `loop_detector::detect` (the Suppress verdict above); spinning here // should only fire on consecutive short PURE-TEXT turns (no action). - let made_tool_call = m.tool_calls.as_ref().is_some_and(|tcs| !tcs.is_empty()); - let is_substantial = made_tool_call || m.content.text.len() >= 500 || tool_args_len >= 100; + let made_tool_call = !m.tool_calls.is_empty(); + let is_substantial = made_tool_call || m.text().len() >= 500 || tool_args_len >= 100; if is_substantial { break; } diff --git a/crates/spark-server/src/api/chat/mod.rs b/crates/spark-server/src/api/chat/mod.rs index d9dd63352..50740527e 100644 --- a/crates/spark-server/src/api/chat/mod.rs +++ b/crates/spark-server/src/api/chat/mod.rs @@ -22,8 +22,10 @@ //! - `sampling_setup` — preset / penalty / stop-token / grammar / //! timeout / logprobs resolution +pub(crate) mod echo; mod loop_detect; mod msg_entry; +pub(crate) mod prepare; mod sampling_setup; mod template; mod thinking; @@ -34,7 +36,37 @@ use axum::response::{IntoResponse, Json, Response}; use std::sync::Arc; use crate::AppState; -use crate::openai::ChatCompletionRequest; + +pub(crate) use echo::ResponseEcho; + +/// Result of the shared chat pipeline. A non-streaming success carries +/// the canonical response IR for the caller's surface encoder; +/// streaming SSE and error envelopes are already-complete HTTP +/// responses (streaming moves onto the delta IR next). +pub(crate) enum ChatOutcome { + Blocking(Box), + /// Streaming success: the neutral delta stream; each surface runs + /// its own SSE encoder over it. + Streaming(crate::ir::DeltaStream), + Http(Response), +} + +/// Test-only accessors: cross-module tests (the Anthropic adapter's +/// rendered-prompt golden) drive the IR → MsgEntry → template-JSON +/// path without an AppState. +#[cfg(test)] +#[allow(clippy::result_large_err)] +pub(crate) fn test_build_msg_entries( + input: &[crate::ir::Message], + tools_active: bool, +) -> Result, axum::response::Response> { + msg_entry::build_msg_entries(None, None, input, tools_active).map(|o| o.messages) +} + +#[cfg(test)] +pub(crate) fn test_build_json_messages(entries: &[msg_entry::MsgEntry]) -> Vec { + template::build_json_messages(entries) +} use super::compact::openai_error_response; @@ -48,7 +80,7 @@ pub async fn chat_completions( // handler path and the `--dump` raw-capture path without // cloning the struct or cascading `Serialize` through every // request type. - let req: ChatCompletionRequest = match serde_json::from_slice(&body) { + let req: crate::openai::ChatCompletionRequest = match serde_json::from_slice(&body) { Ok(r) => r, Err(e) => { return openai_error_response( @@ -70,20 +102,31 @@ pub async fn chat_completions( } }); - chat_completions_inner(state, req_ctx, req, dump_seq).await + // Wire → IR at the edge: echo-only fields peel off beside the + // envelope; everything downstream reads only the IR. + let echo = ResponseEcho::from_wire(&req); + match chat_completions_inner(state.clone(), req_ctx, req.into_ir(), dump_seq).await { + ChatOutcome::Blocking(ir) => { + crate::openai::encode_chat_response(&state, *ir, &echo, dump_seq) + } + ChatOutcome::Streaming(deltas) => { + crate::openai::encode_sse_response(deltas, state.model_name.clone(), echo.include_usage) + } + ChatOutcome::Http(r) => r, + } } -/// Internal entry for the parsed-request path. Called by -/// [`chat_completions`] after body capture, and by the Responses -/// API adapter (which builds a `ChatCompletionRequest` in-memory -/// and skips HTTP body bytes). `dump_seq` is `Some` only on the -/// public handler path. +/// Internal entry for the IR-request path. Called by +/// [`chat_completions`] after body capture and wire→IR lowering, and +/// by the Responses / Anthropic adapters (which lower their own wire +/// formats and skip HTTP body bytes). `dump_seq` is `Some` only on +/// the public handler path. pub(crate) async fn chat_completions_inner( state: Arc, req_ctx: Option>, - mut req: ChatCompletionRequest, + mut req: crate::ir::ChatRequest, dump_seq: Option, -) -> Response { +) -> ChatOutcome { crate::metrics::REQUESTS_TOTAL.inc(); crate::metrics::REQUESTS_ACTIVE.inc(); @@ -93,14 +136,9 @@ pub(crate) async fn chat_completions_inner( // terminal path decrements it, but this fail-fast 400 returns before // reaching a dispatch handler. crate::metrics::REQUESTS_ACTIVE.dec(); - return resp; + return ChatOutcome::Http(resp); } - // Tool-active gating. - let tools_active = state.tool_call_parser.is_some() - && req.tools.as_ref().is_some_and(|t| !t.is_empty()) - && !req.tool_choice.as_ref().is_some_and(|tc| tc.is_none()); - // Tool-parser behavioral system prompt REMOVED again (2026-05-25 PM). // // Re-injecting the qwen3_coder `system_prompt` (with its @@ -121,104 +159,40 @@ pub(crate) async fn chat_completions_inner( // byte-exact + gate-BF16 + thinking_in_tools=true is the live // combination. - // ST-995 fix: restore the parser-specific behavioral system prompt #90 removed. - // For the hermes parser this is the canonical NousResearch function-calling - // prompt ("you MAY call one or more functions... don't make assumptions"), - // which the GDN model needs to correctly DECLINE on irrelevance prompts. With - // it (and compact tool-JSON) hallucination returns to ~96 (vs 30/64 without). - if tools_active && let Some(ref parser) = state.tool_call_parser { - let default_choice = crate::tool_parser::ToolChoice::Mode("auto".to_string()); - let tool_choice = req.tool_choice.as_ref().unwrap_or(&default_choice); - let tool_prompt = parser.system_prompt(req.tools.as_deref().unwrap_or(&[]), tool_choice); - if let Some(first) = req.messages.first_mut().filter(|m| m.role == "system") { - first.content.text = format!("{}\n\n{}", tool_prompt, first.content.text); - } else { - req.messages.insert( - 0, - crate::openai::IncomingMessage::synthetic_system(tool_prompt), - ); - } - } - - tracing::info!( - "Request: model={}, messages={}, tools={}, tools_active={}, tool_choice={:?}, stream={}, temp={:?}, max_tokens={}, freq_pen={:?}, rep_pen={:?}", - req.model, - req.messages.len(), - req.tools.as_ref().map_or(0, |t| t.len()), + // ── Phases 1-5 (prompt-affecting): shared with count_tokens ── + let prepare::PreparedChat { tools_active, - req.tool_choice, - req.stream, - req.temperature, - req.max_tokens, - req.frequency_penalty, - req.repetition_penalty, - ); - - // ── Phase 1: build MsgEntry vec + image preprocess + cwd ──── - let msg_entry::BuildOut { - messages, cwd_hint, image_pixels, - image_pad_counts, - } = match msg_entry::build_msg_entries(&state, &req, tools_active) { - Ok(o) => o, - Err(resp) => return resp, + prompt_tokens, + enable_thinking, + thinking_budget, + } = match prepare::prepare_chat_prompt(&state, &mut req) { + Ok(p) => p, + Err(resp) => return ChatOutcome::Http(resp), }; - // ── Phase 1.5: merge server-level chat_template_kwargs default ─ - // When the client sends no thinking parameters and the server has a - // --default-chat-template-kwargs flag set, inject those kwargs into - // the request so the existing resolve_thinking() chain sees them as - // normal request-body fields. We don't mutate the resolution logic — - // we just pre-populate the field it already checks. - if let Some(ref default_kw) = state.default_chat_template_kwargs - && !req.thinking_explicitly_requested() - { - req.chat_template_kwargs = Some(default_kw.clone()); - } - - // ── Phase 2: thinking resolution (pre-template) ───────────── - let (enable_thinking, thinking_budget) = thinking::resolve_thinking(&state, &req, tools_active); - // ── Phase 4: generic loop / spinning detection + task pin ─── let loop_detect::LoopDetectOut { suppress_tool_call, tool_call_repeat_count, - } = loop_detect::check_loops(&req, tools_active); - - // ── Phase 5: render Jinja template + image-pad expansion ──── - let template::TemplateOut { - prompt_tokens, - enable_thinking, - thinking_budget, - } = match template::render_template( - &state, - &req, - &messages, - &image_pad_counts, - enable_thinking, - thinking_budget, - tools_active, - ) { - Ok(o) => o, - Err(resp) => return resp, - }; + } = loop_detect::check_loops(&req.messages, tools_active); let session_hash = crate::session_manager::compute_session_hash(&prompt_tokens); - let tools_count = req.tools.as_ref().map_or(0, |t| t.len()); + let tools_count = req.tools.len(); tracing::info!( "Session {session_hash:#x}: {prompt_tokens} prompt tokens, tools={tools_active} ({tools_count} defined)", prompt_tokens = prompt_tokens.len() ); let prompt_len = prompt_tokens.len(); if prompt_len >= state.max_seq_len { - return openai_error_response( + return ChatOutcome::Http(openai_error_response( StatusCode::BAD_REQUEST, format!( "Prompt too long: {prompt_len} tokens exceeds max_seq_len {} (leave room for output tokens)", state.max_seq_len ), - ); + )); } // ── Phase 6: sampling preset / stop / grammar / timeout ───── @@ -251,7 +225,7 @@ pub(crate) async fn chat_completions_inner( tool_call_repeat_count, ) { Ok(s) => s, - Err(resp) => return resp, + Err(resp) => return ChatOutcome::Http(resp), }; // ── Phase 7: dispatch streaming or blocking ───────────────── @@ -296,7 +270,6 @@ pub(crate) async fn chat_completions_inner( state, req, req_ctx, - dump_seq, prompt_tokens, session_hash, image_pixels, diff --git a/crates/spark-server/src/api/chat/msg_entry.rs b/crates/spark-server/src/api/chat/msg_entry.rs index 00a83f407..7bcb5080f 100644 --- a/crates/spark-server/src/api/chat/msg_entry.rs +++ b/crates/spark-server/src/api/chat/msg_entry.rs @@ -10,10 +10,10 @@ use axum::http::StatusCode; use axum::response::Response; -use std::sync::Arc; -use crate::AppState; -use crate::openai::ChatCompletionRequest; +use atlas_core::config::VisionConfig; + +use crate::ir::{ContentPart, ImageData, Message, Role}; use super::super::compact::openai_error_response; @@ -21,7 +21,7 @@ use super::super::compact::openai_error_response; /// `tool_calls`, and image-part count for the Jinja vision-marker /// expansion. `pub(super)` so `chat::chat_completions_inner` and /// the other `chat/*` sub-files can read every field. -pub(super) struct MsgEntry { +pub(crate) struct MsgEntry { pub(super) role: String, pub(super) content: String, /// Structured tool_calls for the Jinja template (arguments @@ -51,13 +51,51 @@ pub(super) struct BuildOut { pub(super) image_pad_counts: Vec, } +/// Append the encoder-input string for every image part on `m` to +/// `all_images`, growing `image_pad_counts` in lockstep (each pad count is +/// filled in later by the vision preprocessor). Shared by the tool-message +/// branch and the normal branch so images ride every role uniformly — +/// including tool results, the motivating case for issue #165. +#[allow(clippy::result_large_err)] +fn collect_message_images( + m: &Message, + all_images: &mut Vec, + image_pad_counts: &mut Vec, +) -> Result<(), Response> { + for part in &m.content { + if let ContentPart::Image(src) = part { + let uri = match &src.data { + ImageData::Base64(s) => s.clone(), + // The encoder does not fetch remote URLs. Fed onward, the + // URL string would hit the base64 decoder and fail with a + // confusing "base64 decode failed" — reject with the real + // reason instead (PCND: fail fast). + ImageData::Url(url) => { + let shown: String = url.chars().take(120).collect(); + return Err(openai_error_response( + StatusCode::BAD_REQUEST, + format!( + "image URLs are not fetched by this server (got '{shown}'); \ + send the image as a base64 data: URI" + ), + )); + } + }; + all_images.push(uri); + image_pad_counts.push(0); + } + } + Ok(()) +} + #[allow(clippy::result_large_err)] pub(super) fn build_msg_entries( - state: &Arc, - req: &ChatCompletionRequest, + vision_config: Option<&VisionConfig>, + vision_max_pixels: Option, + input: &[Message], tools_active: bool, ) -> Result { - let mut messages: Vec = Vec::with_capacity(req.messages.len()); + let mut messages: Vec = Vec::with_capacity(input.len()); let mut all_images: Vec = Vec::new(); let mut image_pad_counts: Vec = Vec::new(); let mut consecutive_tool_errors: u32 = 0; @@ -72,49 +110,49 @@ pub(super) fn build_msg_entries( // assistant turns. The Jinja template already does this gating // itself (via its own `ns.last_query_index` computation) and the // injection here was the source of empty-think poisoning. Removed. - for (msg_idx, m) in req.messages.iter().enumerate() { - let _ = msg_idx; - let text = m.content.text.clone(); + for m in input.iter() { + let mut text = m.text(); + // F6: a failed tool result (Anthropic `is_error`, carried as + // `Message::tool_error`) gets an explicit ASCII marker — chat-tuned + // models have no structural error concept and otherwise hallucinate + // success over error text. Rendered here (not in the adapters) so + // every surface gets identical prompt bytes, and applied before the + // error-hint scan below so hints see the final text. + if m.tool_error { + text = format!("[tool error]\n{text}"); + } // Preserve structured tool_calls for the Jinja template. // Always extract from assistant messages — past turns may // carry tool_calls that the template MUST render even when - // the current request didn't pass `tools`. - let tool_calls_json = if m.role == "assistant" { - m.tool_calls.as_ref().and_then(|tcs| { - if tcs.is_empty() { - return None; - } - let parsed: Vec = tcs - .iter() - .map(|tc| { - let args: serde_json::Value = serde_json::from_str(&tc.function.arguments) - .unwrap_or(serde_json::Value::Object(Default::default())); - serde_json::json!({ - "id": tc.id.as_deref().unwrap_or(""), - "type": "function", - "function": { - "name": tc.function.name, - "arguments": args - } - }) + // the current request didn't pass `tools`. `tc.arguments` is + // already structured JSON in the IR (parsed at the adapter + // boundary), so we forward it directly. + let tool_calls_json = if m.role == Role::Assistant && !m.tool_calls.is_empty() { + let parsed: Vec = m + .tool_calls + .iter() + .map(|tc| { + serde_json::json!({ + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": tc.arguments + } }) - .collect(); - Some(parsed) - }) + }) + .collect(); + Some(parsed) } else { None }; // BW1: tally tool-call productivity (write/edit/build-run vs explore). - if m.role == "assistant" - && let Some(ref tcs) = m.tool_calls - { - for tc in tcs { + if m.role == Role::Assistant && !m.tool_calls.is_empty() { + for tc in &m.tool_calls { total_tool_calls += 1; - let args: serde_json::Value = - serde_json::from_str(&tc.function.arguments).unwrap_or_default(); - if crate::hint_injector::tool_call_is_productive(&tc.function.name, &args) { + if crate::hint_injector::tool_call_is_productive(&tc.name, &tc.arguments) { productive_tool_calls += 1; } } @@ -123,7 +161,7 @@ pub(super) fn build_msg_entries( // Tool-response messages: pass raw content; Jinja template // handles `` wrapping and consecutive // grouping. - if tools_active && m.role == "tool" { + if tools_active && m.role == Role::Tool { let mut text = text; if crate::hint_injector::looks_like_error(&text) { consecutive_tool_errors += 1; @@ -135,13 +173,14 @@ pub(super) fn build_msg_entries( role: "tool".into(), content: text, tool_calls: None, - image_count: 0, + image_count: m.image_count(), reasoning_content: None, }); + collect_message_images(m, &mut all_images, &mut image_pad_counts)?; continue; } - let image_count = m.content.images.len(); + let image_count = m.image_count(); // Wave 3 (2026-05-26): `ATLAS_STRIP_REASONING_HISTORY=1` drops // historical reasoning_content entirely. Matches MLC commit // d75d64e (Apr 2026) `strip_reasoning_in_history` for qwen3, @@ -153,8 +192,18 @@ pub(super) fn build_msg_entries( let strip_reasoning = std::env::var("ATLAS_STRIP_REASONING_HISTORY") .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) .unwrap_or(false); + // OpenAI's `developer` role is the successor of `system` (o-series + // clients send it). Normalize at build time so every downstream + // system-message scan (cwd hint, CWD injection, vacuous-strip) and + // the template see one canonical role — previously the mapping + // happened only at JSON render time, so `developer` messages + // bypassed those scans. + let role = match &m.role { + Role::Other(r) if r == "developer" => "system".to_string(), + r => r.as_wire().to_string(), + }; messages.push(MsgEntry { - role: m.role.clone(), + role, content: text, tool_calls: tool_calls_json, image_count, @@ -165,21 +214,16 @@ pub(super) fn build_msg_entries( // empty-`\n\n\n\n` poisoning, because F6's // template change skips the wrapper when reasoning_content // is empty. - reasoning_content: if m.role == "assistant" && !strip_reasoning { - m.reasoning_content + reasoning_content: if m.role == Role::Assistant && !strip_reasoning { + m.reasoning .as_ref() - .map(|s| s.trim().to_string()) + .map(|r| r.text.trim().to_string()) .filter(|s| !s.is_empty()) } else { None }, }); - if !m.content.images.is_empty() { - for img_uri in &m.content.images { - all_images.push(img_uri.clone()); - image_pad_counts.push(0); - } - } + collect_message_images(m, &mut all_images, &mut image_pad_counts)?; } // Extract working directory from the system message if present. @@ -234,16 +278,23 @@ pub(super) fn build_msg_entries( ); } - // Preprocess images if a vision config is available. + // Preprocess images. One shared fail-fast point: if images were + // supplied but the model has no vision encoder, reject the request + // (issue #165) instead of silently dropping the user's input with a + // 200 — the old text-only behavior lost images without any signal. let mut image_pixels: Vec<(Vec, usize, usize)> = Vec::new(); - if !all_images.is_empty() - && let Some(vcfg) = &state.vision_config - { + if !all_images.is_empty() { + let Some(vcfg) = vision_config else { + return Err(openai_error_response( + StatusCode::BAD_REQUEST, + "this model does not accept image input (no vision config)".to_string(), + )); + }; for (idx, uri) in all_images.iter().enumerate() { match spark_model::vision_preprocess::preprocess_image_with_max_pixels( uri, vcfg, - state.vision_max_pixels, + vision_max_pixels, ) { Ok((pixels, grid_h, grid_w)) => { image_pad_counts[idx] = spark_model::vision_preprocess::image_pad_count( @@ -262,8 +313,6 @@ pub(super) fn build_msg_entries( } } } - // If no vision_config (text-only model), image_pad_counts stays - // 0 and images are silently dropped on the encoder side. // BW1 bash-wandering watchdog: if the agent has run many tool calls with // no productive file output, append a steering nudge to the most recent @@ -343,3 +392,108 @@ mod vacuous_system_tests { )); } } + +#[cfg(test)] +mod build_tests { + use super::build_msg_entries; + use crate::ir::message::{ContentPart, ImageData, ImageSource, Message, Role}; + use axum::http::StatusCode; + + fn assert_bad_request(msgs: &[Message], tools_active: bool) { + match build_msg_entries(None, None, msgs, tools_active) { + Ok(_) => panic!("expected 400, got Ok"), + Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), + } + } + + fn text(role: Role, t: &str) -> Message { + Message { + role, + content: vec![ContentPart::Text(t.into())], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + } + } + + fn image(role: Role) -> Message { + Message { + role, + content: vec![ + ContentPart::Image(ImageSource { + data: ImageData::Base64("data:image/png;base64,AAA".into()), + }), + ContentPart::Text("result".into()), + ], + tool_calls: Vec::new(), + tool_call_id: Some("c1".into()), + name: None, + reasoning: None, + tool_error: false, + } + } + + #[test] + fn text_only_builds_without_vision_config() { + let msgs = vec![text(Role::User, "hello")]; + let out = build_msg_entries(None, None, &msgs, false).expect("text-only ok"); + assert_eq!(out.messages.len(), 1); + assert_eq!(out.messages[0].image_count, 0); + } + + #[test] + fn user_image_without_vision_config_is_rejected() { + // Previously: silently dropped (200). Now: fail fast. + assert_bad_request(&[text(Role::User, "hi"), image(Role::User)], false); + } + + #[test] + fn tool_result_image_is_collected_and_rejected_without_vision() { + // Proves the tool branch now COUNTS + COLLECTS images (it used to + // hardcode image_count: 0 and `continue` before collection): the + // fail-fast only fires when an image was actually collected. + assert_bad_request(&[text(Role::User, "look"), image(Role::Tool)], true); + } + + #[test] + fn developer_role_normalized_to_system_at_build() { + // Was previously mapped only at JSON render time, so `developer` + // messages bypassed the cwd-hint / vacuous-system / CWD-injection + // scans that string-compare on "system". + let msgs = vec![text(Role::Other("developer".into()), "be terse")]; + let out = build_msg_entries(None, None, &msgs, false).expect("ok"); + assert_eq!(out.messages[0].role, "system"); + + // Other unknown roles still pass through verbatim for the + // template to handle. + let msgs = vec![text(Role::Other("critic".into()), "hm")]; + let out = build_msg_entries(None, None, &msgs, false).expect("ok"); + assert_eq!(out.messages[0].role, "critic"); + } + + #[test] + fn remote_url_image_rejected_with_clear_reason() { + // A https URL used to be mislabeled as base64 and die later in + // the vision preprocessor with "base64 decode failed". It must + // now be rejected up front with the real reason — even when the + // model HAS no vision config (the URL check fires first, at + // collection time). + let url_msg = Message { + role: Role::User, + content: vec![ContentPart::Image(ImageSource { + data: ImageData::Url("https://example.com/cat.png".into()), + })], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + }; + match build_msg_entries(None, None, &[url_msg], false) { + Ok(_) => panic!("expected 400, got Ok"), + Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), + } + } +} diff --git a/crates/spark-server/src/api/chat/prepare.rs b/crates/spark-server/src/api/chat/prepare.rs new file mode 100644 index 000000000..459f56718 --- /dev/null +++ b/crates/spark-server/src/api/chat/prepare.rs @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Shared prompt-preparation front half of the chat pipeline: tool +// gating + tool-prompt injection + MsgEntry build (image preprocess, +// cwd extraction) + thinking resolution + Jinja render. Extracted from +// `chat_completions_inner` so `/v1/messages/count_tokens` counts +// against the EXACT prompt the serving path renders instead of a +// divergent third lowering. + +use axum::response::Response; +use std::sync::Arc; + +use crate::AppState; +use crate::ir::ChatRequest; + +use super::{msg_entry, template, thinking}; + +/// Outputs of [`prepare_chat_prompt`]. +pub(crate) struct PreparedChat { + pub(crate) tools_active: bool, + pub(crate) cwd_hint: Option, + pub(crate) image_pixels: Vec<(Vec, usize, usize)>, + pub(crate) prompt_tokens: Vec, + pub(crate) enable_thinking: bool, + pub(crate) thinking_budget: Option, +} + +/// Run the prompt-affecting phases against the IR envelope. Mutates +/// `req.messages` (tool-prompt injection). Everything here must stay +/// deterministic for a given `(req, state)` — the rendered +/// `prompt_tokens` are the kv-cache prefix. +#[allow(clippy::result_large_err)] +pub(crate) fn prepare_chat_prompt( + state: &Arc, + req: &mut ChatRequest, +) -> Result { + // Tool-active gating. + let tools_active = state.tool_call_parser.is_some() + && !req.tools.is_empty() + && !req.tool_choice.as_ref().is_some_and(|tc| tc.is_none()); + + // ST-995 fix: restore the parser-specific behavioral system prompt #90 + // removed. For the hermes parser this is the canonical NousResearch + // function-calling prompt ("you MAY call one or more functions... + // don't make assumptions"), which the GDN model needs to correctly + // DECLINE on irrelevance prompts. With it (and compact tool-JSON) + // hallucination returns to ~96 (vs 30/64 without). + if tools_active && let Some(ref parser) = state.tool_call_parser { + let default_choice = crate::tool_parser::ToolChoice::Mode("auto".to_string()); + let tool_choice = req.tool_choice.as_ref().unwrap_or(&default_choice); + let tool_prompt = parser.system_prompt(&req.tools, tool_choice); + if let Some(first) = req + .messages + .first_mut() + .filter(|m| m.role == crate::ir::Role::System) + { + first.prepend_text(&format!("{tool_prompt}\n\n")); + } else { + req.messages + .insert(0, crate::ir::Message::synthetic_system(tool_prompt)); + } + } + + tracing::info!( + "Request: model={}, messages={}, tools={}, tools_active={}, tool_choice={:?}, stream={}, temp={:?}, max_tokens={}, freq_pen={:?}, rep_pen={:?}", + req.model, + req.messages.len(), + req.tools.len(), + tools_active, + req.tool_choice, + req.stream, + req.sampling.temperature, + req.max_tokens, + req.sampling.frequency_penalty, + req.sampling.repetition_penalty, + ); + + // ── Phase 1: build MsgEntry vec + image preprocess + cwd ──── + let msg_entry::BuildOut { + messages, + cwd_hint, + image_pixels, + image_pad_counts, + } = msg_entry::build_msg_entries( + state.vision_config.as_ref(), + state.vision_max_pixels, + &req.messages, + tools_active, + )?; + + // ── Phase 1.5 + 2: thinking directive + resolution (pre-template) ─ + // The client's per-request directive (resolved at the API edge) wins; + // when the client is silent the server-level + // --default-chat-template-kwargs directive applies; when that too is + // unspecified, MODEL.toml decides inside resolve_thinking. + let mut thinking_directive = req.thinking; + if !thinking_directive.is_explicit() { + thinking_directive = state.default_thinking; + } + let (enable_thinking, thinking_budget) = thinking::resolve_thinking( + state, + thinking_directive, + req.max_tokens as u32, + tools_active, + ); + + // ── Phase 5: render Jinja template + image-pad expansion ──── + let template::TemplateOut { + prompt_tokens, + enable_thinking, + thinking_budget, + } = template::render_template( + state, + &req.tools, + &messages, + &image_pad_counts, + enable_thinking, + thinking_budget, + tools_active, + )?; + + Ok(PreparedChat { + tools_active, + cwd_hint, + image_pixels, + prompt_tokens, + enable_thinking, + thinking_budget, + }) +} diff --git a/crates/spark-server/src/api/chat/sampling_setup.rs b/crates/spark-server/src/api/chat/sampling_setup.rs index 4ec2f80e0..897b2fc11 100644 --- a/crates/spark-server/src/api/chat/sampling_setup.rs +++ b/crates/spark-server/src/api/chat/sampling_setup.rs @@ -10,7 +10,7 @@ use axum::response::Response; use std::sync::Arc; use crate::AppState; -use crate::openai::ChatCompletionRequest; +use crate::ir::ChatRequest; use crate::tool_parser; use super::super::compact::openai_error_response; @@ -61,7 +61,7 @@ fn tool_choice_required_for_parser( #[allow(clippy::result_large_err)] pub(super) fn build_sampling( state: &Arc, - req: &ChatCompletionRequest, + req: &ChatRequest, enable_thinking: bool, tools_active: bool, suppress_tool_call: bool, @@ -97,42 +97,52 @@ pub(super) fn build_sampling( let temperature = if force_temp_zero { 0.0 } else { - req.temperature.unwrap_or(state.default_temperature) + req.sampling + .temperature + .unwrap_or(state.default_temperature) }; let top_k = if force_temp_zero { 0 } else { - req.top_k.unwrap_or(state.default_top_k) + req.sampling.top_k.unwrap_or(state.default_top_k) }; let top_p = if force_temp_zero { 1.0 } else { - req.top_p.unwrap_or(state.default_top_p) + req.sampling.top_p.unwrap_or(state.default_top_p) }; let top_n_sigma = if force_temp_zero { 0.0 } else { - req.top_n_sigma.unwrap_or(state.default_top_n_sigma) + req.sampling + .top_n_sigma + .unwrap_or(state.default_top_n_sigma) }; let min_p = if force_temp_zero { 0.0 } else { - req.min_p.unwrap_or(state.default_min_p) + req.sampling.min_p.unwrap_or(state.default_min_p) }; let repetition_penalty = if force_temp_zero { 1.0 } else { - req.repetition_penalty.unwrap_or(preset.repetition_penalty) + req.sampling + .repetition_penalty + .unwrap_or(preset.repetition_penalty) }; let presence_penalty = if force_temp_zero { 0.0 } else { - req.presence_penalty.unwrap_or(preset.presence_penalty) + req.sampling + .presence_penalty + .unwrap_or(preset.presence_penalty) }; let frequency_penalty = if force_temp_zero { 0.0 } else { - req.frequency_penalty.unwrap_or(preset.frequency_penalty) + req.sampling + .frequency_penalty + .unwrap_or(preset.frequency_penalty) }; // Per-model server-side sampling SAFETY FLOOR/CEILING (MODEL.toml // [behavior]). Binds AFTER request/preset resolution so model stability @@ -178,15 +188,11 @@ pub(super) fn build_sampling( )); } - // Logit bias from OpenAI (string keys) → Vec<(u32, f32)>. + // Logit bias (already parsed to typed pairs at the API edge). let mut logit_bias: Vec<(u32, f32)> = if force_temp_zero { Vec::new() } else { - req.logit_bias.as_ref().map_or(Vec::new(), |map| { - map.iter() - .filter_map(|(k, &v)| k.parse::().ok().map(|id| (id, v))) - .collect() - }) + req.logit_bias.clone() }; // Exponential `` bias decay. Skipped under ATLAS_FORCE_TEMP_ZERO @@ -254,10 +260,9 @@ pub(super) fn build_sampling( // is conventionally embedded in the user/system message by the // caller, and capable models (Qwen3.6, etc.) follow it without // server-side enforcement on free-text turns. - let has_response_format = req - .response_format - .as_ref() - .is_some_and(|rf| !matches!(rf, crate::openai::ResponseFormat::Text)); + // The wire's `{"type":"text"}` was mapped to `None` at the edge, so + // presence alone means a real constraint. + let has_response_format = req.response_format.is_some(); let tool_choice_none = req .tool_choice .as_ref() @@ -268,13 +273,10 @@ pub(super) fn build_sampling( let use_triggers = !tool_choice_required; let grammar_spec: Option = if response_format_only { match req.response_format.as_ref().unwrap() { - crate::openai::ResponseFormat::JsonObject => Some(GrammarSpec::JsonObject), - crate::openai::ResponseFormat::JsonSchema { json_schema } => { - Some(GrammarSpec::JsonSchema { - schema: json_schema.schema.to_string(), - }) - } - crate::openai::ResponseFormat::Text => None, + crate::ir::ResponseFormat::JsonObject => Some(GrammarSpec::JsonObject), + crate::ir::ResponseFormat::JsonSchema { schema, .. } => Some(GrammarSpec::JsonSchema { + schema: schema.to_string(), + }), } } else if tools_active && state.behavior.disable_tool_grammar { // Structure-snowballing escape hatch (arXiv:2604.06066): this @@ -291,7 +293,7 @@ pub(super) fn build_sampling( ); } let parser = state.tool_call_parser.as_ref().map(std::sync::Arc::clone); - let mut tools = req.tools.as_ref().cloned().unwrap_or_default(); + let mut tools = req.tools.clone(); if let Some(tool_parser::ToolChoice::Specific { ref function }) = req.tool_choice { tools.retain(|t| t.function.name == function.name); } @@ -305,14 +307,15 @@ pub(super) fn build_sampling( }; // Timeout deadline. - let timeout_secs = req.timeout.unwrap_or(state.request_timeout as f32); + let timeout_secs = req.timeout_secs.unwrap_or(state.request_timeout as f32); let timeout_at = if timeout_secs > 0.0 { Some(std::time::Instant::now() + std::time::Duration::from_secs_f32(timeout_secs)) } else { None }; - let top_logprobs = resolve_top_logprobs(req.logprobs, req.top_logprobs); + // Pre-resolved from the wire's logprobs/top_logprobs pair at the edge. + let top_logprobs = req.top_logprobs; Ok(SamplingSetup { temperature, @@ -337,41 +340,13 @@ pub(super) fn build_sampling( }) } -/// Resolve chat logprobs params (OpenAI spec): an explicit -/// `top_logprobs` count wins (clamped 0-20); `logprobs: true` alone -/// enables sampled-token logprobs with no alternatives (count 0); -/// otherwise disabled. -pub(crate) fn resolve_top_logprobs(logprobs: Option, top_logprobs: Option) -> Option { - match (logprobs, top_logprobs) { - (_, Some(n)) => Some(n.min(20)), - (Some(true), None) => Some(0), - _ => None, - } -} +// `resolve_top_logprobs` moved to the OpenAI edge (`openai/to_ir.rs`) +// — the envelope carries the already-resolved count. #[cfg(test)] mod tests { - use super::resolve_top_logprobs; use super::tool_choice_required_for_parser; - #[test] - fn logprobs_true_alone_enables_with_zero_top() { - assert_eq!(resolve_top_logprobs(Some(true), None), Some(0)); - } - - #[test] - fn explicit_top_logprobs_wins_and_clamps() { - assert_eq!(resolve_top_logprobs(None, Some(5)), Some(5)); - assert_eq!(resolve_top_logprobs(Some(false), Some(3)), Some(3)); - assert_eq!(resolve_top_logprobs(Some(true), Some(99)), Some(20)); - } - - #[test] - fn absent_or_false_disables() { - assert_eq!(resolve_top_logprobs(None, None), None); - assert_eq!(resolve_top_logprobs(Some(false), None), None); - } - use crate::tool_parser::{ToolChoice, ToolChoiceFunction}; #[test] diff --git a/crates/spark-server/src/api/chat/template.rs b/crates/spark-server/src/api/chat/template.rs index 0edebba6d..2ab0a3092 100644 --- a/crates/spark-server/src/api/chat/template.rs +++ b/crates/spark-server/src/api/chat/template.rs @@ -11,7 +11,6 @@ use axum::response::Response; use std::sync::Arc; use crate::AppState; -use crate::openai::ChatCompletionRequest; use super::super::compact::{compact_messages, openai_error_response}; use super::msg_entry::MsgEntry; @@ -29,7 +28,7 @@ pub(super) struct TemplateOut { #[allow(clippy::result_large_err)] pub(super) fn render_template( state: &Arc, - req: &ChatCompletionRequest, + tools: &[crate::tool_parser::ToolDefinition], messages: &[MsgEntry], image_pad_counts: &[usize], enable_thinking: bool, @@ -40,48 +39,7 @@ pub(super) fn render_template( let template_thinking = enable_thinking; // Build JSON messages with structured tool_calls for Jinja. - let json_messages: Vec = messages - .iter() - .map(|m| { - let content_val = if m.image_count > 0 { - let mut items: Vec = Vec::with_capacity(m.image_count + 1); - for _ in 0..m.image_count { - items.push(serde_json::json!({"type": "image"})); - } - if !m.content.is_empty() { - items.push(serde_json::json!({"type": "text", "text": m.content})); - } - serde_json::Value::Array(items) - } else { - serde_json::Value::String(m.content.clone()) - }; - // OpenAI: `developer` is the successor of `system` (o-series - // clients send it). Most chat templates only know - // system/user/assistant/tool, so map it here; a template - // that rejects non-leading system messages was already - // rejecting non-leading system — pre-existing limitation. - let role = if m.role == "developer" { - "system" - } else { - m.role.as_str() - }; - let mut msg = serde_json::json!({"role": role, "content": content_val}); - if let Some(ref tcs) = m.tool_calls { - msg["tool_calls"] = serde_json::Value::Array(tcs.clone()); - } - // F1: forward historical reasoning trace to the Jinja - // template. The template at qwen3_5_moe.jinja:90-104 - // reads `message.reasoning_content` and rehydrates the - // `` block for the historical assistant turns. - // Without this, every historical assistant message - // rendered an empty `\n\n\n\n` wrapper - // (empty-think poisoning, → premature `<|im_end|>`). - if let Some(ref rc) = m.reasoning_content { - msg["reasoning_content"] = serde_json::Value::String(rc.clone()); - } - msg - }) - .collect(); + let json_messages = build_json_messages(messages); // When TSCG is enabled the parser's `system_prompt()` has already // placed the compact tool signatures into messages[0]; passing // `tools` to Jinja as well would re-render the full JSON schema and @@ -90,11 +48,12 @@ pub(super) fn render_template( // still come from `system_prompt()`. let jinja_tools: Option> = if tools_active && !crate::tscg::tscg_enabled() { - req.tools.as_ref().map(|ts| { - ts.iter() + Some( + tools + .iter() .map(|t| serde_json::to_value(t).unwrap_or_default()) - .collect() - }) + .collect(), + ) } else { None }; @@ -178,3 +137,205 @@ pub(super) fn render_template( thinking_budget, }) } + +/// Build the Jinja-facing JSON message array from the processed +/// [`MsgEntry`] vec. Pure (no tokenizer/state) so it can be +/// characterization-tested directly: +/// * `image_count == 0` → `content` is a plain string, +/// * `image_count > 0` → `content` is `[{type:image} * N, {type:text}]` +/// (text part omitted when empty), +/// * `tool_calls` / `reasoning_content` attached when present. +/// +/// Roles arrive canonical (`developer` → `system` normalization happens +/// at MsgEntry build time). +pub(super) fn build_json_messages(messages: &[MsgEntry]) -> Vec { + messages + .iter() + .map(|m| { + let content_val = if m.image_count > 0 { + let mut items: Vec = Vec::with_capacity(m.image_count + 1); + for _ in 0..m.image_count { + items.push(serde_json::json!({"type": "image"})); + } + if !m.content.is_empty() { + items.push(serde_json::json!({"type": "text", "text": m.content})); + } + serde_json::Value::Array(items) + } else { + serde_json::Value::String(m.content.clone()) + }; + // `developer` → `system` normalization happens upstream at + // MsgEntry build time (msg_entry.rs), so the role arrives + // canonical here. + let mut msg = serde_json::json!({"role": m.role, "content": content_val}); + if let Some(ref tcs) = m.tool_calls { + msg["tool_calls"] = serde_json::Value::Array(tcs.clone()); + } + // F1: forward historical reasoning trace to the Jinja + // template. The template at qwen3_5_moe.jinja:90-104 + // reads `message.reasoning_content` and rehydrates the + // `` block for the historical assistant turns. + // Without this, every historical assistant message + // rendered an empty `\n\n\n\n` wrapper + // (empty-think poisoning, → premature `<|im_end|>`). + if let Some(ref rc) = m.reasoning_content { + msg["reasoning_content"] = serde_json::Value::String(rc.clone()); + } + msg + }) + .collect() +} + +#[cfg(test)] +mod json_message_tests { + use super::MsgEntry; + use super::build_json_messages; + + fn entry(role: &str, content: &str, image_count: usize) -> MsgEntry { + MsgEntry { + role: role.to_string(), + content: content.to_string(), + tool_calls: None, + image_count, + reasoning_content: None, + } + } + + /// PROMPT-STABILITY GATE. Characterization golden for the full + /// `Vec` → `build_msg_entries` → `build_json_messages` + /// path — the exact JSON the Jinja chat template consumes. Any + /// change here changes rendered prompts, which breaks kv-cache + /// prefix reuse across requests of the same conversation. Update + /// ONLY for an intentional, documented behavior change. + #[test] + fn prompt_json_stability_gate() { + use crate::ir::message::{Reasoning, ToolCall}; + use crate::ir::{ContentPart, Message, Role}; + + fn text_msg(role: Role, t: &str) -> Message { + Message { + role, + content: vec![ContentPart::Text(t.into())], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + } + } + + let mut assistant = text_msg(Role::Assistant, "Sure."); + assistant.reasoning = Some(Reasoning { + text: "plan the verification".into(), + }); + assistant.tool_calls = vec![ + ToolCall { + id: "c1".into(), + name: "bash".into(), + arguments: serde_json::json!({"command": "cargo test"}), + }, + ToolCall { + id: "c2".into(), + name: "read".into(), + arguments: serde_json::json!({"path": "a.rs"}), + }, + ]; + let mut tool_result = text_msg(Role::Tool, "total 0"); + tool_result.tool_call_id = Some("c1".into()); + tool_result.name = Some("bash".into()); + + let msgs = vec![ + text_msg( + Role::System, + "You are helpful.\nworking directory: /tmp/proj", + ), + text_msg(Role::Other("developer".into()), "be terse"), + text_msg(Role::User, "run the tests"), + assistant, + tool_result, + text_msg(Role::User, "thanks"), + ]; + + let out = super::super::msg_entry::build_msg_entries(None, None, &msgs, true) + .expect("fixture builds"); + assert_eq!(out.cwd_hint.as_deref(), Some("/tmp/proj")); + let json = build_json_messages(&out.messages); + + let expected = serde_json::json!([ + { + "role": "system", + "content": "You are helpful.\nworking directory: /tmp/proj\n\nworking_directory: /tmp/proj\n" + }, + {"role": "system", "content": "be terse"}, + {"role": "user", "content": "run the tests"}, + { + "role": "assistant", + "content": "Sure.", + "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "bash", "arguments": {"command": "cargo test"}}}, + {"id": "c2", "type": "function", "function": {"name": "read", "arguments": {"path": "a.rs"}}} + ], + "reasoning_content": "plan the verification" + }, + {"role": "tool", "content": "total 0"}, + {"role": "user", "content": "thanks"} + ]); + assert_eq!( + serde_json::Value::Array(json.clone()), + expected, + "prompt JSON drifted — this breaks kv-cache prefix stability:\n{}", + serde_json::to_string_pretty(&json).unwrap() + ); + } + + #[test] + fn plain_text_message_serializes_to_string_content() { + let out = build_json_messages(&[entry("user", "hi", 0)]); + assert_eq!( + out, + vec![serde_json::json!({"role": "user", "content": "hi"})] + ); + } + + #[test] + fn images_expand_to_structured_content_array_with_text_last() { + let out = build_json_messages(&[entry("user", "look", 2)]); + assert_eq!( + out, + vec![serde_json::json!({ + "role": "user", + "content": [ + {"type": "image"}, + {"type": "image"}, + {"type": "text", "text": "look"} + ] + })] + ); + } + + #[test] + fn empty_text_with_images_omits_text_part() { + let out = build_json_messages(&[entry("user", "", 1)]); + assert_eq!( + out, + vec![serde_json::json!({"role": "user", "content": [{"type": "image"}]})] + ); + } + + #[test] + fn tool_calls_and_reasoning_are_attached() { + let mut e = entry("assistant", "", 0); + e.tool_calls = Some(vec![serde_json::json!({"id": "c1"})]); + e.reasoning_content = Some("because".to_string()); + let out = build_json_messages(&[e]); + assert_eq!( + out, + vec![serde_json::json!({ + "role": "assistant", + "content": "", + "tool_calls": [{"id": "c1"}], + "reasoning_content": "because" + })] + ); + } +} diff --git a/crates/spark-server/src/api/chat/thinking.rs b/crates/spark-server/src/api/chat/thinking.rs index 90782add8..75ff0b258 100644 --- a/crates/spark-server/src/api/chat/thinking.rs +++ b/crates/spark-server/src/api/chat/thinking.rs @@ -1,41 +1,76 @@ // SPDX-License-Identifier: AGPL-3.0-only // -// Resolve `(enable_thinking, thinking_budget)` for a single -// request. Precedence (highest wins): +// Resolve `(enable_thinking, thinking_budget)` for a single request +// from the neutral thinking directive. Precedence (highest wins): // 1. `--disable-thinking` CLI flag (forces OFF for every request) -// 2. Request body (`reasoning_effort`, `thinking.budget_tokens`, …) +// 2. The request directive (client channels resolved at the API edge, +// or the server-level default directive when the client is silent) // 3. MODEL.toml `[behavior].thinking_default` // -// Lifted out of `chat::chat_completions_inner` (wave 4g). +// Lifted out of `chat::chat_completions_inner` (wave 4g); flipped from +// the OpenAI wire request to `ir::ThinkingDirective` (IR migration). use std::sync::Arc; use crate::AppState; -use crate::openai::ChatCompletionRequest; +use crate::ir::ThinkingDirective; pub(super) fn resolve_thinking( state: &Arc, - req: &ChatCompletionRequest, + directive: ThinkingDirective, + max_tokens: u32, tools_active: bool, ) -> (bool, Option) { - if state.disable_thinking { + resolve( + directive, + Policy { + disable_thinking: state.disable_thinking, + model_default: state.behavior.thinking_default, + thinking_in_tools: state.behavior.thinking_in_tools, + max_thinking_budget: state.behavior.max_thinking_budget, + }, + max_tokens, + tools_active, + ) +} + +/// Server/model policy inputs, split from `AppState` so the resolution +/// core is a pure function. +struct Policy { + disable_thinking: bool, + model_default: bool, + thinking_in_tools: bool, + max_thinking_budget: u32, +} + +fn resolve( + directive: ThinkingDirective, + policy: Policy, + max_tokens: u32, + tools_active: bool, +) -> (bool, Option) { + if policy.disable_thinking { return (false, None); } - let (et, tb) = req.resolve_thinking(state.behavior.thinking_default); - let mt = req.max_tokens as u32; - let max_budget = state.behavior.max_thinking_budget; + let (et, tb) = match directive { + // No client/server directive → MODEL.toml decides. `None` budget + // defers to the per-model `max_thinking_budget` below rather than + // a conservative hardcoded default. + ThinkingDirective::Unspecified => (policy.model_default, None), + ThinkingDirective::Off => (false, None), + ThinkingDirective::On { budget } => (true, budget), + }; // `thinking_in_tools=false` is the MODEL.toml DEFAULT for tool- // active turns: it suppresses thinking when the client is silent. - let et = if tools_active - && !state.behavior.thinking_in_tools - && !req.thinking_explicitly_requested() - { + // An explicit directive (enabled OR disabled — including the + // server-level default directive) still wins. + let et = if tools_active && !policy.thinking_in_tools && !directive.is_explicit() { false } else { et }; let budget = if et { - let b = tb.unwrap_or(max_budget); + let b = tb.unwrap_or(policy.max_thinking_budget); // 2026-05-23 sweep: dropped the 70% special case for // `tools_active && thinking_in_tools` (previously 7/10, now // 9/10 uniformly). With `thinking_in_tools=true` as the @@ -46,10 +81,141 @@ pub(super) fn resolve_thinking( // tool args without crippling reasoning chains that now run // naturally after the F1 reflection-penalty removal. let safety_cap_pct = 9; - let max = ((mt * safety_cap_pct) / 10).max(1); + let max = ((max_tokens * safety_cap_pct) / 10).max(1); Some(b.min(max)) } else { None }; (et, budget) } + +#[cfg(test)] +mod tests { + use super::*; + + fn policy() -> Policy { + Policy { + disable_thinking: false, + model_default: false, + thinking_in_tools: true, + max_thinking_budget: 2048, + } + } + + #[test] + fn kill_switch_overrides_everything() { + let (et, tb) = resolve( + ThinkingDirective::On { budget: Some(512) }, + Policy { + disable_thinking: true, + ..policy() + }, + 4096, + false, + ); + assert!(!et); + assert!(tb.is_none()); + } + + #[test] + fn unspecified_falls_to_model_default() { + let (et, tb) = resolve( + ThinkingDirective::Unspecified, + Policy { + model_default: true, + ..policy() + }, + 4096, + false, + ); + assert!(et); + // Defers to max_thinking_budget, capped at 90% of max_tokens. + assert_eq!(tb, Some(2048)); + + let (et, tb) = resolve(ThinkingDirective::Unspecified, policy(), 4096, false); + assert!(!et); + assert!(tb.is_none()); + } + + #[test] + fn explicit_budget_capped_at_90_pct_of_max_tokens() { + let (et, tb) = resolve( + ThinkingDirective::On { budget: Some(4096) }, + policy(), + 1000, + false, + ); + assert!(et); + assert_eq!(tb, Some(900)); + } + + #[test] + fn budgetless_on_defers_to_model_cap() { + let (et, tb) = resolve( + ThinkingDirective::On { budget: None }, + policy(), + 4096, + false, + ); + assert!(et); + assert_eq!(tb, Some(2048)); + } + + #[test] + fn tools_suppression_only_when_client_silent() { + let no_tools_thinking = Policy { + model_default: true, + thinking_in_tools: false, + ..policy() + }; + // Silent client on a tool turn → suppressed. + let (et, _) = resolve( + ThinkingDirective::Unspecified, + Policy { + ..no_tools_thinking + }, + 4096, + true, + ); + assert!(!et); + // Explicit enable survives the suppression. + let (et, _) = resolve( + ThinkingDirective::On { budget: None }, + Policy { + model_default: true, + thinking_in_tools: false, + ..policy() + }, + 4096, + true, + ); + assert!(et); + // Explicit disable is likewise respected (no double negation). + let (et, _) = resolve( + ThinkingDirective::Off, + Policy { + model_default: true, + thinking_in_tools: false, + ..policy() + }, + 4096, + true, + ); + assert!(!et); + } + + #[test] + fn explicit_off_wins_over_model_default() { + let (et, tb) = resolve( + ThinkingDirective::Off, + Policy { + model_default: true, + ..policy() + }, + 4096, + false, + ); + assert!(!et); + assert!(tb.is_none()); + } +} diff --git a/crates/spark-server/src/api/chat_blocking.rs b/crates/spark-server/src/api/chat_blocking.rs index 46afd611b..24231aa68 100644 --- a/crates/spark-server/src/api/chat_blocking.rs +++ b/crates/spark-server/src/api/chat_blocking.rs @@ -14,7 +14,7 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; use crate::AppState; -use crate::openai::{ChatCompletionRequest, ChatCompletionResponse, Usage}; +use crate::ir; use crate::tool_parser; use super::compact::openai_error_response; @@ -23,9 +23,8 @@ use super::inference_types::{GrammarSpec, InferenceRequest}; pub(super) struct BlockingPathArgs { pub state: Arc, - pub req: ChatCompletionRequest, + pub req: crate::ir::ChatRequest, pub req_ctx: Option>, - pub dump_seq: Option, pub prompt_tokens: Vec, pub session_hash: u64, pub image_pixels: Vec<(Vec, usize, usize)>, @@ -56,12 +55,11 @@ pub(super) struct BlockingPathArgs { pub prompt_len: usize, } -pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { +pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> super::chat::ChatOutcome { let BlockingPathArgs { state, req, req_ctx, - dump_seq, prompt_tokens, session_hash, image_pixels, @@ -93,7 +91,7 @@ pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { } = args; let n = req.n.max(1); - let mut all_choices: Vec = Vec::with_capacity(n); + let mut all_choices: Vec = Vec::with_capacity(n); let mut total_completion_tokens = 0usize; let mut first_ttft = 0.0f64; let mut last_decode_time_ms = 0.0f64; @@ -133,7 +131,7 @@ pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { stop_tokens: stop_tokens.clone(), enable_thinking, thinking_budget, - repetition_detection: req.repetition_detection(), + repetition_detection: req.repetition_detection, require_tool_call: tool_choice_required, tools_present: tools_active, suppress_tool_call, @@ -149,27 +147,27 @@ pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { if state.request_tx.send(request).await.is_err() { crate::metrics::REQUESTS_ACTIVE.dec(); - return openai_error_response( + return super::chat::ChatOutcome::Http(openai_error_response( StatusCode::SERVICE_UNAVAILABLE, "Scheduler queue full".to_string(), - ); + )); } let response = match rx.await { Ok(Ok(r)) => r, Ok(Err(e)) => { crate::metrics::REQUESTS_ACTIVE.dec(); - return openai_error_response( + return super::chat::ChatOutcome::Http(openai_error_response( StatusCode::INTERNAL_SERVER_ERROR, format!("Inference error: {e}"), - ); + )); } Err(_) => { crate::metrics::REQUESTS_ACTIVE.dec(); - return openai_error_response( + return super::chat::ChatOutcome::Http(openai_error_response( StatusCode::INTERNAL_SERVER_ERROR, "Inference cancelled".to_string(), - ); + )); } }; @@ -187,9 +185,10 @@ pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { let (reasoning_content_i, output_text_i) = decode_response_text(&state, &response, enable_thinking); - let output_text_i = strip_stop_sequences(output_text_i, &req.stop); + let (output_text_i, matched_stop) = + super::inference_impl::strip_stop_sequences_matched(output_text_i, &req.stop); - let (message, finish_reason_i) = build_choice_message( + let mut choice = build_choice_message( &state, &req, &response, @@ -200,20 +199,15 @@ pub(super) async fn run_blocking_path(args: BlockingPathArgs) -> Response { choice_idx, ) .await; - - all_choices.push(crate::openai::ChatChoice { - index: choice_idx, - message, - finish_reason: finish_reason_i, - logprobs: build_logprobs(&state, &response), - }); + choice.index = choice_idx; + choice.matched_stop = matched_stop; + choice.logprobs = build_logprobs(&state, &response); + all_choices.push(choice); } finalize_response( state, - req, req_ctx, - dump_seq, all_choices, total_completion_tokens, first_ttft, @@ -282,23 +276,21 @@ fn decode_response_text( #[allow(clippy::too_many_arguments)] async fn build_choice_message( state: &AppState, - req: &ChatCompletionRequest, + req: &crate::ir::ChatRequest, response: &super::inference_types::InferenceResponse, reasoning_content_i: Option, output_text_i: String, tools_active: bool, cwd_hint: Option<&str>, choice_idx: usize, -) -> (crate::openai::ChatMessage, String) { +) -> ir::Choice { let _ = response; // currently only used for finish_reason.clone() below - let mut message = crate::openai::ChatMessage { - role: "assistant".to_string(), - reasoning_content: reasoning_content_i, - annotations: crate::citation::merged_annotations(&output_text_i), - refusal: None, - content: Some(output_text_i.clone()), - tool_calls: None, - }; + // Neutral locals — the wire annotations (URL citations) are derived + // at encode time by the surfaces that emit them. + let mut reasoning_content = reasoning_content_i; + let mut msg_content: Option = Some(output_text_i.clone()); + let mut msg_tool_calls: Option> = None; + let mut msg_refusal: Option = None; let mut finish_reason_i = response.finish_reason.clone(); if tools_active { @@ -318,7 +310,7 @@ async fn build_choice_message( // scrub the residual XML from the reasoning trace so it isn't // double-emitted to the client. let (hoisted_reasoning, hoisted_tool_calls): (Option, Vec<_>) = - if let Some(ref rc) = message.reasoning_content { + if let Some(ref rc) = reasoning_content { let (scrubbed, tcs) = tool_parser::parse_tool_calls(rc); (scrubbed, tcs) } else { @@ -329,13 +321,13 @@ async fn build_choice_message( "F7: hoisted {} tool-call(s) from inside block (would have been silently dropped)", hoisted_tool_calls.len() ); - message.reasoning_content = hoisted_reasoning; + reasoning_content = hoisted_reasoning; } let (content, parsed_tool_calls) = tool_parser::parse_tool_calls(&output_text_i); let mut tool_calls_i = hoisted_tool_calls; tool_calls_i.extend(parsed_tool_calls); if !tool_calls_i.is_empty() { - let tools_ref = req.tools.as_ref().cloned().unwrap_or_default(); + let tools_ref = req.tools.clone(); tool_parser::backfill_required_params(&mut tool_calls_i, &tools_ref); if state .tool_call_parser @@ -377,7 +369,7 @@ async fn build_choice_message( } c.trim().to_string() }); - message.content = content; + msg_content = content; if !validated.valid.is_empty() { for tc in &validated.valid { let p: String = tc.function.arguments.chars().take(120).collect(); @@ -385,7 +377,7 @@ async fn build_choice_message( tracing::info!("Tool call: {}({p}{s})", tc.function.name); crate::metrics::TOOL_CALLS_TOTAL.inc(); } - message.tool_calls = Some(validated.valid); + msg_tool_calls = Some(validated.valid); finish_reason_i = "tool_calls".to_string(); } } @@ -393,44 +385,62 @@ async fn build_choice_message( // Refusal classifier: when the model's assistant text opens with // a known refusal pattern AND no tool call fired, populate - // `message.refusal` and null out `content` per the OpenAI spec. - if message.tool_calls.is_none() - && let Some(content_text) = message.content.as_deref() + // `refusal` and null out `content` per the OpenAI spec. + if msg_tool_calls.is_none() + && let Some(content_text) = msg_content.as_deref() && let Some(refusal_sentence) = crate::refusal::detect(content_text) { - message.refusal = Some(refusal_sentence); - message.content = None; - message.annotations = None; + msg_refusal = Some(refusal_sentence); + msg_content = None; } - (message, finish_reason_i) + // Validated wire tool calls → IR (arguments are serde-normalized + // strings from the parser, so the parse here is lossless). + let tool_calls: Vec = msg_tool_calls + .unwrap_or_default() + .into_iter() + .map(|tc| ir::message::ToolCall { + id: tc.id, + name: tc.function.name, + arguments: serde_json::from_str(&tc.function.arguments) + .unwrap_or_else(|_| serde_json::Value::Object(Default::default())), + }) + .collect(); + + ir::Choice { + index: choice_idx, + content: msg_content, + reasoning: reasoning_content, + tool_calls, + refusal: msg_refusal, + finish_reason: ir::FinishReason::from_wire(&finish_reason_i), + matched_stop: None, // caller fills + logprobs: None, // caller fills + } } /// Convert internal logprobs to OpenAI `ChoiceLogprobs` format. fn build_logprobs( state: &AppState, response: &super::inference_types::InferenceResponse, -) -> Option { +) -> Option { if response.logprobs.is_empty() { return None; } - Some(crate::openai::ChoiceLogprobs { + Some(ir::ChoiceLogprobs { content: response .logprobs .iter() .map(|lp| { let token_str = state.tokenizer.decode(&[lp.token_id]).unwrap_or_default(); - crate::openai::TokenLogprobInfo { + ir::TokenLogprob { token: token_str, logprob: lp.logprob, - bytes: None, - top_logprobs: lp + top: lp .top .iter() - .map(|&(tid, lp_val)| crate::openai::TopLogprob { - token: state.tokenizer.decode(&[tid]).unwrap_or_default(), - logprob: lp_val, - bytes: None, + .map(|&(tid, lp_val)| { + (state.tokenizer.decode(&[tid]).unwrap_or_default(), lp_val) }) .collect(), } @@ -439,83 +449,40 @@ fn build_logprobs( }) } -/// Build the final `ChatCompletionResponse` plus metrics, store, and -/// rate-limit refund. Returns the JSON-encoded HTTP response. +/// Core finalization: usage assembly, metrics, and the rate-limit +/// true-up. Returns the canonical response IR — wire encoding (plus +/// `store:`/`--dump` handling) happens in the per-surface encoders. #[allow(clippy::too_many_arguments)] fn finalize_response( state: Arc, - req: ChatCompletionRequest, req_ctx: Option>, - dump_seq: Option, - all_choices: Vec, + all_choices: Vec, total_completion_tokens: usize, first_ttft: f64, last_decode_time_ms: f64, total_reasoning_tokens: u32, total_cached_prompt_tokens: u32, prompt_len: usize, -) -> Response { +) -> super::chat::ChatOutcome { let tokens_per_second = if last_decode_time_ms > 0.0 && total_completion_tokens > 0 { (total_completion_tokens.saturating_sub(1)) as f64 / (last_decode_time_ms / 1000.0) } else { 0.0 }; - let usage = Usage { + let usage = ir::Usage { prompt_tokens: prompt_len, completion_tokens: total_completion_tokens, - total_tokens: prompt_len + total_completion_tokens, - prompt_tokens_details: Some(crate::openai::PromptTokensDetails { - cached_tokens: total_cached_prompt_tokens as usize, - audio_tokens: 0, - }), - completion_tokens_details: Some(crate::openai::CompletionTokensDetails { - reasoning_tokens: total_reasoning_tokens as usize, - audio_tokens: 0, - accepted_prediction_tokens: 0, - rejected_prediction_tokens: 0, - }), + cached_prompt_tokens: total_cached_prompt_tokens as usize, + reasoning_tokens: total_reasoning_tokens as usize, time_to_first_token_ms: first_ttft, response_tokens_per_second: tokens_per_second, }; - let completion_id = format!("chatcmpl-{}", crate::openai::uuid_v4()); - let created_at = crate::openai::unix_timestamp(); - let completion = ChatCompletionResponse { - id: completion_id.clone(), - object: "chat.completion".to_string(), - created: created_at, - model: state.model_name.clone(), - system_fingerprint: Some("fp_atlas".to_string()), - choices: all_choices, - usage: usage.clone(), - service_tier: req.service_tier.clone(), - metadata: req.metadata.clone(), - }; - crate::metrics::REQUESTS_ACTIVE.dec(); crate::metrics::PROMPT_TOKENS_TOTAL.inc_by(prompt_len as u64); crate::metrics::GENERATION_TOKENS_TOTAL.inc_by(total_completion_tokens as u64); crate::metrics::TTFT_SECONDS.observe(first_ttft / 1000.0); - // Completion-storage backend: when `store: true`, persist the - // serialized body so a subsequent GET /v1/chat/completions/{id} - // can return it. Bounded LRU + TTL in response_store. - if req.store.unwrap_or(false) - && let Ok(body) = serde_json::to_value(&completion) - { - state - .response_store - .insert(crate::response_store::StoredEntry { - id: completion_id, - kind: crate::response_store::StoredKind::ChatCompletion, - model: state.model_name.clone(), - created_at, - messages: Vec::new(), - body, - last_access: std::time::Instant::now(), - }); - } - // Rate-limit true-up. Middleware admitted with a conservative // reservation of `max_seq_len` tokens; refund the difference. if let Some(axum::extract::Extension(ref ctx)) = req_ctx { @@ -526,11 +493,11 @@ fn finalize_response( } } - // --dump: record the non-streaming response body, correlated with - // the request via the shared seq number. - if let (Some(seq), Some(dump)) = (dump_seq, state.dump_writer.as_ref()) { - dump.dump_response("/v1/chat/completions", seq, &completion, false); - } - - Json(completion).into_response() + super::chat::ChatOutcome::Blocking(Box::new(ir::ChatResponse { + id: crate::ids::uuid_v4(), + model: state.model_name.clone(), + created: crate::ids::unix_timestamp(), + choices: all_choices, + usage, + })) } diff --git a/crates/spark-server/src/api/chat_phases.rs b/crates/spark-server/src/api/chat_phases.rs index 5bfa23d20..31ed6eda3 100644 --- a/crates/spark-server/src/api/chat_phases.rs +++ b/crates/spark-server/src/api/chat_phases.rs @@ -10,16 +10,16 @@ use axum::http::StatusCode; use axum::response::Response; -use crate::openai::ChatCompletionRequest; +use crate::ir::ChatRequest; use super::compact::{openai_error_response, openai_error_response_with_param}; -/// Validate the OpenAI input contract: messages length, max_tokens > 0, -/// temperature/top_p ranges, tool_choice mode/required compatibility. -/// Returns `Err(Response)` for fail-fast 400 paths so the caller can -/// `?` directly into a Response. +/// Validate the input contract on the IR envelope: messages length, +/// max_tokens > 0, temperature/top_p ranges, tool_choice mode/required +/// compatibility. Returns `Err(Response)` for fail-fast 400 paths so +/// the caller can `?` directly into a Response. #[allow(clippy::result_large_err)] -pub(super) fn validate_input(req: &ChatCompletionRequest) -> Result<(), Response> { +pub(super) fn validate_input(req: &ChatRequest) -> Result<(), Response> { if req.messages.is_empty() { return Err(openai_error_response_with_param( StatusCode::BAD_REQUEST, @@ -36,7 +36,7 @@ pub(super) fn validate_input(req: &ChatCompletionRequest) -> Result<(), Response None, )); } - if let Some(t) = req.temperature + if let Some(t) = req.sampling.temperature && (!(0.0..=2.0).contains(&t)) { return Err(openai_error_response_with_param( @@ -46,7 +46,7 @@ pub(super) fn validate_input(req: &ChatCompletionRequest) -> Result<(), Response None, )); } - if let Some(p) = req.top_p + if let Some(p) = req.sampling.top_p && (p <= 0.0 || p > 1.0) { return Err(openai_error_response_with_param( @@ -85,7 +85,7 @@ pub(super) fn validate_input(req: &ChatCompletionRequest) -> Result<(), Response ), )); } - if s == "required" && req.tools.as_ref().is_none_or(|t| t.is_empty()) { + if s == "required" && req.tools.is_empty() { return Err(openai_error_response( StatusCode::BAD_REQUEST, "tool_choice is 'required' but no tools were provided".into(), diff --git a/crates/spark-server/src/api/chat_stream/ctx.rs b/crates/spark-server/src/api/chat_stream/ctx.rs index 624e53608..9c1d0f9b3 100644 --- a/crates/spark-server/src/api/chat_stream/ctx.rs +++ b/crates/spark-server/src/api/chat_stream/ctx.rs @@ -47,7 +47,6 @@ pub(super) struct StreamCtx { /// (string → integer/boolean/array/object). True for qwen3_xml. pub(super) wants_typed_arguments: bool, pub(super) max_tool_calls_per_response: usize, - pub(super) req_stream_include_usage: bool, pub(super) req_return_token_ids: bool, pub(super) req_ctx: Option, pub(super) dump_seq: Option, diff --git a/crates/spark-server/src/api/chat_stream/handle_done.rs b/crates/spark-server/src/api/chat_stream/handle_done.rs index 3827ae2c9..d3312b4bc 100644 --- a/crates/spark-server/src/api/chat_stream/handle_done.rs +++ b/crates/spark-server/src/api/chat_stream/handle_done.rs @@ -3,9 +3,7 @@ // `StreamEvent::Done { ... }` arm of the streaming `flat_map` // closure (originally ~396 LoC). -use axum::response::sse::Event; - -use crate::openai::{ChatCompletionChunk, Usage}; +use crate::ir::StreamDelta; use crate::tool_parser; use super::super::sanitizer::sanitize_content_chunk; @@ -17,7 +15,7 @@ use super::tool_handlers::{ handle_tool_call_end, handle_tool_call_start, }; -type SseVec = Vec>; +type DeltaVec = Vec; #[allow(clippy::too_many_arguments)] pub(super) fn handle_done( @@ -29,8 +27,8 @@ pub(super) fn handle_done( decode_time_ms: f64, reasoning_tokens: u32, cached_prompt_tokens: u32, -) -> SseVec { - let mut sse_events: SseVec = Vec::new(); +) -> DeltaVec { + let mut deltas: DeltaVec = Vec::new(); // ── Stop-string hold-back flush ───────────────────────────────── // vLLM's `IncrementalDetokenizer` releases any bytes still in the @@ -62,34 +60,27 @@ pub(super) fn handle_done( &ctx.leak_markers, ); if !sanitized.is_empty() { - let chunk = ChatCompletionChunk::content_chunk( - &ctx.model, &ctx.id, sanitized, - ); - sse_events.push(Ok(Event::default() - .data(serde_json::to_string(&chunk).unwrap_or_default()))); + deltas.push(StreamDelta::Content { + text: sanitized, + token_ids: Vec::new(), + }); } } tool_parser::DetectorOutput::ToolCall(mut tc, tc_idx) => { - handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut sse_events); + handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallStart { id: tc_id, name, idx, } => { - handle_tool_call_start(state, ctx, tc_id, name, idx, &mut sse_events); + handle_tool_call_start(state, ctx, tc_id, name, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallDelta { args, idx } => { - handle_tool_call_delta(state, ctx, args, idx, &mut sse_events); + handle_tool_call_delta(state, ctx, args, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallArgsFragment { fragment, idx } => { - handle_tool_call_args_fragment( - state, - ctx, - fragment, - idx, - &mut sse_events, - ); + handle_tool_call_args_fragment(state, ctx, fragment, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallEnd { idx } => { handle_tool_call_end(state, ctx, idx); @@ -108,10 +99,10 @@ pub(super) fn handle_done( if state.refusal_scan_buf.len() < 16_384 { state.refusal_scan_buf.push_str(&sanitized); } - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, sanitized); - sse_events - .push(Ok(Event::default() - .data(serde_json::to_string(&chunk).unwrap_or_default()))); + deltas.push(StreamDelta::Content { + text: sanitized, + token_ids: Vec::new(), + }); } } } @@ -134,28 +125,27 @@ pub(super) fn handle_done( &ctx.leak_markers, ); if !sanitized.is_empty() { - let chunk = - ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, sanitized) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - sse_events.push(Ok(Event::default() - .data(serde_json::to_string(&chunk).unwrap_or_default()))); + deltas.push(StreamDelta::Content { + text: sanitized, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } } tool_parser::DetectorOutput::ToolCall(mut tc, tc_idx) => { - handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut sse_events); + handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallStart { id: tc_id, name, idx, } => { - handle_tool_call_start(state, ctx, tc_id, name, idx, &mut sse_events); + handle_tool_call_start(state, ctx, tc_id, name, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallDelta { args, idx } => { - handle_tool_call_delta(state, ctx, args, idx, &mut sse_events); + handle_tool_call_delta(state, ctx, args, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallArgsFragment { fragment, idx } => { - handle_tool_call_args_fragment(state, ctx, fragment, idx, &mut sse_events); + handle_tool_call_args_fragment(state, ctx, fragment, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallEnd { idx } => { handle_tool_call_end(state, ctx, idx); @@ -174,33 +164,24 @@ pub(super) fn handle_done( if state.refusal_scan_buf.len() < 16_384 { state.refusal_scan_buf.push_str(&tail); } - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, tail) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&chunk).unwrap_or_default()) - )); + deltas.push(StreamDelta::Content { + text: tail, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } - // ── Usage block ───────────────────────────────────────────────── + // ── Usage block (neutral IR; the wire encoder derives + // total_tokens and the details sub-objects from it) ─────────── let tps = if decode_time_ms > 0.0 { completion_tokens.saturating_sub(1) as f64 / (decode_time_ms / 1000.0) } else { 0.0 }; - let usage = Usage { + let usage = crate::ir::Usage { prompt_tokens: ctx.prompt_len, completion_tokens, - total_tokens: ctx.prompt_len + completion_tokens, - prompt_tokens_details: Some(crate::openai::PromptTokensDetails { - cached_tokens: cached_prompt_tokens as usize, - audio_tokens: 0, - }), - completion_tokens_details: Some(crate::openai::CompletionTokensDetails { - reasoning_tokens: reasoning_tokens as usize, - audio_tokens: 0, - accepted_prediction_tokens: 0, - rejected_prediction_tokens: 0, - }), + cached_prompt_tokens: cached_prompt_tokens as usize, + reasoning_tokens: reasoning_tokens as usize, time_to_first_token_ms, response_tokens_per_second: tps, }; @@ -231,31 +212,21 @@ pub(super) fn handle_done( None }; if let Some(ref r) = refusal_signal { - let chunk = ChatCompletionChunk::refusal_chunk(&ctx.model, &ctx.id, r.clone()); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); + deltas.push(StreamDelta::Refusal { text: r.clone() }); } - // Usage emission strategy. - let emit_separate_usage = ctx.req_stream_include_usage; - let usage_for_dump = usage.clone(); - if emit_separate_usage { - let usage_chunk = ChatCompletionChunk::usage_only_chunk(&ctx.model, &ctx.id, usage.clone()); - let json = serde_json::to_string(&usage_chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); - // Residual: tokens whose decoded text was buffered/suppressed - // and never rode a content chunk. Stamping them here keeps - // Σ token_ids == completion_tokens exactly. - let final_chunk = ChatCompletionChunk::final_chunk_no_usage(&ctx.model, &ctx.id, fr) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&final_chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); - } else { - let chunk = ChatCompletionChunk::done_chunk(&ctx.model, &ctx.id, fr, usage) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); - } + // Terminal delta: finish reason + usage. The `include_usage` + // two-chunk framing (usage-only chunk before a usage-less finish + // chunk) is the OpenAI encoder's decision + // (`openai::delta_to_chunk_events`), not the core's. Residual + // token ids — tokens whose decoded text was buffered/suppressed + // and never rode a content delta — ride the Finish delta so + // Σ token_ids == completion_tokens exactly. + deltas.push(StreamDelta::Finish { + reason: crate::ir::FinishReason::from_wire(fr), + usage, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); // Metrics. crate::metrics::REQUESTS_ACTIVE.dec(); @@ -272,9 +243,28 @@ pub(super) fn handle_done( } } - // --dump synthesized response entry. + // --dump synthesized response entry. Diagnostics, not the stream: + // the dump keeps the OpenAI wire-usage shape (same numbers the + // encoder derives for the terminal chunk). if let (Some(seq), Some(dump)) = (ctx.dump_seq, ctx.state.dump_writer.as_ref()) { let has_tool_calls = state.detector.as_ref().is_some_and(|d| d.has_tool_calls()); + let usage_for_dump = crate::openai::Usage { + prompt_tokens: usage.prompt_tokens, + completion_tokens: usage.completion_tokens, + total_tokens: usage.prompt_tokens + usage.completion_tokens, + prompt_tokens_details: Some(crate::openai::PromptTokensDetails { + cached_tokens: usage.cached_prompt_tokens, + audio_tokens: 0, + }), + completion_tokens_details: Some(crate::openai::CompletionTokensDetails { + reasoning_tokens: usage.reasoning_tokens, + audio_tokens: 0, + accepted_prediction_tokens: 0, + rejected_prediction_tokens: 0, + }), + time_to_first_token_ms: usage.time_to_first_token_ms, + response_tokens_per_second: usage.response_tokens_per_second, + }; let body = serde_json::json!({ "id": ctx.id, "model": ctx.model, @@ -292,5 +282,5 @@ pub(super) fn handle_done( dump.dump_response("/v1/chat/completions", seq, &body, true); } - sse_events + deltas } diff --git a/crates/spark-server/src/api/chat_stream/handle_error.rs b/crates/spark-server/src/api/chat_stream/handle_error.rs index 122b24c78..f8c035aea 100644 --- a/crates/spark-server/src/api/chat_stream/handle_error.rs +++ b/crates/spark-server/src/api/chat_stream/handle_error.rs @@ -2,13 +2,13 @@ // // `StreamEvent::Error(msg)` arm of the streaming `flat_map` closure. -use axum::response::sse::Event; +use crate::ir::StreamDelta; use super::ctx::StreamCtx; -type SseVec = Vec>; +type DeltaVec = Vec; -pub(super) fn handle_error(ctx: &StreamCtx, msg: String) -> SseVec { +pub(super) fn handle_error(ctx: &StreamCtx, msg: String) -> DeltaVec { crate::metrics::REQUESTS_ACTIVE.dec(); // Abandoned stream — refund the full reservation. if let Some(ref rctx) = ctx.req_ctx { @@ -16,9 +16,12 @@ pub(super) fn handle_error(ctx: &StreamCtx, msg: String) -> SseVec { .rate_limiter .refund_tokens(&rctx.identity, rctx.reserved_tokens); } + // Wire-ready OpenAI error envelope; the encoder forwards it + // verbatim as SSE data. let err = serde_json::json!({ "error": {"message": msg, "type": "server_error", "code": 500} }); - let events: SseVec = vec![Ok(Event::default().data(err.to_string()))]; - events + vec![StreamDelta::Error { + message: err.to_string(), + }] } diff --git a/crates/spark-server/src/api/chat_stream/handle_token.rs b/crates/spark-server/src/api/chat_stream/handle_token.rs index 80b0f1d3d..76b6ac062 100644 --- a/crates/spark-server/src/api/chat_stream/handle_token.rs +++ b/crates/spark-server/src/api/chat_stream/handle_token.rs @@ -4,13 +4,11 @@ // streaming `flat_map` closure (originally ~672 LoC at the top of the // `chat_stream::chat_completions_stream` body). // -// Returns the SSE events produced for this single token. Callers -// invoke `futures::stream::iter(...)` on the result to feed the -// `flat_map` output stream. +// Returns the neutral stream deltas produced for this single token. +// The caller encodes them into wire SSE events at the surface seam +// (`openai::delta_to_chunk_events`). -use axum::response::sse::Event; - -use crate::openai::ChatCompletionChunk; +use crate::ir::StreamDelta; use crate::tool_parser; use super::super::sanitizer::sanitize_content_chunk; @@ -25,7 +23,7 @@ use super::tool_handlers::{ handle_tool_call_end, handle_tool_call_start, }; -type SseVec = Vec>; +type DeltaVec = Vec; /// Maximum consecutive tokens the stream may spend with /// `state.suppressing_param_leak == true` (sanitizer holding content @@ -67,7 +65,7 @@ pub(super) fn strip_bare_role_literal(delta: &mut String, inside_tool_call: bool } } -/// Process one token. Returns the SSE events to forward to the +/// Process one token. Returns the stream deltas to forward to the /// client (empty `Vec` is valid). /// /// Thin wrapper around [`handle_token_inner`] that runs the @@ -78,7 +76,7 @@ pub(super) fn strip_bare_role_literal(delta: &mut String, inside_tool_call: bool /// at the end of the body would only fire when the natural fall- /// through is taken, leaving the doom-loop case (long suppressed /// stream of orphan `` openers) uncaught. -pub(super) fn handle_token(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> SseVec { +pub(super) fn handle_token(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> DeltaVec { let result = handle_token_inner(state, ctx, tok); // Orphan-suppression streak watchdog. The sanitizer flips @@ -107,8 +105,8 @@ pub(super) fn handle_token(state: &mut StreamState, ctx: &StreamCtx, tok: u32) - result } -fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> SseVec { - let mut sse_events: SseVec = Vec::new(); +fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> DeltaVec { + let mut deltas: DeltaVec = Vec::new(); state.all_toks.push(tok); // One push per call == one sampled token == one increment of // `usage.completion_tokens`. Drained onto the next client-visible @@ -142,14 +140,10 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse // are legitimate `\n ` indents that the model emitted; // dropping them would lose chars permanently. if !residual.is_empty() { - let chunk = ChatCompletionChunk::reasoning_chunk( - &ctx.model, - &ctx.id, - residual.to_string(), - ) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); + deltas.push(StreamDelta::Reasoning { + text: residual.to_string(), + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } } } @@ -163,9 +157,10 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse // Whitespace-only tail can be a real trailing `\n ` indent // — emit anything non-empty so byte boundaries align. if !tail.is_empty() { - let chunk = ChatCompletionChunk::reasoning_chunk(&ctx.model, &ctx.id, tail); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(axum::response::sse::Event::default().data(json))); + deltas.push(StreamDelta::Reasoning { + text: tail, + token_ids: Vec::new(), + }); } } // Reset tool detector to clear any thinking-era tag fragments. @@ -177,7 +172,7 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse state.content_decoded.clear(); state.detok_prefix_offset = 0; state.detok_read_offset = 0; - return sse_events; + return deltas; } // Still in thinking — accumulate but don't emit as content if ctx.enable_thinking { @@ -189,7 +184,7 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse // guard catches the in-flight token race so the next // opener never reaches the client. if state.reasoning_xml_leak_detected { - return sse_events; + return deltas; } // Open thinking: emit as reasoning_content. Incrementally extend // the stable decoded text instead of re-decoding all_toks (O(n²)). @@ -302,7 +297,7 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse tail = %tail, "in-think tool-call leak detected; cancelling sequence (finish_reason will be \"length\")" ); - return sse_events; + return deltas; } } // F19: final structured sanitisation pass catches @@ -325,14 +320,14 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse // non-streaming response on temp=0 seed=42 (live A/B // 2026-05-25). Drop only TRULY empty chunks. if !cleaned.is_empty() { - let chunk = ChatCompletionChunk::reasoning_chunk(&ctx.model, &ctx.id, cleaned) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); + deltas.push(StreamDelta::Reasoning { + text: cleaned, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } } } - return sse_events; + return deltas; } // ── Content phase: full-decode + slice (matches reasoning path) ── @@ -373,7 +368,7 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse state.emitted = stable_end; raw } else { - return sse_events; + return deltas; }; // Retire the lazy `content_decoder` field — kept in StreamState // only to avoid a wider state-struct migration. The HF decoder is @@ -418,7 +413,7 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse } if delta.is_empty() { - return sse_events; + return deltas; } // Multi-token stop sequences via string matching, with a vLLM-style @@ -439,18 +434,18 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse // Either everything is sitting in the hold-back window // (waiting for the next chunk / stream close) or a match // already truncated the emittable bytes to nothing. - return sse_events; + return deltas; } } if state.stop_string_triggered { if !delta.is_empty() { - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, delta) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - sse_events.push(Ok(Event::default().data(json))); + deltas.push(StreamDelta::Content { + text: delta, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } - return sse_events; + return deltas; } // Fork: detector-active vs pure-content path. @@ -466,25 +461,25 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse match output { tool_parser::DetectorOutput::Content(text) => { if let Some(events_out) = detector_content_arm(state, ctx, &text) { - sse_events.extend(events_out); - return sse_events; + deltas.extend(events_out); + return deltas; } } tool_parser::DetectorOutput::ToolCall(mut tc, tc_idx) => { - handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut sse_events); + handle_complete_tool_call(state, ctx, &mut tc, tc_idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallStart { id: tc_id, name, idx, } => { - handle_tool_call_start(state, ctx, tc_id, name, idx, &mut sse_events); + handle_tool_call_start(state, ctx, tc_id, name, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallDelta { args, idx } => { - handle_tool_call_delta(state, ctx, args, idx, &mut sse_events); + handle_tool_call_delta(state, ctx, args, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallArgsFragment { fragment, idx } => { - handle_tool_call_args_fragment(state, ctx, fragment, idx, &mut sse_events); + handle_tool_call_args_fragment(state, ctx, fragment, idx, &mut deltas); } tool_parser::DetectorOutput::ToolCallEnd { idx } => { handle_tool_call_end(state, ctx, idx); @@ -500,20 +495,20 @@ fn handle_token_inner(state: &mut StreamState, ctx: &StreamCtx, tok: u32) -> Sse &ctx.leak_markers, ); if let Some(events_out) = process_detector_content(state, ctx, &sanitized) { - sse_events.extend(events_out); - return sse_events; + deltas.extend(events_out); + return deltas; } // process_detector_content does NOT pre-sanitize when called // from the no-detector branch — but the sanitizer was already // run above, so the helper's branch handling matches. } - sse_events + deltas } /// Common processing for a sanitized content chunk: SimHash semantic /// guard, token-level loop watchdog, salvage on trip, otherwise -/// emit a `content_chunk`. Returns `Some(events)` when the watchdog +/// emit a `Content` delta. Returns `Some(deltas)` when the watchdog /// fired (caller must short-circuit), else `None` (caller continues). /// /// Note: when called from the detector-active branch, `sanitized` @@ -524,7 +519,7 @@ fn process_detector_content( state: &mut StreamState, ctx: &StreamCtx, sanitized_or_raw: &str, -) -> Option { +) -> Option { // From the detector-active branch the input is the Content(text) // payload that still needs sanitization. From the no-detector // branch the input is already sanitized. Distinguish via a thin @@ -580,25 +575,25 @@ fn process_detector_content( // Watchdog fired: short-circuit the stream with no further // content. The model emitted a degenerate loop; we end the // response here rather than salvaging a synthetic tool call. - return Some(SseVec::new()); + return Some(DeltaVec::new()); } if !sanitized.is_empty() { if state.refusal_scan_buf.len() < 16_384 { state.refusal_scan_buf.push_str(sanitized); } - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, sanitized.to_string()) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - let json = serde_json::to_string(&chunk).unwrap_or_default(); - let events: SseVec = vec![Ok(Event::default().data(json))]; - return Some(events); + let out: DeltaVec = vec![StreamDelta::Content { + text: sanitized.to_string(), + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }]; + return Some(out); } None } /// Detector-active branch's `Content(text)` arm: sanitize first, /// then run the shared semantic/token watchdog + emit pipeline. -fn detector_content_arm(state: &mut StreamState, ctx: &StreamCtx, text: &str) -> Option { +fn detector_content_arm(state: &mut StreamState, ctx: &StreamCtx, text: &str) -> Option { let sanitized = sanitize_content_chunk( text, &mut state.tag_scan_buf, diff --git a/crates/spark-server/src/api/chat_stream/mod.rs b/crates/spark-server/src/api/chat_stream/mod.rs index 8c9543552..3f75f3138 100644 --- a/crates/spark-server/src/api/chat_stream/mod.rs +++ b/crates/spark-server/src/api/chat_stream/mod.rs @@ -36,7 +36,6 @@ use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; use crate::AppState; -use crate::openai::ChatCompletionChunk; use crate::tool_parser; use super::inference_types::{GrammarSpec, InferenceRequest, StreamEvent}; @@ -45,7 +44,7 @@ use ctx::StreamCtx; use state::StreamState; #[allow(clippy::too_many_arguments)] -pub(crate) async fn chat_completions_stream( +pub(crate) async fn run_chat_stream( state: Arc, prompt_tokens: Vec, session_hash: u64, @@ -67,7 +66,7 @@ pub(crate) async fn chat_completions_stream( logit_bias: Vec<(u32, f32)>, enable_thinking: bool, thinking_budget: Option, - repetition_detection: Option, + repetition_detection: Option, tools_active: bool, tool_choice_required: bool, suppress_tool_call: bool, @@ -79,18 +78,10 @@ pub(crate) async fn chat_completions_stream( top_logprobs: Option, timeout_at: Option, stop_strings: Vec, - req_stream_include_usage: bool, req_return_token_ids: bool, - req_service_tier: Option, - req_metadata: Option>, req_ctx: Option, dump_seq: Option, -) -> Result { - // service_tier + metadata are request echoes only; the chat-completion- - // chunk schema doesn't carry them, but we surface them via the final - // usage block when `include_usage=true`. Accept and retain for future - // wiring — see gap #18/#19 in the OpenAI compat plan. - let _ = (&req_service_tier, &req_metadata); +) -> Result { // Channel capacity sized for ~30s of decode at 50 tok/s. The previous // 64-slot buffer would fill in <2s under any HTTP-flush stall and silently // drop the seq via emit_step.rs's `try_send().is_err()` (now fixed to @@ -161,13 +152,11 @@ pub(crate) async fn chat_completions_stream( ) })?; - let chunk_id = crate::openai::new_chunk_id(); + // `ctx.id`/`ctx.model` feed the --dump synthesis; each surface + // encoder mints its own wire ids. + let chunk_id = format!("chatcmpl-{}", crate::ids::uuid_v4()); let model_name = state.model_name.clone(); - // First chunk: role announcement - let role_chunk = ChatCompletionChunk::role_chunk(&model_name, &chunk_id); - let role_json = serde_json::to_string(&role_chunk).unwrap_or_default(); - // Resolve the active parser's leak-marker vocabulary once, at request // setup. `'static` slices so we borrow by reference throughout the // stream without cloning. If no parser is active, the sanitizer runs @@ -216,7 +205,6 @@ pub(crate) async fn chat_completions_stream( .as_ref() .is_some_and(|p| p.wants_typed_arguments()), max_tool_calls_per_response, - req_stream_include_usage, req_return_token_ids, req_ctx, dump_seq, @@ -237,7 +225,10 @@ pub(crate) async fn chat_completions_stream( let token_stream = ReceiverStream::new(token_rx).flat_map(move |event| { use futures::StreamExt; - let events = match event { + // The handlers emit provider-neutral deltas (`ir::StreamDelta`); + // they never touch the wire format. Encoding happens in the + // caller's surface encoder. + let deltas = match event { StreamEvent::Token(tok) | StreamEvent::TokenWithLogprobs(tok, _) => { handle_token::handle_token(&mut stream_state, &ctx, tok) } @@ -265,17 +256,8 @@ pub(crate) async fn chat_completions_stream( StreamEvent::Error(msg) => handle_error::handle_error(&ctx, msg), }; - futures::stream::iter(events).boxed() - }); - - // Prepend role chunk, append [DONE] sentinel - let role_event = futures::stream::once(async move { Ok(Event::default().data(role_json)) }); - let done_event = futures::stream::once(async { - Ok::<_, std::convert::Infallible>(Event::default().data("[DONE]")) + futures::stream::iter(deltas).boxed() }); - let full_stream = role_event.chain(token_stream).chain(done_event); - Ok(Sse::new(full_stream) - .keep_alive(KeepAlive::default()) - .into_response()) + Ok(Box::pin(token_stream)) } diff --git a/crates/spark-server/src/api/chat_stream/state.rs b/crates/spark-server/src/api/chat_stream/state.rs index 42d277ab0..e576b5fa3 100644 --- a/crates/spark-server/src/api/chat_stream/state.rs +++ b/crates/spark-server/src/api/chat_stream/state.rs @@ -146,10 +146,10 @@ pub(super) struct StreamState { /// `true` when the request did not enable thinking. pub(super) thinking_done: bool, /// Dead after the tool-call retry stack was removed (`tool_retry_enabled` - /// is now constant `false`, so chunks are always streamed in real time + /// is now constant `false`, so deltas are always streamed in real time /// and this map stays empty). Retained so the buffering helpers in /// `tool_handlers.rs` still type-check. - pub(super) buffered_tool_chunks: std::collections::HashMap>, + pub(super) buffered_tool_chunks: std::collections::HashMap>, /// Dead after the tool-call retry stack was removed; never set now that /// `tool_retry_enabled` is constant `false`. pub(super) pending_retry: Option, diff --git a/crates/spark-server/src/api/chat_stream/tool_handlers.rs b/crates/spark-server/src/api/chat_stream/tool_handlers.rs index 5826a91b7..362e092fc 100644 --- a/crates/spark-server/src/api/chat_stream/tool_handlers.rs +++ b/crates/spark-server/src/api/chat_stream/tool_handlers.rs @@ -5,53 +5,49 @@ // stream `process()` outputs) and `handle_done` (end-of-stream // `flush()` outputs). -use axum::response::sse::Event; - -use crate::openai::ChatCompletionChunk; +use crate::ir::StreamDelta; use crate::tool_parser; use super::super::stream_guards::{bump_f12_tool_call_count, flush_content_sanitizer}; use super::ctx::StreamCtx; use super::state::{PendingRetry, StreamState}; -type SseVec = Vec>; +type DeltaVec = Vec; -/// Tier 5c (2026-05-26): emit `chunk_json` to either the client SSE -/// stream OR a per-tool-call-index buffer in `StreamState`. When tool -/// retry is enabled we hold all tool_call SSE chunks until -/// `handle_tool_call_delta` runs validation; on pass the buffered chunks -/// flush to the client, on fail they're discarded and the retry fires -/// at `handle_done`. When tool retry is disabled this is a direct emit -/// (preserves the existing real-time streaming behaviour). -fn emit_or_buffer_tool_chunk( +/// Tier 5c (2026-05-26): emit `delta` to either the client stream OR a +/// per-tool-call-index buffer in `StreamState`. When tool retry is +/// enabled we hold all tool_call deltas until `handle_tool_call_delta` +/// runs validation; on pass the buffered deltas flush to the client, on +/// fail they're discarded and the retry fires at `handle_done`. When +/// tool retry is disabled this is a direct emit (preserves the existing +/// real-time streaming behaviour). +fn emit_or_buffer_tool_delta( state: &mut StreamState, ctx: &StreamCtx, idx: usize, - chunk_json: String, - sse_events: &mut SseVec, + delta: StreamDelta, + deltas: &mut DeltaVec, ) { if ctx.tool_retry_enabled { state .buffered_tool_chunks .entry(idx) .or_default() - .push(chunk_json); + .push(delta); } else { - sse_events.push(Ok(Event::default().data(chunk_json))); + deltas.push(delta); } } -/// Flush all buffered SSE chunks for tool-call `idx` into `sse_events`. -/// No-op when retry is disabled (chunks were emitted directly). -fn flush_buffered_tool_chunks(state: &mut StreamState, idx: usize, sse_events: &mut SseVec) { +/// Flush all buffered deltas for tool-call `idx` into `deltas`. +/// No-op when retry is disabled (deltas were emitted directly). +fn flush_buffered_tool_chunks(state: &mut StreamState, idx: usize, deltas: &mut DeltaVec) { if let Some(chunks) = state.buffered_tool_chunks.remove(&idx) { - for chunk_json in chunks { - sse_events.push(Ok(Event::default().data(chunk_json))); - } + deltas.extend(chunks); } } -/// Drop all buffered SSE chunks for tool-call `idx` without emitting. +/// Drop all buffered deltas for tool-call `idx` without emitting. /// Called when validation fails and we're going to fire a Tier 5c retry. fn drop_buffered_tool_chunks(state: &mut StreamState, idx: usize) { state.buffered_tool_chunks.remove(&idx); @@ -63,7 +59,7 @@ pub(super) fn handle_complete_tool_call( ctx: &StreamCtx, tc: &mut tool_parser::ToolCall, tc_idx: usize, - sse_events: &mut SseVec, + deltas: &mut DeltaVec, ) { // Content → Tool boundary: flush sanitiser tail. let pre_tool_tail = flush_content_sanitizer( @@ -72,11 +68,10 @@ pub(super) fn handle_complete_tool_call( &ctx.leak_markers, ); if !pre_tool_tail.is_empty() { - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, pre_tool_tail) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&chunk).unwrap_or_default()) - )); + deltas.push(StreamDelta::Content { + text: pre_tool_tail, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } tool_parser::backfill_required_params(std::slice::from_mut(tc), &ctx.tool_defs_for_backfill); if ctx.wants_typed_arguments { @@ -107,11 +102,10 @@ pub(super) fn handle_complete_tool_call( "tool call validation error (hard): {e}; replacing with content and ending" ); let msg = format!("[atlas] Tool call rejected: {e}"); - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, msg) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&chunk).unwrap_or_default()) - )); + deltas.push(StreamDelta::Content { + text: msg, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); state.stop_string_triggered = true; } else if let Err((_, e)) = &validation { // Soft validation error (missing param / empty required string) — @@ -135,19 +129,16 @@ pub(super) fn handle_complete_tool_call( }; tracing::info!("Tool call: {}({preview}{s})", tc.function.name); crate::metrics::TOOL_CALLS_TOTAL.inc(); - let start = ChatCompletionChunk::tool_call_start_chunk(&ctx.model, &ctx.id, tc, tc_idx); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&start).unwrap_or_default()) - )); - let frag = ChatCompletionChunk::tool_call_args_fragment( - &ctx.model, - &ctx.id, - tc_idx, - &tc.function.arguments, - ); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&frag).unwrap_or_default()) - )); + deltas.push(StreamDelta::ToolCallStart { + index: tc_idx, + id: tc.id.clone(), + name: tc.function.name.clone(), + }); + deltas.push(StreamDelta::ToolCallArgs { + index: tc_idx, + fragment: tc.function.arguments.clone(), + token_ids: Vec::new(), + }); } else if state .tool_arg_dedup .check(&tc.function.name, &tc.function.arguments) @@ -191,19 +182,16 @@ pub(super) fn handle_complete_tool_call( }; tracing::info!("Tool call: {}({preview}{s})", tc.function.name); crate::metrics::TOOL_CALLS_TOTAL.inc(); - let start = ChatCompletionChunk::tool_call_start_chunk(&ctx.model, &ctx.id, tc, tc_idx); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&start).unwrap_or_default()) - )); - let frag = ChatCompletionChunk::tool_call_args_fragment( - &ctx.model, - &ctx.id, - tc_idx, - &tc.function.arguments, - ); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&frag).unwrap_or_default()) - )); + deltas.push(StreamDelta::ToolCallStart { + index: tc_idx, + id: tc.id.clone(), + name: tc.function.name.clone(), + }); + deltas.push(StreamDelta::ToolCallArgs { + index: tc_idx, + fragment: tc.function.arguments.clone(), + token_ids: Vec::new(), + }); } } @@ -214,7 +202,7 @@ pub(super) fn handle_tool_call_start( tc_id: String, name: String, idx: usize, - sse_events: &mut SseVec, + deltas: &mut DeltaVec, ) { let pre_tool_tail = flush_content_sanitizer( &mut state.tag_scan_buf, @@ -222,31 +210,25 @@ pub(super) fn handle_tool_call_start( &ctx.leak_markers, ); if !pre_tool_tail.is_empty() { - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, pre_tool_tail) - .with_token_ids(state.take_ids_if(ctx.req_return_token_ids)); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&chunk).unwrap_or_default()) - )); + deltas.push(StreamDelta::Content { + text: pre_tool_tail, + token_ids: state.take_ids_if(ctx.req_return_token_ids), + }); } state .streaming_tool_args .insert(idx, (name.clone(), String::new())); - let tc = tool_parser::ToolCall { - id: tc_id, - call_type: "function".to_string(), - function: tool_parser::FunctionCall { - name, - arguments: String::new(), - }, - }; bump_f12_tool_call_count( &mut state.tool_calls_emitted_count, ctx.max_tool_calls_per_response, &mut state.stop_string_triggered, ); - let start = ChatCompletionChunk::tool_call_start_chunk(&ctx.model, &ctx.id, &tc, idx); - let start_json = serde_json::to_string(&start).unwrap_or_default(); - emit_or_buffer_tool_chunk(state, ctx, idx, start_json, sse_events); + let start = StreamDelta::ToolCallStart { + index: idx, + id: tc_id, + name, + }; + emit_or_buffer_tool_delta(state, ctx, idx, start, deltas); } /// `DetectorOutput::ToolCallDelta` — incremental: append args. @@ -272,7 +254,7 @@ pub(super) fn handle_tool_call_delta( ctx: &StreamCtx, args: String, idx: usize, - sse_events: &mut SseVec, + deltas: &mut DeltaVec, ) { let mut emit_args = args.clone(); if let Some(entry) = state.streaming_tool_args.get_mut(&idx) { @@ -359,10 +341,10 @@ pub(super) fn handle_tool_call_delta( "tool call validation error (stream Δ, hard): {e}; replacing with content and ending" ); let msg = format!("[atlas] Tool call rejected: {e}"); - let chunk = ChatCompletionChunk::content_chunk(&ctx.model, &ctx.id, msg); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&chunk).unwrap_or_default()) - )); + deltas.push(StreamDelta::Content { + text: msg, + token_ids: Vec::new(), + }); state.stop_string_triggered = true; entry.1.push_str(&args); return; @@ -375,17 +357,19 @@ pub(super) fn handle_tool_call_delta( // No prior ToolCallStart for this idx — keep legacy passthrough. } if !emit_args.is_empty() { - let frag = - ChatCompletionChunk::tool_call_args_fragment(&ctx.model, &ctx.id, idx, &emit_args); - let frag_json = serde_json::to_string(&frag).unwrap_or_default(); - // Either flush previously-buffered start + this args chunk + let frag = StreamDelta::ToolCallArgs { + index: idx, + fragment: emit_args, + token_ids: Vec::new(), + }; + // Either flush previously-buffered start + this args delta // together (success path under retry), or emit directly (retry - // disabled). When retry is disabled the start chunk was already - // emitted in real time, so `emit_or_buffer_tool_chunk` just adds - // the args chunk. - emit_or_buffer_tool_chunk(state, ctx, idx, frag_json, sse_events); + // disabled). When retry is disabled the start delta was already + // emitted in real time, so `emit_or_buffer_tool_delta` just adds + // the args delta. + emit_or_buffer_tool_delta(state, ctx, idx, frag, deltas); if ctx.tool_retry_enabled { - flush_buffered_tool_chunks(state, idx, sse_events); + flush_buffered_tool_chunks(state, idx, deltas); } } } @@ -399,19 +383,20 @@ pub(super) fn handle_tool_call_delta( /// must precede its arguments). pub(super) fn handle_tool_call_args_fragment( state: &mut StreamState, - ctx: &StreamCtx, + _ctx: &StreamCtx, fragment: String, idx: usize, - sse_events: &mut SseVec, + deltas: &mut DeltaVec, ) { let Some(entry) = state.streaming_tool_args.get_mut(&idx) else { return; }; entry.1.push_str(&fragment); - let frag = ChatCompletionChunk::tool_call_args_fragment(&ctx.model, &ctx.id, idx, &fragment); - sse_events.push(Ok( - Event::default().data(serde_json::to_string(&frag).unwrap_or_default()) - )); + deltas.push(StreamDelta::ToolCallArgs { + index: idx, + fragment, + token_ids: Vec::new(), + }); } /// Advance the same-name run counter for a completed call, returning the new diff --git a/crates/spark-server/src/api/chat_stream_dispatch.rs b/crates/spark-server/src/api/chat_stream_dispatch.rs index 6c4089331..ebcb627e5 100644 --- a/crates/spark-server/src/api/chat_stream_dispatch.rs +++ b/crates/spark-server/src/api/chat_stream_dispatch.rs @@ -9,19 +9,18 @@ use std::sync::Arc; use axum::http::StatusCode; -use axum::response::Response; use crate::AppState; -use crate::openai::ChatCompletionRequest; +use crate::ir::ChatRequest; use crate::tool_parser; -use super::chat_stream::chat_completions_stream; +use super::chat_stream::run_chat_stream; use super::compact::openai_error_response; use super::inference_types::GrammarSpec; pub(super) async fn dispatch_streaming( state: Arc, - req: &ChatCompletionRequest, + req: &ChatRequest, req_ctx: Option>, dump_seq: Option, prompt_tokens: Vec, @@ -51,28 +50,25 @@ pub(super) async fn dispatch_streaming( grammar_spec: Option, top_logprobs: Option, timeout_at: Option, -) -> Response { +) -> super::chat::ChatOutcome { if req.n > 1 { crate::metrics::REQUESTS_ACTIVE.dec(); - return openai_error_response( + return super::chat::ChatOutcome::Http(openai_error_response( StatusCode::BAD_REQUEST, "n > 1 is not supported in streaming mode".to_string(), - ); + )); } - let tool_defs: Vec = req.tools.clone().unwrap_or_default(); + let tool_defs: Vec = req.tools.clone(); // Sort by length descending so the streaming stop-string scan // matches the longest overlapping prefix first (e.g. when the // caller provides [""], `find` would // otherwise truncate at the shorter, wrong boundary). let mut stop_strings = req.stop.clone(); stop_strings.sort_by_key(|s| std::cmp::Reverse(s.len())); - let stream_include_usage = req.stream_options.map(|o| o.include_usage).unwrap_or(false); let req_return_token_ids = req.return_token_ids; - let req_service_tier = req.service_tier.clone(); - let req_metadata = req.metadata.clone(); let ctx_for_stream = req_ctx.as_ref().map(|e| e.0.clone()); - let repetition_detection = req.repetition_detection(); - match chat_completions_stream( + let repetition_detection = req.repetition_detection; + match run_chat_stream( state, prompt_tokens, session_hash, @@ -106,16 +102,13 @@ pub(super) async fn dispatch_streaming( top_logprobs, timeout_at, stop_strings, - stream_include_usage, req_return_token_ids, - req_service_tier, - req_metadata, ctx_for_stream, dump_seq, ) .await { - Ok(r) => r, - Err((status, msg)) => openai_error_response(status, msg), + Ok(deltas) => super::chat::ChatOutcome::Streaming(deltas), + Err((status, msg)) => super::chat::ChatOutcome::Http(openai_error_response(status, msg)), } } diff --git a/crates/spark-server/src/api/completions.rs b/crates/spark-server/src/api/completions.rs index 6342a6bb8..5a9f18529 100644 --- a/crates/spark-server/src/api/completions.rs +++ b/crates/spark-server/src/api/completions.rs @@ -452,7 +452,7 @@ pub async fn list_models(State(state): State>) -> Json Option { + pub fn repetition_detection( + &self, + ) -> Option { match self { InferenceRequest::Blocking { repetition_detection, @@ -364,7 +366,17 @@ pub(crate) fn tokenize_stop_sequences( /// Strip any matching stop sequence from the end of the output text. /// Per OpenAI spec, returned text must not contain the stop sequence. -pub(crate) fn strip_stop_sequences(mut text: String, stops: &[String]) -> String { +pub(crate) fn strip_stop_sequences(text: String, stops: &[String]) -> String { + strip_stop_sequences_matched(text, stops).0 +} + +/// [`strip_stop_sequences`], also reporting WHICH stop sequence matched +/// (feeds Anthropic's `stop_sequence` response field via the IR's +/// `matched_stop`). +pub(crate) fn strip_stop_sequences_matched( + mut text: String, + stops: &[String], +) -> (String, Option) { // Try longest-first so overlapping prefixes (`[""]`) // don't truncate at the shorter (wrong) match boundary. strip_suffix // is end-anchored, so usually only one of the two end-matches at a @@ -374,10 +386,10 @@ pub(crate) fn strip_stop_sequences(mut text: String, stops: &[String]) -> String for s in sorted { if let Some(stripped) = text.strip_suffix(s.as_str()) { text.truncate(stripped.len()); - break; + return (text, Some(s.clone())); } } - text + (text, None) } /// Strip `...` reasoning content from model output. diff --git a/crates/spark-server/src/api/inference_types.rs b/crates/spark-server/src/api/inference_types.rs index b88f54d77..91e6398a2 100644 --- a/crates/spark-server/src/api/inference_types.rs +++ b/crates/spark-server/src/api/inference_types.rs @@ -18,10 +18,30 @@ use crate::openai::{ }; use crate::tool_parser; -// Re-export so scheduler / chat-stream code can refer to it via -// `crate::api::RepetitionDetectionParams` without depending directly on -// the `openai` module. Matches how `GrammarSpec` is plumbed. -pub use crate::openai::RepetitionDetectionParams; +/// Per-request override for the vLLM-anchored token-loop detector. +/// +/// Mirrors vLLM's `RepetitionDetectionParams` +/// (`vllm/sampling_params.py:111-144`): +/// - `min_pattern_size` → smallest pattern length (in tokens) to consider +/// - `max_pattern_size` → largest pattern length to consider +/// - `min_count` → number of end-anchored back-to-back repeats that +/// constitute a "loop" +/// +/// When attached to a request, these override the boot-global +/// thresholds (`CONTENT_LOOP_PERIOD_MIN` / `_MAX` / +/// `CONTENT_LOOP_MIN_REPEATS` and the thinking-loop equivalents) for +/// that single sequence's content-loop and thinking-loop detectors. +/// +/// Defined here (not in a wire module) because the scheduler and +/// chat-stream internals consume it; the API surfaces deserialize into +/// it at their edges. +#[derive(Debug, Clone, Copy, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct RepetitionDetectionParams { + pub min_pattern_size: u32, + pub max_pattern_size: u32, + pub min_count: u32, +} /// Grammar specification for constrained decoding. /// diff --git a/crates/spark-server/src/api/misc_handlers.rs b/crates/spark-server/src/api/misc_handlers.rs index 3d5c539cd..3353bb45d 100644 --- a/crates/spark-server/src/api/misc_handlers.rs +++ b/crates/spark-server/src/api/misc_handlers.rs @@ -11,12 +11,12 @@ use futures::StreamExt; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; -use super::chat_stream::chat_completions_stream; +use super::chat_stream::run_chat_stream; use super::responses_stream::responses_endpoint_stream; use super::responses_translate::{ build_responses_usage, emit, find_frame_end, translate_chat_response_to_responses, }; -use super::stored::extract_assistant_incoming_message; +use super::stored::assistant_incoming_from_ir; use crate::AppState; use crate::openai::{ ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, CompletionChunk, diff --git a/crates/spark-server/src/api/mod.rs b/crates/spark-server/src/api/mod.rs index da64d2946..65589cd62 100644 --- a/crates/spark-server/src/api/mod.rs +++ b/crates/spark-server/src/api/mod.rs @@ -60,6 +60,7 @@ mod tests; // `compact_messages` whose handler is wired directly in serve_router). pub use chat::chat_completions; pub(crate) use chat::chat_completions_inner; +pub(crate) use chat::{ChatOutcome, ResponseEcho}; #[allow(unused_imports)] pub use compact::compact_messages; pub use completions::{completions, embeddings_stub, get_model, list_models}; diff --git a/crates/spark-server/src/api/responses.rs b/crates/spark-server/src/api/responses.rs index 378340491..c191d895c 100644 --- a/crates/spark-server/src/api/responses.rs +++ b/crates/spark-server/src/api/responses.rs @@ -11,12 +11,12 @@ use futures::StreamExt; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; -use super::chat_stream::chat_completions_stream; +use super::chat_stream::run_chat_stream; use super::responses_stream::responses_endpoint_stream; use super::responses_translate::{ build_responses_usage, emit, find_frame_end, translate_chat_response_to_responses, }; -use super::stored::extract_assistant_incoming_message; +use super::stored::assistant_incoming_from_ir; use crate::AppState; use crate::openai::{ ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, CompletionChunk, @@ -144,7 +144,7 @@ pub async fn responses_endpoint( // request. Use the _inner variant because we already have a parsed // struct (no raw bytes available to dump at this layer; the Responses // handler dumps at its own entry point if --dump is enabled). - let resp = chat_completions_inner(state.0.clone(), None, chat_req, None).await; + let resp = chat_completions_inner(state.0.clone(), None, chat_req.into_ir(), None).await; let conv_pair = conversation_id.map(|cid| (state.conversation_store.clone(), cid)); translate_chat_response_to_responses( resp, diff --git a/crates/spark-server/src/api/responses_stream.rs b/crates/spark-server/src/api/responses_stream.rs index d3befa015..154f6e6f1 100644 --- a/crates/spark-server/src/api/responses_stream.rs +++ b/crates/spark-server/src/api/responses_stream.rs @@ -11,14 +11,14 @@ use futures::StreamExt; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; -use super::chat_stream::chat_completions_stream; +use super::chat_stream::run_chat_stream; use super::responses_stream_finalize::{ CloseOpenCtx, FinalizeCtx, close_open_items, emit_responses_prologue, finalize_responses_stream, }; use super::responses_translate::{ build_responses_usage, emit, find_frame_end, translate_chat_response_to_responses, }; -use super::stored::extract_assistant_incoming_message; +use super::stored::assistant_incoming_from_ir; use crate::AppState; use crate::openai::{ ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, CompletionChunk, @@ -87,17 +87,25 @@ pub(super) async fn responses_endpoint_stream( // Run the chat-completions streaming handler (re-using the full // pipeline: scheduler, tool detection, thinking, logprobs, ...). - let chat_resp = chat_completions_inner(state.0, None, chat_req, None).await; - let (parts, body) = chat_resp.into_parts(); - if !parts.status.is_success() { - return Response::from_parts(parts, body); - } + let deltas = match chat_completions_inner(state.0, None, chat_req.into_ir(), None).await { + super::chat::ChatOutcome::Streaming(d) => d, + // Error envelope — forward unchanged. + super::chat::ChatOutcome::Http(r) => return r, + // Unreachable by construction: this endpoint lowers with + // stream=true, so a success is always Streaming. + super::chat::ChatOutcome::Blocking(_) => { + return openai_error_response( + StatusCode::INTERNAL_SERVER_ERROR, + "internal: expected streaming outcome".to_string(), + ); + } + }; // Channel the transformer pushes Responses SSE events into. Sized to // match chat_stream/mod.rs (~30s decode buffer at 50 tok/s). let (tx, rx) = mpsc::channel::>(1024); - let created_at = crate::openai::unix_timestamp(); - let resp_id = format!("resp_{}", crate::openai::uuid_v4()); + let created_at = crate::ids::unix_timestamp(); + let resp_id = format!("resp_{}", crate::ids::uuid_v4()); let metadata_for_done = metadata.clone(); tokio::spawn(async move { @@ -122,78 +130,34 @@ pub(super) async fn responses_endpoint_stream( let mut completed_items: Vec = Vec::new(); let mut final_usage: Option = None; let mut finish_reason = "stop".to_string(); - let mut accumulator = Vec::::new(); let mut refusal_text: Option = None; seq = emit_responses_prologue(&tx, seq, &resp_id, created_at, &model, &metadata).await; - let mut data_stream = body.into_data_stream(); - while let Some(next) = data_stream.next().await { - let bytes = match next { - Ok(b) => b, - Err(e) => { - tracing::warn!("responses stream read error: {e}"); - break; - } - }; - accumulator.extend_from_slice(&bytes); - - // Parse SSE frames: look for "\n\n". - while let Some(idx) = find_frame_end(&accumulator) { - let frame_bytes: Vec = accumulator.drain(..idx + 2).collect(); - // Strip the trailing \n\n; find the data: line. - let frame_str = std::str::from_utf8(&frame_bytes).unwrap_or(""); - let data_line = frame_str.lines().find_map(|l| l.strip_prefix("data: ")); - let Some(data) = data_line else { continue }; - if data == "[DONE]" { - continue; - } - // Parse as ChatCompletionChunk JSON. - let Ok(chunk): Result = serde_json::from_str(data) else { - continue; - }; - // Chat chunks: { choices: [{ delta: {content|tool_calls|reasoning_content}, finish_reason, ... }], usage } - if let Some(u) = chunk.get("usage") - && !u.is_null() - { - final_usage = Some(u.clone()); - } - let Some(choice) = chunk - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - else { - continue; - }; - if let Some(fr) = choice.get("finish_reason").and_then(|v| v.as_str()) { - finish_reason = fr.to_string(); - } - let delta = choice - .get("delta") - .cloned() - .unwrap_or(serde_json::Value::Null); - - // Refusal delta (post-hoc: one chunk carries the full - // refusal sentence, emitted by the inner chat streamer). - if let Some(r) = delta.get("refusal").and_then(|v| v.as_str()) - && !r.is_empty() - { - refusal_text = Some(r.to_string()); + let mut deltas = deltas; + while let Some(delta) = deltas.next().await { + use crate::ir::StreamDelta; + match delta { + // Reasoning has no Responses-stream representation (the + // old transformer ignored reasoning_content chunks too). + StreamDelta::Reasoning { .. } => {} + // Refusal delta (post-hoc: one delta carries the full + // refusal sentence). + StreamDelta::Refusal { text } if !text.is_empty() => { + refusal_text = Some(text.clone()); let ev = crate::openai::ResponsesStreamEvent::RefusalDelta { sequence_number: seq, item_id: message_item_id.clone(), output_index, content_index: 0, - delta: r.to_string(), + delta: text, }; emit(&tx, &ev).await; seq += 1; } - + StreamDelta::Refusal { .. } => {} // Text content delta. - if let Some(text) = delta.get("content").and_then(|v| v.as_str()) - && !text.is_empty() - { + StreamDelta::Content { text, .. } if !text.is_empty() => { // If a function_call is currently open, close it // before opening a fresh message item — otherwise // the message would collide with the function_call @@ -263,108 +227,116 @@ pub(super) async fn responses_endpoint_stream( emit(&tx, &ev).await; seq += 1; } - content_text.push_str(text); + content_text.push_str(&text); let ev = crate::openai::ResponsesStreamEvent::OutputTextDelta { sequence_number: seq, item_id: message_item_id.clone(), output_index, content_index: 0, - delta: text.to_string(), + delta: text, }; emit(&tx, &ev).await; seq += 1; } - - // Tool-call fragments. - if let Some(tool_calls) = delta.get("tool_calls").and_then(|v| v.as_array()) { - for tc in tool_calls { - let call_id = tc.get("id").and_then(|v| v.as_str()).map(str::to_string); - let name = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|v| v.as_str()) - .map(str::to_string); - let args_frag = tc - .get("function") - .and_then(|f| f.get("arguments")) - .and_then(|v| v.as_str()) - .unwrap_or(""); - if let Some(n) = name.clone() { - current_tool_name = Some(n.clone()); - if let Some(id) = call_id.clone() { - current_tool_call_id = Some(id); - } - if !fc_started { - // Close any open message before starting function call. - if message_started { - let ev = crate::openai::ResponsesStreamEvent::OutputTextDone { - sequence_number: seq, - item_id: message_item_id.clone(), - output_index, - content_index: 0, - text: content_text.clone(), - }; - emit(&tx, &ev).await; - seq += 1; - let done = crate::openai::ResponsesOutputItem::Message { - id: message_item_id.clone(), - status: "completed", - role: "assistant", - content: vec![ - crate::openai::ResponsesContentPart::OutputText { - text: content_text.clone(), - annotations: crate::citation::merged_annotations( - &content_text, - ), - }, - ], - }; - completed_items.push(done.clone()); - let ev = crate::openai::ResponsesStreamEvent::OutputItemDone { - sequence_number: seq, - output_index, - item: done, - }; - emit(&tx, &ev).await; - seq += 1; - output_index += 1; - message_started = false; - content_text.clear(); - } - let fcid = format!("fc_{}_{}", resp_id, output_index); - fc_item_id = Some(fcid.clone()); - let item = crate::openai::ResponsesOutputItem::FunctionCall { - id: fcid, - call_id: current_tool_call_id.clone().unwrap_or_default(), - name: n, - arguments: String::new(), - status: "in_progress", - }; - let ev = crate::openai::ResponsesStreamEvent::OutputItemAdded { - sequence_number: seq, - output_index, - item, - }; - emit(&tx, &ev).await; - seq += 1; - fc_started = true; - } - } - if !args_frag.is_empty() { - tool_args.push_str(args_frag); - if let Some(fcid) = fc_item_id.clone() { - let ev = crate::openai::ResponsesStreamEvent::FunctionCallArgumentsDelta { - sequence_number: seq, - item_id: fcid, - output_index, - delta: args_frag.to_string(), - }; - emit(&tx, &ev).await; - seq += 1; - } + StreamDelta::Content { .. } => {} + // A tool call opens (name always present on the start + // delta). + StreamDelta::ToolCallStart { id, name, .. } => { + current_tool_name = Some(name.clone()); + current_tool_call_id = Some(id); + if !fc_started { + // Close any open message before starting function call. + if message_started { + let ev = crate::openai::ResponsesStreamEvent::OutputTextDone { + sequence_number: seq, + item_id: message_item_id.clone(), + output_index, + content_index: 0, + text: content_text.clone(), + }; + emit(&tx, &ev).await; + seq += 1; + let done = crate::openai::ResponsesOutputItem::Message { + id: message_item_id.clone(), + status: "completed", + role: "assistant", + content: vec![crate::openai::ResponsesContentPart::OutputText { + text: content_text.clone(), + annotations: crate::openai::merged_annotations(&content_text), + }], + }; + completed_items.push(done.clone()); + let ev = crate::openai::ResponsesStreamEvent::OutputItemDone { + sequence_number: seq, + output_index, + item: done, + }; + emit(&tx, &ev).await; + seq += 1; + output_index += 1; + message_started = false; + content_text.clear(); } + let fcid = format!("fc_{}_{}", resp_id, output_index); + fc_item_id = Some(fcid.clone()); + let item = crate::openai::ResponsesOutputItem::FunctionCall { + id: fcid, + call_id: current_tool_call_id.clone().unwrap_or_default(), + name, + arguments: String::new(), + status: "in_progress", + }; + let ev = crate::openai::ResponsesStreamEvent::OutputItemAdded { + sequence_number: seq, + output_index, + item, + }; + emit(&tx, &ev).await; + seq += 1; + fc_started = true; + } + } + // Argument JSON fragment for the open tool call. + StreamDelta::ToolCallArgs { fragment, .. } if !fragment.is_empty() => { + tool_args.push_str(&fragment); + if let Some(fcid) = fc_item_id.clone() { + let ev = crate::openai::ResponsesStreamEvent::FunctionCallArgumentsDelta { + sequence_number: seq, + item_id: fcid, + output_index, + delta: fragment, + }; + emit(&tx, &ev).await; + seq += 1; } } + StreamDelta::ToolCallArgs { .. } => {} + StreamDelta::Finish { reason, usage, .. } => { + finish_reason = reason.as_wire().to_string(); + // Wire-shaped usage for the finalizer (same JSON the + // OpenAI chunk carried before the delta migration). + final_usage = Some(serde_json::json!({ + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.prompt_tokens + usage.completion_tokens, + "prompt_tokens_details": { + "cached_tokens": usage.cached_prompt_tokens, + "audio_tokens": 0, + }, + "completion_tokens_details": { + "reasoning_tokens": usage.reasoning_tokens, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0, + }, + })); + } + // Stream-level error: the old transformer ignored the + // error payload line; the stream simply ends and the + // finalizer closes the response. + StreamDelta::Error { message } => { + tracing::warn!("responses stream: upstream error delta: {message}"); + } } } diff --git a/crates/spark-server/src/api/responses_stream_finalize.rs b/crates/spark-server/src/api/responses_stream_finalize.rs index 4d88c73d2..bbdce9c50 100644 --- a/crates/spark-server/src/api/responses_stream_finalize.rs +++ b/crates/spark-server/src/api/responses_stream_finalize.rs @@ -124,7 +124,7 @@ pub(super) async fn close_open_items( role: "assistant", content: vec![crate::openai::ResponsesContentPart::OutputText { text: ctx.content_text.to_string(), - annotations: crate::citation::merged_annotations(ctx.content_text), + annotations: crate::openai::merged_annotations(ctx.content_text), }], }; completed_items.push(done.clone()); diff --git a/crates/spark-server/src/api/responses_translate.rs b/crates/spark-server/src/api/responses_translate.rs index 9b27051de..638853949 100644 --- a/crates/spark-server/src/api/responses_translate.rs +++ b/crates/spark-server/src/api/responses_translate.rs @@ -11,7 +11,7 @@ use futures::StreamExt; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; -use super::stored::extract_assistant_incoming_message; +use super::stored::assistant_incoming_from_ir; use crate::AppState; use crate::openai::{ ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, CompletionChunk, @@ -76,107 +76,52 @@ pub(super) fn build_responses_usage(u: &serde_json::Value) -> crate::openai::Res } } -/// Translate a `ChatCompletionResponse` (already serialized in the Response) -/// into a `ResponsesResponse`. Only handles the JSON success path — -/// error responses are forwarded unchanged. +/// Encode the pipeline outcome for the Responses API surface. A +/// blocking success arrives as the canonical response IR (typed — no +/// serialized-body re-parse); error envelopes pass through unchanged. /// /// When `store` is provided and `store_flag` is true, the final /// `{input_messages + assistant turn}` transcript is persisted under /// `resp_` so a follow-up `previous_response_id` lookup can resume. pub(super) async fn translate_chat_response_to_responses( - resp: Response, + outcome: super::chat::ChatOutcome, req_metadata: Option>, store: Option>, input_messages: Vec, store_flag: bool, conversation: Option<(Arc, String)>, ) -> Response { - let (parts, body) = resp.into_parts(); - if !parts.status.is_success() { - return Response::from_parts(parts, body); - } - let bytes = match axum::body::to_bytes(body, 16 * 1024 * 1024).await { - Ok(b) => b, - Err(e) => { + let chat = match outcome { + super::chat::ChatOutcome::Http(r) => return r, + // Unreachable: this encoder serves the non-streaming endpoint. + super::chat::ChatOutcome::Streaming(_) => { return openai_error_response( StatusCode::INTERNAL_SERVER_ERROR, - format!("adapter body read failed: {e}"), + "internal: expected blocking outcome".to_string(), ); } + super::chat::ChatOutcome::Blocking(ir) => *ir, }; - let chat: serde_json::Value = match serde_json::from_slice(&bytes) { - Ok(c) => c, - Err(_) => { - // Not a chat-completion success body — pass through. - return axum::response::Response::builder() - .status(parts.status) - .body(axum::body::Body::from(bytes)) - .unwrap_or_else(|_| { - openai_error_response( - StatusCode::INTERNAL_SERVER_ERROR, - "adapter passthrough failed".into(), - ) - }); - } - }; - let id = chat - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("chatcmpl-unknown") - .to_string(); - let model = chat - .get("model") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let created = chat - .get("created") - .and_then(|v| v.as_u64()) - .unwrap_or_else(crate::openai::unix_timestamp); + // Wire-compatible ids: the historical path embedded the full + // `chatcmpl-` id inside the item ids. + let full_id = format!("chatcmpl-{}", chat.id); let mut output: Vec = Vec::new(); - if let Some(choice) = chat - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - { - let msg = choice - .get("message") - .cloned() - .unwrap_or(serde_json::Value::Null); - if let Some(tool_calls) = msg.get("tool_calls").and_then(|t| t.as_array()) { - for (i, tc) in tool_calls.iter().enumerate() { - let call_id = tc - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let name = tc - .get("function") - .and_then(|f| f.get("name")) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let arguments = tc - .get("function") - .and_then(|f| f.get("arguments")) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - output.push(crate::openai::ResponsesOutputItem::FunctionCall { - id: format!("fc_{}_{}", id, i), - call_id, - name, - arguments, - status: "completed", - }); - } + if let Some(choice) = chat.choices.first() { + for (i, tc) in choice.tool_calls.iter().enumerate() { + output.push(crate::openai::ResponsesOutputItem::FunctionCall { + id: format!("fc_{}_{}", full_id, i), + call_id: tc.id.clone(), + name: tc.name.clone(), + arguments: tc.arguments.to_string(), + status: "completed", + }); } - if let Some(text) = msg.get("content").and_then(|v| v.as_str()) { - let annotations: Option> = msg - .get("annotations") - .and_then(|a| serde_json::from_value(a.clone()).ok()); + if let Some(text) = choice.content.as_deref() { + // URL-citation annotations derived from the final content, + // matching the chat surface's encoder. + let annotations = crate::openai::merged_annotations(text); output.push(crate::openai::ResponsesOutputItem::Message { - id: format!("msg_{}", id), + id: format!("msg_{}", full_id), status: "completed", role: "assistant", content: vec![crate::openai::ResponsesContentPart::OutputText { @@ -186,30 +131,27 @@ pub(super) async fn translate_chat_response_to_responses( }); } } - let u = chat - .get("usage") - .cloned() - .unwrap_or(serde_json::Value::Null); let usage = crate::openai::ResponsesUsage { - input_tokens: u.get("prompt_tokens").and_then(|v| v.as_u64()).unwrap_or(0) as usize, - input_tokens_details: u - .get("prompt_tokens_details") - .and_then(|v| serde_json::from_value(v.clone()).ok()), - output_tokens: u - .get("completion_tokens") - .and_then(|v| v.as_u64()) - .unwrap_or(0) as usize, - output_tokens_details: u - .get("completion_tokens_details") - .and_then(|v| serde_json::from_value(v.clone()).ok()), - total_tokens: u.get("total_tokens").and_then(|v| v.as_u64()).unwrap_or(0) as usize, + input_tokens: chat.usage.prompt_tokens, + input_tokens_details: Some(crate::openai::PromptTokensDetails { + cached_tokens: chat.usage.cached_prompt_tokens, + audio_tokens: 0, + }), + output_tokens: chat.usage.completion_tokens, + output_tokens_details: Some(crate::openai::CompletionTokensDetails { + reasoning_tokens: chat.usage.reasoning_tokens, + audio_tokens: 0, + accepted_prediction_tokens: 0, + rejected_prediction_tokens: 0, + }), + total_tokens: chat.usage.prompt_tokens + chat.usage.completion_tokens, }; - let resp_id = format!("resp_{}", id.trim_start_matches("chatcmpl-")); + let resp_id = format!("resp_{}", chat.id); let resp = crate::openai::ResponsesResponse { id: resp_id.clone(), object: "response", - created_at: created, - model: model.clone(), + created_at: chat.created, + model: chat.model.clone(), status: "completed", error: None, output, @@ -233,14 +175,14 @@ pub(super) async fn translate_chat_response_to_responses( if store_flag && let Some(store) = store { let mut transcript = input_messages.clone(); // Append the assistant turn so subsequent resumes see it. - if let Some(assistant_msg) = extract_assistant_incoming_message(&chat) { + if let Some(assistant_msg) = assistant_incoming_from_ir(&chat) { transcript.push(assistant_msg); } store.insert(crate::response_store::StoredEntry { id: resp_id, kind: crate::response_store::StoredKind::Response, - model: model.clone(), - created_at: created, + model: chat.model.clone(), + created_at: chat.created, messages: transcript, body: body.clone(), last_access: std::time::Instant::now(), @@ -262,14 +204,10 @@ pub(super) async fn translate_chat_response_to_responses( }) .collect(); let assistant_text = chat - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first()) - .and_then(|c| c.get("message")) - .and_then(|m| m.get("content")) - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); + .choices + .first() + .and_then(|c| c.content.as_deref()) + .unwrap_or(""); if !assistant_text.is_empty() { batch.push(serde_json::json!({ "type": "message", diff --git a/crates/spark-server/src/api/stored.rs b/crates/spark-server/src/api/stored.rs index 5981676b9..0fa820f59 100644 --- a/crates/spark-server/src/api/stored.rs +++ b/crates/spark-server/src/api/stored.rs @@ -11,7 +11,7 @@ use futures::StreamExt; use std::sync::Arc; use tokio_stream::wrappers::ReceiverStream; -use super::chat_stream::chat_completions_stream; +use super::chat_stream::run_chat_stream; use super::responses_stream::responses_endpoint_stream; use super::responses_translate::{ build_responses_usage, emit, find_frame_end, translate_chat_response_to_responses, @@ -42,37 +42,37 @@ use super::strip::strip_thinking_tags; // Re-export sibling helpers via crate::api::* for short paths. use super::inference_types::*; use super::sanitizer::*; - -pub(super) fn extract_assistant_incoming_message( - chat: &serde_json::Value, +/// Build the assistant transcript turn from the canonical response IR +/// (for `previous_response_id` resume storage). +pub(super) fn assistant_incoming_from_ir( + chat: &crate::ir::ChatResponse, ) -> Option { - let choice = chat - .get("choices") - .and_then(|c| c.as_array()) - .and_then(|a| a.first())?; - let msg = choice.get("message")?; - let text = msg - .get("content") - .and_then(|v| v.as_str()) - .unwrap_or("") - .to_string(); - let tool_calls = msg.get("tool_calls").and_then(|tc| { - serde_json::from_value::>(tc.clone()).ok() - }); + let choice = chat.choices.first()?; + let tool_calls: Vec = choice + .tool_calls + .iter() + .map(|tc| tool_parser::IncomingToolCall { + id: Some(tc.id.clone()), + function: tool_parser::IncomingFunction { + name: tc.name.clone(), + arguments: tc.arguments.to_string(), + }, + }) + .collect(); Some(crate::openai::IncomingMessage { role: "assistant".to_string(), content: crate::openai::ParsedContent { - text, + text: choice.content.clone().unwrap_or_default(), images: Vec::new(), }, - tool_calls, + tool_calls: if tool_calls.is_empty() { + None + } else { + Some(tool_calls) + }, tool_call_id: None, name: None, - reasoning_content: msg - .get("reasoning_content") - .or_else(|| msg.get("reasoning")) - .and_then(|v| v.as_str()) - .map(|s| s.to_string()), + reasoning_content: choice.reasoning.clone(), }) } diff --git a/crates/spark-server/src/citation.rs b/crates/spark-server/src/citation.rs index efc11ec0b..8968d34f4 100644 --- a/crates/spark-server/src/citation.rs +++ b/crates/spark-server/src/citation.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only -//! Structured citation parser for assistant content. +//! Citation extraction from assistant content — provider-neutral. //! -//! Complements [`crate::openai::extract_url_annotations`] (which finds -//! bare + markdown-link URLs) by recognizing three common -//! "model-emitted" citation patterns that would otherwise miss the -//! structured `url_citation` shape OpenAI clients expect: +//! Two extractors work together: +//! +//! * [`extract_url_citations`] finds bare + markdown-link URLs (skipping +//! code spans so `curl https://…` examples don't become citations). +//! * [`extract`] recognizes three common "model-emitted" structured +//! citation patterns: //! //! 1. Markdown footnotes //! ```text @@ -33,22 +35,31 @@ //! Each bullet → one citation at the bullet's URL span. //! //! The parser is conservative: it only fires when the definition / -//! bullet contains an http(s) URL. Output is a `Vec` that -//! the handler merges with the bare-URL extractor's output (dedupe on -//! URL to avoid double-counting). +//! bullet contains an http(s) URL. [`merged_citations`] runs both and +//! dedupes on URL. Output is the neutral [`Citation`] struct — each API +//! surface converts it to its own wire annotation shape (e.g. +//! `openai::Annotation`). //! //! This is still post-hoc parsing — Atlas has no web-search tool, so //! "model-sourced" here means "the model emitted a structured citation //! pattern we recognize". The shape clients receive is identical to //! what a real web-search backend would produce. -use crate::openai::{Annotation, UrlCitation}; +/// A URL citation found in assistant text, with byte offsets into the +/// content it was extracted from. Provider-neutral. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Citation { + pub start_index: usize, + pub end_index: usize, + pub url: String, + pub title: String, +} /// Convenience: run the bare-URL extractor + the structured citation -/// extractor and return the deduped annotations. `None` when nothing -/// matched, so the field gets serde-skipped on the wire. -pub fn merged_annotations(content: &str) -> Option> { - let bare = crate::openai::extract_url_annotations(content).unwrap_or_default(); +/// extractor and return the deduped citations. `None` when nothing +/// matched, so wire fields can be serde-skipped. +pub fn merged_citations(content: &str) -> Option> { + let bare = extract_url_citations(content); let structured = extract(content); let merged = merge_dedupe(bare, structured); if merged.is_empty() { @@ -58,23 +69,137 @@ pub fn merged_annotations(content: &str) -> Option> { } } -/// Find citations in `content` and return an annotation for each +/// Find structured citations in `content` and return one entry per /// reference site. Returns an empty vec when nothing matched. -pub fn extract(content: &str) -> Vec { +pub fn extract(content: &str) -> Vec { let mut out = Vec::new(); out.extend(footnote_citations(content)); out.extend(numeric_ref_citations(content)); out.extend(sources_block_citations(content)); // Sort by start position so consumers see document order. - out.sort_by_key(|a| match a { - Annotation::UrlCitation { url_citation } => url_citation.start_index, - }); + out.sort_by_key(|c| c.start_index); + out +} + +/// Scan `content` for http(s) URLs and emit a citation per hit. +/// +/// The extractor handles three shapes: +/// - Markdown links `[title](url)` — title from the `[...]` text. +/// - Bare URLs — title is the URL itself. +/// - URLs inside fenced code blocks (triple backticks) or inline code +/// (`` `...` ``) are **skipped** — illustrative code, not citations. +/// This prevents false positives on model output like +/// `curl https://example.com`. +pub fn extract_url_citations(content: &str) -> Vec { + let mut out: Vec = Vec::new(); + let masked = mask_code_spans(content); + // First pass: markdown links. We scan the masked copy so the + // start/end indices we record match the original `content` exactly. + let mut scan = 0usize; + while scan < masked.len() { + let rest = &masked[scan..]; + let Some(lb_rel) = rest.find('[') else { break }; + let lb = scan + lb_rel; + // Find the matching `]` then immediate `(`. + let after_lb = &masked[lb + 1..]; + let Some(rb_rel) = after_lb.find(']') else { + scan = lb + 1; + continue; + }; + let rb = lb + 1 + rb_rel; + if masked.as_bytes().get(rb + 1) != Some(&b'(') { + scan = rb + 1; + continue; + } + // Find the matching close paren that respects nested `()` pairs + // inside the URL — Wikipedia and other URLs commonly contain + // parentheses (e.g. `https://en.wikipedia.org/wiki/Foo_(bar)`). + // The bare-find shortcut would terminate at the first `)`, + // truncating the URL. + let after_paren = &masked[rb + 2..]; + let Some(rparen_rel) = balanced_paren_close(after_paren) else { + scan = rb + 2; + continue; + }; + let rparen = rb + 2 + rparen_rel; + let title = &content[lb + 1..rb]; + let target = content[rb + 2..rparen].trim(); + if (target.starts_with("http://") || target.starts_with("https://")) + && target.len() > "https://".len() + { + out.push(Citation { + start_index: rb + 2, + end_index: rparen, + url: target.to_string(), + title: title.to_string(), + }); + } + scan = rparen + 1; + } + + // Second pass: bare URLs in regions that aren't masked AND aren't + // already covered by a markdown link. + let covered: Vec<(usize, usize)> = out.iter().map(|c| (c.start_index, c.end_index)).collect(); + let mut i = 0usize; + while i < masked.len() { + let rest = &masked[i..]; + let Some(off) = rest.find("http") else { break }; + let abs_start = i + off; + let tail_masked = &masked[abs_start..]; + let is_url = tail_masked.starts_with("http://") || tail_masked.starts_with("https://"); + if !is_url { + i = abs_start + 4; + continue; + } + let tail = &content[abs_start..]; + let end_rel = tail + .find(|c: char| { + c.is_whitespace() || matches!(c, ']' | '}' | '"' | '<' | '>' | '`' | '\\') + }) + .unwrap_or(tail.len()); + let mut raw = &tail[..end_rel]; + // Strip trailing sentence punctuation and unmatched close-parens / + // markdown emphasis markers. Parens match pairs so URLs like + // Wikipedia's `https://en.wikipedia.org/wiki/Foo_(bar)` survive. + while let Some(last) = raw.chars().last() { + let strip = match last { + '.' | ',' | ';' | ':' | '!' | '?' | '*' | '_' => true, + ')' => { + let opens = raw.matches('(').count(); + let closes = raw.matches(')').count(); + closes > opens + } + _ => false, + }; + if !strip { + break; + } + raw = &raw[..raw.len() - last.len_utf8()]; + } + if raw.len() > "https://".len() { + let start = abs_start; + let end = abs_start + raw.len(); + let overlaps = covered.iter().any(|(s, e)| start < *e && end > *s); + if !overlaps { + out.push(Citation { + start_index: start, + end_index: end, + url: raw.to_string(), + title: raw.to_string(), + }); + } + } + i = abs_start + end_rel.max(1); + } + // Sort by start index so downstream consumers see citations in + // document order regardless of which pass emitted them. + out.sort_by_key(|c| c.start_index); out } /// Parse markdown footnote references: every `[^label]` in text paired /// with a `[^label]: url [title]` definition. -fn footnote_citations(content: &str) -> Vec { +fn footnote_citations(content: &str) -> Vec { let defs = collect_footnote_defs(content); if defs.is_empty() { return Vec::new(); @@ -96,13 +221,11 @@ fn footnote_citations(content: &str) -> Vec { } let label = &content[start + 2..close]; if let Some((url, title)) = defs.get(label) { - out.push(Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: start, - end_index: close + 1, - url: url.clone(), - title: title.clone().unwrap_or_else(|| label.to_string()), - }, + out.push(Citation { + start_index: start, + end_index: close + 1, + url: url.clone(), + title: title.clone().unwrap_or_else(|| label.to_string()), }); } i = close + 1; @@ -151,7 +274,7 @@ fn collect_footnote_defs( } /// Parse `[N]` refs paired with a `[N] url` definition line. -fn numeric_ref_citations(content: &str) -> Vec { +fn numeric_ref_citations(content: &str) -> Vec { let defs = collect_numeric_defs(content); if defs.is_empty() { return Vec::new(); @@ -178,13 +301,11 @@ fn numeric_ref_citations(content: &str) -> Vec { }; let is_definition = at_line_start && next_is_url_hint; if !is_definition && let Some((url, title)) = defs.get(label) { - out.push(Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: start, - end_index: close + 1, - url: url.clone(), - title: title.clone().unwrap_or_else(|| label.to_string()), - }, + out.push(Citation { + start_index: start, + end_index: close + 1, + url: url.clone(), + title: title.clone().unwrap_or_else(|| label.to_string()), }); } i = close + 1; @@ -233,7 +354,7 @@ fn collect_numeric_defs( /// Find `Sources:` or `References:` heading and emit a citation for /// each bullet line containing an http(s) URL until a blank line or a /// non-bullet line breaks the section. -fn sources_block_citations(content: &str) -> Vec { +fn sources_block_citations(content: &str) -> Vec { let lower = content.to_ascii_lowercase(); let markers = ["sources:", "references:", "citations:"]; let mut out = Vec::new(); @@ -290,13 +411,11 @@ fn sources_block_citations(content: &str) -> Vec { } else { title.to_string() }; - out.push(Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: url_abs_start, - end_index: url_abs_end, - url, - title: title_out, - }, + out.push(Citation { + start_index: url_abs_start, + end_index: url_abs_end, + url, + title: title_out, }); } search_from = after; @@ -321,47 +440,120 @@ fn split_url(s: &str) -> Option<(String, &str)> { Some((url.to_string(), &s[end..])) } -/// Merge two annotation lists, deduping by URL (keeping the first hit +/// Merge two citation lists, deduping by URL (keeping the first hit /// in document order). Used to combine the bare-URL extractor's output /// with structured citations without emitting the same URL twice. -pub fn merge_dedupe(mut primary: Vec, secondary: Vec) -> Vec { +pub fn merge_dedupe(mut primary: Vec, secondary: Vec) -> Vec { use std::collections::HashSet; let mut seen: HashSet = HashSet::new(); - for a in &primary { - let Annotation::UrlCitation { url_citation } = a; - seen.insert(url_citation.url.clone()); + for c in &primary { + seen.insert(c.url.clone()); } - for a in secondary { - let url = match &a { - Annotation::UrlCitation { url_citation } => url_citation.url.clone(), - }; - if seen.insert(url) { - primary.push(a); + for c in secondary { + if seen.insert(c.url.clone()) { + primary.push(c); } } - primary.sort_by_key(|a| match a { - Annotation::UrlCitation { url_citation } => url_citation.start_index, - }); + primary.sort_by_key(|c| c.start_index); primary } +/// Return the byte offset of the `)` that balances the implicit `(` +/// before the slice (i.e. the URL's matching close), or None if no +/// balanced close exists. Handles nested `()` pairs that appear in +/// real URLs (Wikipedia article slugs, GitHub anchors, etc). +fn balanced_paren_close(s: &str) -> Option { + let bytes = s.as_bytes(); + let mut depth: i32 = 0; + for (i, &b) in bytes.iter().enumerate() { + match b { + b'(' => depth += 1, + b')' => { + if depth == 0 { + return Some(i); + } + depth -= 1; + } + _ => {} + } + } + None +} + +/// Return a copy of `content` where the insides of fenced code blocks +/// (```) and inline code spans (`) are replaced with ASCII spaces while +/// preserving byte offsets and UTF-8 validity. Used so the URL scan can +/// skip over code regions without rebuilding indices. +/// +/// We walk char-by-char to keep multi-byte characters intact: each non- +/// newline char inside a code region becomes N ASCII spaces where N is +/// the char's UTF-8 byte length. Newlines are kept so line-based parsers +/// still see the structure. +fn mask_code_spans(content: &str) -> String { + let bytes = content.as_bytes(); + let mut out: Vec = bytes.to_vec(); + + // Blank bytes [start, end) by char: non-newline chars → ASCII spaces + // of the same byte length, newlines preserved. Char boundaries taken + // from the original content (which is guaranteed UTF-8). + fn blank(out: &mut [u8], content: &str, start: usize, end: usize) { + let region = &content[start..end]; + let mut cursor = start; + for ch in region.chars() { + let len = ch.len_utf8(); + if ch != '\n' { + for b in out.iter_mut().skip(cursor).take(len) { + *b = b' '; + } + } + cursor += len; + } + } + + let mut i = 0usize; + while i < bytes.len() { + if bytes[i..].starts_with(b"```") { + let after = i + 3; + let rest = &content[after..]; + let end = match rest.find("```") { + Some(r) => after + r + 3, + None => bytes.len(), + }; + blank(&mut out, content, i, end); + i = end; + continue; + } + if bytes[i] == b'`' { + let after = i + 1; + let rest = &content[after..]; + let end = match rest.find('`') { + Some(r) => after + r + 1, + None => bytes.len(), + }; + blank(&mut out, content, i, end); + i = end; + continue; + } + // Step one whole UTF-8 codepoint. + let step = match bytes[i] { + 0x00..=0x7f => 1, + 0xc0..=0xdf => 2, + 0xe0..=0xef => 3, + 0xf0..=0xf7 => 4, + _ => 1, + }; + i += step; + } + String::from_utf8(out).expect("mask preserves UTF-8") +} + #[cfg(test)] mod tests { use super::*; - fn urls(anns: &[Annotation]) -> Vec<(&str, &str, usize, usize)> { - anns.iter() - .map(|a| match a { - Annotation::UrlCitation { - url_citation: - UrlCitation { - url, - title, - start_index, - end_index, - }, - } => (url.as_str(), title.as_str(), *start_index, *end_index), - }) + fn urls(cits: &[Citation]) -> Vec<(&str, &str, usize, usize)> { + cits.iter() + .map(|c| (c.url.as_str(), c.title.as_str(), c.start_index, c.end_index)) .collect() } @@ -430,23 +622,89 @@ mod tests { #[test] fn merge_dedupe_by_url() { - let a = vec![Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: 0, - end_index: 20, - url: "https://example.com".into(), - title: "first".into(), - }, + let a = vec![Citation { + start_index: 0, + end_index: 20, + url: "https://example.com".into(), + title: "first".into(), }]; - let b = vec![Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: 50, - end_index: 70, - url: "https://example.com".into(), - title: "second".into(), - }, + let b = vec![Citation { + start_index: 50, + end_index: 70, + url: "https://example.com".into(), + title: "second".into(), }]; let merged = merge_dedupe(a, b); assert_eq!(merged.len(), 1); } + + // ── bare/markdown URL scanner (moved from openai/annotations.rs) ── + + #[test] + fn bare_url_extracted() { + let got = extract_url_citations("see https://example.com/foo for more"); + assert_eq!(got.len(), 1); + assert_eq!(got[0].url, "https://example.com/foo"); + assert_eq!(got[0].title, "https://example.com/foo"); + } + + #[test] + fn trailing_punctuation_stripped() { + let got = extract_url_citations("go to https://example.com."); + assert_eq!(got[0].url, "https://example.com"); + } + + #[test] + fn wikipedia_parens_survive() { + let got = extract_url_citations("see https://en.wikipedia.org/wiki/Foo_(bar) now"); + assert_eq!(got[0].url, "https://en.wikipedia.org/wiki/Foo_(bar)"); + } + + #[test] + fn markdown_link_title_used() { + let got = extract_url_citations("read [the docs](https://example.com/api) today"); + assert_eq!(got.len(), 1); + assert_eq!(got[0].url, "https://example.com/api"); + assert_eq!(got[0].title, "the docs"); + } + + #[test] + fn markdown_link_with_parens_in_url() { + let got = + extract_url_citations("see [Foo (bar)](https://en.wikipedia.org/wiki/Foo_(bar)) here"); + assert_eq!(got.len(), 1); + assert_eq!(got[0].url, "https://en.wikipedia.org/wiki/Foo_(bar)"); + } + + #[test] + fn code_spans_skipped() { + assert!(extract_url_citations("run `curl https://example.com` locally").is_empty()); + assert!(extract_url_citations("```\ncurl https://example.com\n```\nno cite").is_empty()); + } + + #[test] + fn non_http_schemes_ignored() { + assert!(extract_url_citations("ftp://example.com not a citation").is_empty()); + } + + #[test] + fn empty_and_plain_content() { + assert!(extract_url_citations("").is_empty()); + assert!(extract_url_citations("no URLs here").is_empty()); + } + + #[test] + fn query_and_fragment_kept() { + let got = extract_url_citations("see https://example.com/p?q=1&r=2#frag here"); + assert_eq!(got[0].url, "https://example.com/p?q=1&r=2#frag"); + } + + #[test] + fn merged_citations_combines_and_dedupes() { + let input = "Intro https://a.example.com text.\n\nSources:\n- https://a.example.com\n- https://b.example.com\n"; + let got = merged_citations(input).unwrap(); + let mut u: Vec<&str> = got.iter().map(|c| c.url.as_str()).collect(); + u.dedup(); + assert_eq!(u.len(), 2); + } } diff --git a/crates/spark-server/src/conversation_store.rs b/crates/spark-server/src/conversation_store.rs index 0f793e0bf..658c92ca7 100644 --- a/crates/spark-server/src/conversation_store.rs +++ b/crates/spark-server/src/conversation_store.rs @@ -94,7 +94,7 @@ impl ConversationStore { initial_items: Vec, metadata: HashMap, ) -> String { - let id = format!("conv_{}", crate::openai::uuid_v4()); + let id = format!("conv_{}", crate::ids::uuid_v4()); let now_unix = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) diff --git a/crates/spark-server/src/ids.rs b/crates/spark-server/src/ids.rs new file mode 100644 index 000000000..c48d1becc --- /dev/null +++ b/crates/spark-server/src/ids.rs @@ -0,0 +1,58 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Process-local ID and timestamp helpers shared by every API surface and +// the internal stores. Provider-neutral: wire prefixes (`chatcmpl-`, +// `msg_`, `resp_`, `conv_`, `req_`) are applied by the callers that own +// the respective wire formats. + +/// Read random bytes from the OS (Linux: getrandom syscall). +pub(crate) fn getrandom(buf: &mut [u8]) -> Result<(), ()> { + use std::fs::File; + use std::io::Read; + File::open("/dev/urandom") + .and_then(|mut f| f.read_exact(buf)) + .map_err(|_| ()) +} + +/// UUID v4 generation using OS randomness (no external crate needed). +pub(crate) fn uuid_v4() -> String { + let mut bytes = [0u8; 16]; + if let Ok(()) = getrandom(&mut bytes) { + bytes[6] = (bytes[6] & 0x0F) | 0x40; // version 4 + bytes[8] = (bytes[8] & 0x3F) | 0x80; // variant 1 + } else { + // Fallback: nanosecond timestamp (unique but not random). + let t = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos(); + bytes = t.to_le_bytes(); + } + format!( + "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + bytes[0], + bytes[1], + bytes[2], + bytes[3], + bytes[4], + bytes[5], + bytes[6], + bytes[7], + bytes[8], + bytes[9], + bytes[10], + bytes[11], + bytes[12], + bytes[13], + bytes[14], + bytes[15], + ) +} + +/// Seconds since the Unix epoch (`created` fields on wire responses). +pub(crate) fn unix_timestamp() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} diff --git a/crates/spark-server/src/ir/message.rs b/crates/spark-server/src/ir/message.rs new file mode 100644 index 000000000..96a249b28 --- /dev/null +++ b/crates/spark-server/src/ir/message.rs @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Canonical chat message IR (request direction). Each API surface +// converts its wire messages into `Vec`; `build_msg_entries` +// and the template renderer read only these types. + +/// Message author role. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Role { + System, + User, + Assistant, + Tool, + /// Any other wire role string (e.g. `"developer"`, `"function"`), + /// preserved verbatim. The canonical roles above are matched by the + /// pipeline; unknown roles fall through to the template unchanged + /// (which is where today's "Unexpected message role" handling lives), + /// so introducing the IR does not change their behavior. + Other(String), +} + +impl Role { + /// Wire string used by the OpenAI/Anthropic envelopes and the Jinja + /// templates (`"system" | "user" | "assistant" | "tool"`, or the + /// preserved string for [`Role::Other`]). + pub fn as_wire(&self) -> &str { + match self { + Role::System => "system", + Role::User => "user", + Role::Assistant => "assistant", + Role::Tool => "tool", + Role::Other(s) => s, + } + } + + /// Parse a known wire role. Returns `None` for anything unknown — + /// callers decide the fallback explicitly (PCND: no silent default). + /// Use [`Role::from_wire_lossless`] when an unknown role must be + /// preserved rather than rejected. + pub fn from_wire(s: &str) -> Option { + match s { + "system" => Some(Role::System), + "user" => Some(Role::User), + "assistant" => Some(Role::Assistant), + "tool" => Some(Role::Tool), + _ => None, + } + } + + /// Map any wire role string to a `Role`, preserving unknown roles as + /// [`Role::Other`] (lossless — the template still decides what to do + /// with them). + pub fn from_wire_lossless(s: &str) -> Self { + Self::from_wire(s).unwrap_or_else(|| Role::Other(s.to_string())) + } +} + +/// One canonical chat message. `content` is ALWAYS a list of parts for +/// every role, so text and images interleave uniformly and an image on +/// a tool result is not a special case. +#[derive(Debug, Clone, PartialEq)] +pub struct Message { + pub role: Role, + /// Ordered content parts. Text and images interleave. + pub content: Vec, + /// Assistant-emitted tool calls (empty for other roles). + pub tool_calls: Vec, + /// Links a `Tool` message back to the `ToolCall.id` it answers. + pub tool_call_id: Option, + /// Tool/function name (tool messages; some surfaces echo it). + pub name: Option, + /// First-class historical reasoning trace (a prior assistant + /// `` body) instead of a `\n\n` string prefix. + pub reasoning: Option, + /// `Tool` role only: the tool result reported an error. The intended + /// first-class home for Anthropic's `is_error` flag. NOT wired yet — the + /// Anthropic path still emits the `[tool error]\n` text prefix; dropping + /// that in favor of this field is gated on flipping + /// `ChatCompletionRequest.messages` to the IR (issue #165 follow-up), so + /// no adapter sets it to `true` today. + pub tool_error: bool, +} + +impl Message { + /// A synthetic system message carrying only `text`. Used for + /// server-injected prompts (tool-parser behavioral prompt). + pub fn synthetic_system(text: String) -> Self { + Message { + role: Role::System, + content: vec![ContentPart::Text(text)], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + } + } + + /// Prepend `prefix` to the first text part (appending a new text + /// part when none exists — matching the canonical `[image*, text]` + /// part order). The flattened `text()` output equals + /// `prefix + old_text()` either way. Used for system-prompt + /// injection. + pub fn prepend_text(&mut self, prefix: &str) { + for part in &mut self.content { + if let ContentPart::Text(t) = part { + *t = format!("{prefix}{t}"); + return; + } + } + self.content.push(ContentPart::Text(prefix.to_string())); + } + + /// Concatenate the text parts in order (images skipped). Mirrors the + /// historical `text_parts.join("")` flattening so downstream + /// text-scanning logic (cwd hint, vacuous-system, error detection) + /// is unchanged. + pub fn text(&self) -> String { + let mut out = String::new(); + for part in &self.content { + if let ContentPart::Text(t) = part { + out.push_str(t); + } + } + out + } + + /// Number of image parts on this message (drives the Jinja + /// vision-marker expansion). + pub fn image_count(&self) -> usize { + self.content + .iter() + .filter(|p| matches!(p, ContentPart::Image(_))) + .count() + } +} + +/// A single piece of message content. Open for future modalities +/// (audio, video, file). +#[derive(Debug, Clone, PartialEq)] +pub enum ContentPart { + Text(String), + Image(ImageSource), +} + +/// Where an image comes from. The encoder consumes the inner string +/// directly (see `spark_model::vision_preprocess::preprocess_image`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageSource { + pub data: ImageData, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImageData { + /// A `data:image/...;base64,...` URI (or raw base64) — the exact + /// string `preprocess_image` decodes. + Base64(String), + /// Remote URL. The encoder does not fetch URLs; the pipeline rejects + /// these with an explicit 400 (`collect_message_images`) instead of + /// feeding them to the base64 decoder and surfacing a confusing + /// "base64 decode failed". + Url(String), +} + +impl ImageData { + /// Classify a wire image string. http(s) URLs → [`ImageData::Url`]; + /// everything else (data: URIs, raw base64) → [`ImageData::Base64`]. + /// Every adapter uses this so a remote URL can never masquerade as + /// decodable base64. + pub fn from_uri(s: String) -> Self { + if s.starts_with("http://") || s.starts_with("https://") { + ImageData::Url(s) + } else { + ImageData::Base64(s) + } + } +} + +/// Assistant-emitted tool call. `arguments` is structured JSON (already +/// parsed), not a string. +#[derive(Debug, Clone, PartialEq)] +pub struct ToolCall { + pub id: String, + pub name: String, + pub arguments: serde_json::Value, +} + +/// First-class reasoning/thinking trace. +#[derive(Debug, Clone, PartialEq)] +pub struct Reasoning { + pub text: String, +} diff --git a/crates/spark-server/src/ir/mod.rs b/crates/spark-server/src/ir/mod.rs new file mode 100644 index 000000000..a285a5587 --- /dev/null +++ b/crates/spark-server/src/ir/mod.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Canonical, model-agnostic chat IR. One IR, N adapters: each API +// surface (OpenAI chat, OpenAI Responses, Anthropic) converts its wire +// format into these types, and the core pipeline (`build_msg_entries`, +// template rendering) reads only the IR — never an endpoint-specific +// request/response type. This is the narrow waist that keeps the +// OpenAI/Anthropic surfaces from drifting (AGENTS.md). + +pub mod message; +pub mod request; +pub mod response; +pub mod stream; + +pub use message::{ContentPart, ImageData, Message, Role}; +pub use request::{ChatRequest, ResponseFormat, SamplingParams, ThinkingDirective}; +pub use response::{ChatResponse, Choice, ChoiceLogprobs, FinishReason, TokenLogprob, Usage}; +pub use stream::{DeltaStream, StreamDelta}; + +#[cfg(test)] +mod tests; diff --git a/crates/spark-server/src/ir/request.rs b/crates/spark-server/src/ir/request.rs new file mode 100644 index 000000000..ece97ecf1 --- /dev/null +++ b/crates/spark-server/src/ir/request.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Canonical chat IR (request direction): the provider-agnostic request +// envelope the whole internal pipeline consumes. Each API surface +// (OpenAI chat, OpenAI Responses, Anthropic Messages) resolves its wire +// format into `ChatRequest` at the edge; no internal module reads a +// wire type. +// +// Deliberately absent: echo-only wire fields (service_tier, store, +// metadata, stream_options.include_usage) — those never influence +// generation, so each surface handler keeps its parsed wire request +// and echoes them at encode time (`api/chat/echo.rs`). Accepted-and- +// ignored compat fields (audio, prediction, modalities, user, …) stay +// wire-only for the same reason (PCND: the IR carries exactly what +// internals read). + +/// Provider-agnostic chat request envelope. +#[derive(Debug, Clone)] +pub struct ChatRequest { + /// Client-requested model name (logging + response echo). + pub model: String, + pub messages: Vec, + /// Tool definitions; empty = none. (`tool_parser::ToolDefinition` + /// is already provider-neutral — every surface parses into it.) + pub tools: Vec, + pub tool_choice: Option, + pub sampling: SamplingParams, + pub max_tokens: usize, + pub min_tokens: usize, + /// Stop sequences (generation halts when one is produced). + pub stop: Vec, + pub stream: bool, + /// Number of choices (>1 only reachable from the OpenAI surface; + /// other adapters pin 1). + pub n: usize, + /// Output shape constraint. `None` = unconstrained text (the wire + /// `{"type":"text"}` maps to `None` at the edge). + pub response_format: Option, + /// Client thinking intent (edge-resolved). Server/model defaults + /// fold in later (`api/chat/thinking.rs`). + pub thinking: ThinkingDirective, + /// Per-request token-loop detector override. + pub repetition_detection: Option, + /// Per-token logit bias, already parsed from the wire's string-key + /// map at the edge (non-numeric keys dropped, matching history). + pub logit_bias: Vec<(u32, f32)>, + /// Logprob alternatives per token, already resolved from the wire's + /// `logprobs` + `top_logprobs` pair at the edge. `None` = disabled. + pub top_logprobs: Option, + /// Deterministic-sampling seed. + pub seed: Option, + /// Request timeout in seconds; `None` = server default. + pub timeout_secs: Option, + /// Emit sampled token IDs on stream chunks (vLLM extension). + pub return_token_ids: bool, +} + +/// Client sampling parameters. `None` = client silent → the server's +/// generation_config / MODEL.toml preset default applies downstream +/// (`api/chat/sampling_setup.rs`). +#[derive(Debug, Clone, Copy, Default)] +pub struct SamplingParams { + pub temperature: Option, + pub top_k: Option, + pub top_p: Option, + pub top_n_sigma: Option, + pub min_p: Option, + pub repetition_penalty: Option, + pub presence_penalty: Option, + pub frequency_penalty: Option, +} + +/// Output shape constraint (neutral form of OpenAI's `response_format`). +#[derive(Debug, Clone)] +pub enum ResponseFormat { + /// Output must be valid JSON (any shape). + JsonObject, + /// Output must match the given JSON schema. + JsonSchema { + /// Schema name (used for logging). + name: String, + schema: serde_json::Value, + /// Enforce strict adherence. + strict: bool, + }, +} + +/// Client thinking intent, resolved at the API edge from each wire's +/// channels (Anthropic `thinking`, OpenAI `reasoning.effort`, vLLM +/// `chat_template_kwargs` / `thinking_token_budget`, Atlas legacy +/// `enable_thinking`). Internal code reads only this — never the wire +/// fields. +/// +/// Model-default folding (MODEL.toml `[behavior].thinking_default`) +/// happens in `api/chat/thinking.rs` when the directive is +/// [`Unspecified`](ThinkingDirective::Unspecified); the directive itself +/// never encodes a server or model default. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ThinkingDirective { + /// No client signal. The server-level default directive (from + /// `--default-chat-template-kwargs`) applies if set, else the model + /// default from MODEL.toml. + #[default] + Unspecified, + /// Client explicitly disabled thinking. + Off, + /// Client explicitly enabled thinking. `budget: None` means no + /// explicit budget — defer to the per-model `max_thinking_budget` + /// cap rather than a conservative hardcoded default (a hard low cap + /// force-injects `` mid-reasoning and wrecks tool + /// selection). + On { budget: Option }, +} + +impl ThinkingDirective { + /// True when the client stated any explicit thinking intent — + /// enabled OR disabled. Server-side policies (e.g. MODEL.toml + /// `thinking_in_tools=false`) only apply when this is false. + pub fn is_explicit(&self) -> bool { + !matches!(self, ThinkingDirective::Unspecified) + } +} diff --git a/crates/spark-server/src/ir/response.rs b/crates/spark-server/src/ir/response.rs new file mode 100644 index 000000000..e31093caa --- /dev/null +++ b/crates/spark-server/src/ir/response.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Canonical chat IR (response direction). The blocking pipeline +// produces exactly one of these per request; each API surface encodes +// it into its own wire format (OpenAI chat JSON, Anthropic +// MessagesResponse, Responses API JSON). No surface re-parses another +// surface's serialized body. + +use super::message::ToolCall; + +/// A complete (non-streaming) chat response. +#[derive(Debug, Clone, PartialEq)] +pub struct ChatResponse { + /// Bare response id (a uuid) — surfaces apply their wire prefixes + /// (`chatcmpl-`, `msg_`, `resp_`). + pub id: String, + /// Served model name (encoders read this — no side-channel param). + pub model: String, + /// Unix seconds at response build time. + pub created: u64, + /// One entry per requested choice. `n > 1` is only reachable from + /// the OpenAI surface; other adapters pin `n = 1` and their + /// encoders read the first choice. + pub choices: Vec, + pub usage: Usage, +} + +/// One generated choice. +#[derive(Debug, Clone, PartialEq)] +pub struct Choice { + pub index: usize, + /// Assistant text (`None` mirrors the wire's `content: null`, e.g. + /// after a refusal strip). + pub content: Option, + /// Reasoning/thinking trace, when the model produced one. + pub reasoning: Option, + pub tool_calls: Vec, + /// Refusal message (safety classifier), when set. + pub refusal: Option, + pub finish_reason: FinishReason, + /// The client stop sequence that terminated generation, when one + /// did. Feeds Anthropic's `stop_sequence` field. + pub matched_stop: Option, + /// Per-token logprobs (opt-in). Only the OpenAI surface encodes + /// these today. + pub logprobs: Option, +} + +/// Neutral logprob report: sampled token + alternatives. +#[derive(Debug, Clone, PartialEq)] +pub struct ChoiceLogprobs { + pub content: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct TokenLogprob { + pub token: String, + pub logprob: f32, + /// `(token, logprob)` alternatives, highest first. + pub top: Vec<(String, f32)>, +} + +/// Token accounting, including the detail counters the wire formats +/// surface (prefix-cache hits, reasoning tokens) and Atlas's +/// performance extensions. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct Usage { + pub prompt_tokens: usize, + pub completion_tokens: usize, + /// Prompt tokens served from the prefix cache + /// (OpenAI `prompt_tokens_details.cached_tokens`, Anthropic + /// `cache_read_input_tokens`). + pub cached_prompt_tokens: usize, + /// Completion tokens spent inside the thinking block + /// (OpenAI `completion_tokens_details.reasoning_tokens`). + pub reasoning_tokens: usize, + /// Atlas perf extensions; encoders may ignore. + pub time_to_first_token_ms: f64, + pub response_tokens_per_second: f64, +} + +/// Why generation stopped. `Other` preserves unknown engine reasons +/// losslessly (PCND: no silent default). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FinishReason { + Stop, + Length, + ToolCalls, + ContentFilter, + Other(String), +} + +impl FinishReason { + /// Map the engine's finish-reason string (the scheduler's internal + /// vocabulary, which happens to match OpenAI's wire strings). + pub fn from_wire(s: &str) -> Self { + match s { + "stop" => FinishReason::Stop, + "length" => FinishReason::Length, + "tool_calls" => FinishReason::ToolCalls, + "content_filter" => FinishReason::ContentFilter, + other => FinishReason::Other(other.to_string()), + } + } + + /// The canonical wire string (OpenAI-compatible surfaces emit it + /// verbatim; other surfaces map per their own vocabulary). + pub fn as_wire(&self) -> &str { + match self { + FinishReason::Stop => "stop", + FinishReason::Length => "length", + FinishReason::ToolCalls => "tool_calls", + FinishReason::ContentFilter => "content_filter", + FinishReason::Other(s) => s, + } + } +} diff --git a/crates/spark-server/src/ir/stream.rs b/crates/spark-server/src/ir/stream.rs new file mode 100644 index 000000000..9f13854b6 --- /dev/null +++ b/crates/spark-server/src/ir/stream.rs @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Canonical chat IR (streaming direction). The streaming generation +// core emits a flat sequence of these provider-neutral deltas; each API +// surface owns a thin encoder that turns them into its SSE wire format +// (OpenAI chat: `openai::delta_to_chunk_events`). Emission sites never +// build wire chunks directly, so surfaces cannot drift on framing. + +/// One streaming output event from the generation core. Each API +/// surface encodes these into its own SSE wire format. +#[derive(Debug, Clone, PartialEq)] +pub enum StreamDelta { + /// Assistant text fragment. `token_ids` are the exact sampled ids + /// for this fragment (empty unless the client opted in). + Content { text: String, token_ids: Vec }, + /// Reasoning/thinking fragment. + Reasoning { text: String, token_ids: Vec }, + /// A tool call opens: OpenAI-slot `index`, call id, tool name. + ToolCallStart { + index: usize, + id: String, + name: String, + }, + /// Argument JSON fragment for the open tool call at `index`. + ToolCallArgs { + index: usize, + fragment: String, + token_ids: Vec, + }, + /// Refusal message (safety classifier). + Refusal { text: String }, + /// Terminal event: finish reason + final usage. `token_ids` are the + /// residual sampled ids not yet attached to any earlier delta + /// (tokens whose decoded text was buffered or suppressed, tool-call + /// body tokens); carrying them on the terminal event keeps + /// Σ token_ids == `usage.completion_tokens` for clients that opted + /// into `return_token_ids`. + Finish { + reason: super::response::FinishReason, + usage: super::response::Usage, + token_ids: Vec, + }, + /// Stream-level error payload, sent verbatim as SSE data. + Error { message: String }, +} + +/// Boxed stream of deltas — the streaming counterpart of +/// [`super::ChatResponse`]: the pipeline produces it, each surface +/// encoder consumes it. +pub type DeltaStream = std::pin::Pin + Send>>; diff --git a/crates/spark-server/src/ir/tests.rs b/crates/spark-server/src/ir/tests.rs new file mode 100644 index 000000000..be0e20cf7 --- /dev/null +++ b/crates/spark-server/src/ir/tests.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +use super::message::*; + +#[test] +fn role_wire_roundtrip() { + for (role, wire) in [ + (Role::System, "system"), + (Role::User, "user"), + (Role::Assistant, "assistant"), + (Role::Tool, "tool"), + ] { + assert_eq!(role.as_wire(), wire); + assert_eq!(Role::from_wire(wire), Some(role)); + } + // Unknown roles do not silently map to a default. + assert_eq!(Role::from_wire("function"), None); +} + +#[test] +fn text_concatenates_text_parts_in_order_ignoring_images() { + let m = Message { + role: Role::User, + content: vec![ + ContentPart::Text("a".into()), + ContentPart::Image(ImageSource { + data: ImageData::Base64("img".into()), + }), + ContentPart::Text("b".into()), + ], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + }; + // Mirrors the historical `text_parts.join("")` flattening. + assert_eq!(m.text(), "ab"); + assert_eq!(m.image_count(), 1); +} + +#[test] +fn assistant_tool_call_and_reasoning_are_first_class() { + let m = Message { + role: Role::Assistant, + content: vec![ContentPart::Text("hi".into())], + tool_calls: vec![ToolCall { + id: "call_1".into(), + name: "get_weather".into(), + arguments: serde_json::json!({"city": "SF"}), + }], + tool_call_id: None, + name: None, + reasoning: Some(Reasoning { + text: "let me think".into(), + }), + tool_error: false, + }; + assert_eq!(m.role.as_wire(), "assistant"); + assert_eq!(m.tool_calls[0].name, "get_weather"); + assert_eq!(m.tool_calls[0].arguments["city"], "SF"); + assert_eq!(m.reasoning.as_ref().unwrap().text, "let me think"); + assert_eq!(m.image_count(), 0); +} + +#[test] +fn tool_message_carries_error_flag_and_call_id() { + let m = Message { + role: Role::Tool, + content: vec![ContentPart::Text("exit 127".into())], + tool_calls: Vec::new(), + tool_call_id: Some("call_1".into()), + name: Some("bash".into()), + reasoning: None, + tool_error: true, + }; + assert!(m.tool_error); + assert_eq!(m.tool_call_id.as_deref(), Some("call_1")); + assert_eq!(m.text(), "exit 127"); +} diff --git a/crates/spark-server/src/main.rs b/crates/spark-server/src/main.rs index 3ee1d718d..944a81d24 100644 --- a/crates/spark-server/src/main.rs +++ b/crates/spark-server/src/main.rs @@ -31,6 +31,8 @@ mod conversation_store; pub mod grammar; mod halluc_probe; mod hint_injector; +mod ids; +mod ir; mod llmlingua; mod lookback_lens; mod loop_detector; diff --git a/crates/spark-server/src/main_modules/app_state.rs b/crates/spark-server/src/main_modules/app_state.rs index 358678f7a..a62be9621 100644 --- a/crates/spark-server/src/main_modules/app_state.rs +++ b/crates/spark-server/src/main_modules/app_state.rs @@ -74,10 +74,11 @@ pub struct AppState { /// thinking is forced OFF regardless of the request body or the /// model's MODEL.toml default. Wired from `--disable-thinking`. pub disable_thinking: bool, - /// Server-level default chat template kwargs applied when the client + /// Server-level default thinking directive applied when the client /// sends no thinking parameters. Overridden per-request by the request - /// body. Wired from `--default-chat-template-kwargs`. - pub default_chat_template_kwargs: Option, + /// body. Wired from `--default-chat-template-kwargs` (parsed at the + /// CLI edge into the neutral directive). + pub default_thinking: crate::ir::ThinkingDirective, /// Shared in-memory store for stateful Responses API resume /// (`previous_response_id`) and opt-in Chat-Completions storage /// (`store: true`). Bounded LRU + TTL; env-configured at startup. diff --git a/crates/spark-server/src/main_modules/middleware.rs b/crates/spark-server/src/main_modules/middleware.rs index e2f874038..8150530e4 100644 --- a/crates/spark-server/src/main_modules/middleware.rs +++ b/crates/spark-server/src/main_modules/middleware.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use crate::main_modules::AppState; -use crate::{openai, rate_limiter}; +use crate::rate_limiter; /// OpenAI-compatible observability headers. Injects on every `/v1/*` /// response: @@ -34,7 +34,7 @@ pub(crate) async fn openai_observability_middleware( return resp; } let headers = resp.headers_mut(); - let rid = incoming_req_id.unwrap_or_else(|| format!("req_{}", openai::uuid_v4())); + let rid = incoming_req_id.unwrap_or_else(|| format!("req_{}", crate::ids::uuid_v4())); if let Ok(v) = HeaderValue::from_str(&rid) { headers.insert(HeaderName::from_static("x-request-id"), v); } diff --git a/crates/spark-server/src/main_modules/serve.rs b/crates/spark-server/src/main_modules/serve.rs index adbe4e12b..f617a2276 100644 --- a/crates/spark-server/src/main_modules/serve.rs +++ b/crates/spark-server/src/main_modules/serve.rs @@ -646,10 +646,11 @@ pub(crate) async fn serve(mut args: cli::ServeArgs) -> Result<()> { b }, disable_thinking: args.disable_thinking, - default_chat_template_kwargs: args + default_thinking: args .default_chat_template_kwargs - .as_ref() - .and_then(|s| crate::openai::ChatTemplateKwargs::from_json(s)), + .as_deref() + .map(parse_default_thinking) + .unwrap_or_default(), response_store, rate_limiter, conversation_store, @@ -664,6 +665,41 @@ pub(crate) async fn serve(mut args: cli::ServeArgs) -> Result<()> { .await } +/// Parse the vLLM-style `--default-chat-template-kwargs` JSON +/// (`{"enable_thinking":bool,"thinking_budget":u32}`) into the neutral +/// thinking directive. The CLI is its own edge: the JSON shape is parsed +/// here, not in the openai wire module. Mapping matches the +/// `chat_template_kwargs` rung of the request-body ladder: an explicit +/// budget wins, then the enable flag; unknown/empty JSON → Unspecified +/// (with a warning — the old code ignored bad JSON silently). +fn parse_default_thinking(s: &str) -> crate::ir::ThinkingDirective { + use crate::ir::ThinkingDirective; + + #[derive(serde::Deserialize)] + struct Kwargs { + enable_thinking: Option, + thinking_budget: Option, + } + + if s.trim().is_empty() { + return ThinkingDirective::Unspecified; + } + let kw: Kwargs = match serde_json::from_str(s) { + Ok(kw) => kw, + Err(e) => { + tracing::warn!("--default-chat-template-kwargs is not valid JSON ({e}); ignoring"); + return ThinkingDirective::Unspecified; + } + }; + match (kw.thinking_budget, kw.enable_thinking) { + (Some(b), _) if b > 0 => ThinkingDirective::On { budget: Some(b) }, + (Some(_), _) => ThinkingDirective::Off, + (None, Some(true)) => ThinkingDirective::On { budget: None }, + (None, Some(false)) => ThinkingDirective::Off, + (None, None) => ThinkingDirective::Unspecified, + } +} + /// Resolve `--require-auth` / `--auth-tokens-file` / `--auth-token` into an /// optional `AuthConfig`. Validates at startup so misconfigurations fail /// loudly instead of letting an unauthenticated server run silently. diff --git a/crates/spark-server/src/metrics.rs b/crates/spark-server/src/metrics.rs index 068a623b2..56c762c5c 100644 --- a/crates/spark-server/src/metrics.rs +++ b/crates/spark-server/src/metrics.rs @@ -40,19 +40,6 @@ lazy_static! { &["verdict", "channel", "spinning"] ).unwrap(); - // ── Anthropic translation-drift counter (P5.1) ── - // - // Increments whenever the Anthropic→OpenAI translator produces a - // round-trip diff against the original Anthropic shape. Diffs - // indicate translation bugs that compound across long agentic - // sessions. Logging the actual diff is gated behind the - // ATLAS_DEBUG_TRANSLATION_DRIFT env var (anthropic.rs). - pub static ref ANTHROPIC_TRANSLATION_DRIFTS: IntCounter = - register_int_counter!( - "atlas_anthropic_translation_drifts_total", - "Anthropic ↔ OpenAI translator round-trip mismatches detected" - ).unwrap(); - // ── Speculative-decode telemetry (A.2 EASD scaffolding) ── // // Per-K acceptance counters. Enables measuring baseline accept diff --git a/crates/spark-server/src/openai/annotations.rs b/crates/spark-server/src/openai/annotations.rs index d346d67d9..a63e92025 100644 --- a/crates/spark-server/src/openai/annotations.rs +++ b/crates/spark-server/src/openai/annotations.rs @@ -46,230 +46,34 @@ pub struct UrlCitation { pub title: String, } -/// Scan `content` for http(s) URLs and emit OpenAI-compatible -/// `url_citation` annotations. -/// -/// The extractor handles three shapes: -/// - Markdown links `[title](url)` — title from the `[...]` text. -/// - Bare URLs — title is the URL itself. -/// - URLs inside fenced code blocks (triple backticks) or inline code (`` `...` ``) -/// are **skipped** — illustrative code, not citations. This prevents -/// false positives on model output like `curl https://example.com`. -/// -/// Returns `None` when no URLs remain so the wire format stays identical -/// for non-web-search responses. -pub fn extract_url_annotations(content: &str) -> Option> { - let mut out: Vec = Vec::new(); - let masked = mask_code_spans(content); - // First pass: markdown links. We scan the masked copy so the - // start/end indices we record match the original `content` exactly. - let mut scan = 0usize; - while scan < masked.len() { - let rest = &masked[scan..]; - let Some(lb_rel) = rest.find('[') else { break }; - let lb = scan + lb_rel; - // Find the matching `]` then immediate `(`. - let after_lb = &masked[lb + 1..]; - let Some(rb_rel) = after_lb.find(']') else { - scan = lb + 1; - continue; - }; - let rb = lb + 1 + rb_rel; - if masked.as_bytes().get(rb + 1) != Some(&b'(') { - scan = rb + 1; - continue; - } - // Find the matching close paren that respects nested `()` pairs - // inside the URL — Wikipedia and other URLs commonly contain - // parentheses (e.g. `https://en.wikipedia.org/wiki/Foo_(bar)`). - // The bare-find shortcut would terminate at the first `)`, - // truncating the URL. - let after_paren = &masked[rb + 2..]; - let Some(rparen_rel) = balanced_paren_close(after_paren) else { - scan = rb + 2; - continue; - }; - let rparen = rb + 2 + rparen_rel; - let title = &content[lb + 1..rb]; - let target = content[rb + 2..rparen].trim(); - if (target.starts_with("http://") || target.starts_with("https://")) - && target.len() > "https://".len() - { - out.push(Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: rb + 2, - end_index: rparen, - url: target.to_string(), - title: title.to_string(), - }, - }); - } - scan = rparen + 1; - } - - // Second pass: bare URLs in regions that aren't masked AND aren't - // already covered by a markdown link. - let covered: Vec<(usize, usize)> = out - .iter() - .map(|a| match a { - Annotation::UrlCitation { - url_citation: - UrlCitation { - start_index, - end_index, - .. - }, - } => (*start_index, *end_index), - }) - .collect(); - let mut i = 0usize; - while i < masked.len() { - let rest = &masked[i..]; - let Some(off) = rest.find("http") else { break }; - let abs_start = i + off; - let tail_masked = &masked[abs_start..]; - let is_url = tail_masked.starts_with("http://") || tail_masked.starts_with("https://"); - if !is_url { - i = abs_start + 4; - continue; - } - let tail = &content[abs_start..]; - let end_rel = tail - .find(|c: char| { - c.is_whitespace() || matches!(c, ']' | '}' | '"' | '<' | '>' | '`' | '\\') - }) - .unwrap_or(tail.len()); - let mut raw = &tail[..end_rel]; - // Strip trailing sentence punctuation and unmatched close-parens / - // markdown emphasis markers. Parens match pairs so URLs like - // Wikipedia's `https://en.wikipedia.org/wiki/Foo_(bar)` survive. - while let Some(last) = raw.chars().last() { - let strip = match last { - '.' | ',' | ';' | ':' | '!' | '?' | '*' | '_' => true, - ')' => { - let opens = raw.matches('(').count(); - let closes = raw.matches(')').count(); - closes > opens - } - _ => false, - }; - if !strip { - break; - } - raw = &raw[..raw.len() - last.len_utf8()]; - } - if raw.len() > "https://".len() { - let start = abs_start; - let end = abs_start + raw.len(); - let overlaps = covered.iter().any(|(s, e)| start < *e && end > *s); - if !overlaps { - out.push(Annotation::UrlCitation { - url_citation: UrlCitation { - start_index: start, - end_index: end, - url: raw.to_string(), - title: raw.to_string(), - }, - }); - } - } - i = abs_start + end_rel.max(1); - } - // Sort by start index so downstream consumers see annotations in - // document order regardless of which pass emitted them. - out.sort_by_key(|a| match a { +impl From for Annotation { + fn from(c: crate::citation::Citation) -> Self { Annotation::UrlCitation { - url_citation: UrlCitation { start_index, .. }, - } => *start_index, - }); - if out.is_empty() { None } else { Some(out) } -} - -/// Return a copy of `content` where the insides of fenced code blocks -/// (```) and inline code spans (`) are replaced with ASCII spaces while -/// preserving byte offsets and UTF-8 validity. Used so the URL scan can -/// skip over code regions without rebuilding indices. -/// -/// Return the byte offset of the `)` that balances the implicit `(` -/// before the slice (i.e. the URL's matching close), or None if no -/// balanced close exists. Handles nested `()` pairs that appear in -/// real URLs (Wikipedia article slugs, GitHub anchors, etc). -fn balanced_paren_close(s: &str) -> Option { - let bytes = s.as_bytes(); - let mut depth: i32 = 0; - for (i, &b) in bytes.iter().enumerate() { - match b { - b'(' => depth += 1, - b')' => { - if depth == 0 { - return Some(i); - } - depth -= 1; - } - _ => {} + url_citation: UrlCitation { + start_index: c.start_index, + end_index: c.end_index, + url: c.url, + title: c.title, + }, } } - None } -/// We walk char-by-char to keep multi-byte characters intact: each non- -/// newline char inside a code region becomes N ASCII spaces where N is -/// the char's UTF-8 byte length. Newlines are kept so line-based parsers -/// still see the structure. -fn mask_code_spans(content: &str) -> String { - let bytes = content.as_bytes(); - let mut out: Vec = bytes.to_vec(); - - // Blank bytes [start, end) by char: non-newline chars → ASCII spaces - // of the same byte length, newlines preserved. Char boundaries taken - // from the original content (which is guaranteed UTF-8). - fn blank(out: &mut [u8], content: &str, start: usize, end: usize) { - let region = &content[start..end]; - let mut cursor = start; - for ch in region.chars() { - let len = ch.len_utf8(); - if ch != '\n' { - for b in out.iter_mut().skip(cursor).take(len) { - *b = b' '; - } - } - cursor += len; - } +/// Bare/markdown URL scan (`citation::extract_url_citations`) in the +/// OpenAI wire shape. `None` when no URLs remain so the wire format +/// stays identical for non-web-search responses. +pub fn extract_url_annotations(content: &str) -> Option> { + let cits = crate::citation::extract_url_citations(content); + if cits.is_empty() { + None + } else { + Some(cits.into_iter().map(Annotation::from).collect()) } +} - let mut i = 0usize; - while i < bytes.len() { - if bytes[i..].starts_with(b"```") { - let after = i + 3; - let rest = &content[after..]; - let end = match rest.find("```") { - Some(r) => after + r + 3, - None => bytes.len(), - }; - blank(&mut out, content, i, end); - i = end; - continue; - } - if bytes[i] == b'`' { - let after = i + 1; - let rest = &content[after..]; - let end = match rest.find('`') { - Some(r) => after + r + 1, - None => bytes.len(), - }; - blank(&mut out, content, i, end); - i = end; - continue; - } - // Step one whole UTF-8 codepoint. - let step = match bytes[i] { - 0x00..=0x7f => 1, - 0xc0..=0xdf => 2, - 0xe0..=0xef => 3, - 0xf0..=0xf7 => 4, - _ => 1, - }; - i += step; - } - String::from_utf8(out).expect("mask preserves UTF-8") +/// Combined bare + structured citation extraction +/// (`citation::merged_citations`) in the OpenAI wire shape. +pub fn merged_annotations(content: &str) -> Option> { + crate::citation::merged_citations(content) + .map(|cits| cits.into_iter().map(Annotation::from).collect()) } diff --git a/crates/spark-server/src/openai/chat_message.rs b/crates/spark-server/src/openai/chat_message.rs index a5f277aa3..b489c7c55 100644 --- a/crates/spark-server/src/openai/chat_message.rs +++ b/crates/spark-server/src/openai/chat_message.rs @@ -91,30 +91,9 @@ impl IncomingMessage { } .to_string(); let content_val = obj.get("content")?; - let mut text = String::new(); - match content_val { - serde_json::Value::String(s) => text.push_str(s), - serde_json::Value::Array(parts) => { - for part in parts { - if let Some(po) = part.as_object() { - let part_kind = - po.get("type").and_then(|t| t.as_str()).unwrap_or(""); - if matches!(part_kind, "input_text" | "output_text" | "text") - && let Some(t) = po.get("text").and_then(|t| t.as_str()) - { - text.push_str(t); - } - } - } - } - _ => {} - } Some(Self { role, - content: ParsedContent { - text, - images: Vec::new(), - }, + content: responses_content_to_parsed(content_val), tool_calls: None, tool_call_id: None, name: None, @@ -161,10 +140,34 @@ impl IncomingMessage { .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); - let output_text = match obj.get("output") { - Some(serde_json::Value::String(s)) => s.clone(), - Some(other) => other.to_string(), - None => String::new(), + let output = match obj.get("output") { + Some(serde_json::Value::String(s)) => ParsedContent { + text: s.clone(), + images: Vec::new(), + }, + // Structured output parts (`output_text` / `input_image` / + // …): carry text AND images so a screenshot returned by a + // tool reaches the vision encoder — parity with the + // Anthropic tool_result path and chat-completions + // `role:"tool"` array content (#165). Arrays that contain + // no recognizable parts keep the old stringified-JSON + // behavior so out-of-spec payloads still reach the model. + Some(arr @ serde_json::Value::Array(_)) => { + let parsed = responses_content_to_parsed(arr); + if parsed.text.is_empty() && parsed.images.is_empty() { + ParsedContent { + text: arr.to_string(), + images: Vec::new(), + } + } else { + parsed + } + } + Some(other) => ParsedContent { + text: other.to_string(), + images: Vec::new(), + }, + None => ParsedContent::default(), }; let name = obj .get("name") @@ -173,10 +176,7 @@ impl IncomingMessage { .to_string(); Some(Self { role: "tool".to_string(), - content: ParsedContent { - text: output_text, - images: Vec::new(), - }, + content: output, tool_calls: None, tool_call_id: Some(call_id), name: if name.is_empty() { None } else { Some(name) }, @@ -194,6 +194,51 @@ impl IncomingMessage { } } +/// Flatten a Responses-API content value (string, or array of +/// `input_text`/`output_text`/`input_image`/… parts) into text + image +/// lists. Shared by `message` items and `function_call_output` items so +/// images are carried on both — the pipeline collects them into the +/// vision encoder. +fn responses_content_to_parsed(v: &serde_json::Value) -> ParsedContent { + let mut text = String::new(); + let mut images: Vec = Vec::new(); + match v { + serde_json::Value::String(s) => text.push_str(s), + serde_json::Value::Array(parts) => { + for part in parts { + if let Some(po) = part.as_object() { + let part_kind = po.get("type").and_then(|t| t.as_str()).unwrap_or(""); + if matches!(part_kind, "input_text" | "output_text" | "text") + && let Some(t) = po.get("text").and_then(|t| t.as_str()) + { + text.push_str(t); + } else if matches!(part_kind, "input_image" | "image_url" | "image") + && let Some(url) = responses_image_url(po) + { + images.push(url); + } + } + } + } + _ => {} + } + ParsedContent { text, images } +} + +/// Extract the image URL / data-URI from a Responses `input_image` +/// content part. Accepts both the flat string form +/// (`"image_url": "..."`) and the nested object form +/// (`"image_url": {"url": "..."}`). +fn responses_image_url(po: &serde_json::Map) -> Option { + match po.get("image_url") { + Some(serde_json::Value::String(s)) => Some(s.clone()), + Some(serde_json::Value::Object(o)) => { + o.get("url").and_then(|v| v.as_str()).map(|s| s.to_string()) + } + _ => None, + } +} + fn deserialize_message_content<'de, D>(d: D) -> Result where D: serde::Deserializer<'de>, diff --git a/crates/spark-server/src/openai/chat_request.rs b/crates/spark-server/src/openai/chat_request.rs index 5675e1c50..2092f217b 100644 --- a/crates/spark-server/src/openai/chat_request.rs +++ b/crates/spark-server/src/openai/chat_request.rs @@ -3,6 +3,8 @@ use serde::Deserialize; use super::*; +use crate::api::inference_types::RepetitionDetectionParams; +use crate::ir::ThinkingDirective; /// Chat completion request (subset of OpenAI spec). #[derive(Debug, Deserialize)] @@ -196,27 +198,6 @@ pub struct ChatCompletionRequest { pub reasoning_effort: Option, } -/// Per-request override for the vLLM-anchored token-loop detector. -/// -/// Mirrors vLLM's `RepetitionDetectionParams` -/// (`vllm/sampling_params.py:111-144`): -/// - `min_pattern_size` → smallest pattern length (in tokens) to consider -/// - `max_pattern_size` → largest pattern length to consider -/// - `min_count` → number of end-anchored back-to-back repeats that -/// constitute a "loop" -/// -/// When attached to a request, these override the boot-global -/// thresholds (`CONTENT_LOOP_PERIOD_MIN` / `_MAX` / -/// `CONTENT_LOOP_MIN_REPEATS` and the thinking-loop equivalents) for -/// that single sequence's content-loop and thinking-loop detectors. -#[derive(Debug, Clone, Copy, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct RepetitionDetectionParams { - pub min_pattern_size: u32, - pub max_pattern_size: u32, - pub min_count: u32, -} - /// Stream options (OpenAI-compatible). #[derive(Debug, Clone, Copy, Deserialize, Default)] #[serde(default)] @@ -281,104 +262,75 @@ pub struct ReasoningConfig { pub effort: Option, } -/// vLLM-style chat template kwargs. +/// vLLM-style chat template kwargs (request-body wire field). The +/// server-level `--default-chat-template-kwargs` CLI flag is parsed at +/// the CLI edge (`main_modules/serve.rs`), not through this type. #[derive(Debug, Clone, Deserialize)] pub struct ChatTemplateKwargs { pub enable_thinking: Option, pub thinking_budget: Option, } -impl ChatTemplateKwargs { - /// Parse from a JSON string. Returns `None` if parsing fails or string is empty. - pub fn from_json(s: &str) -> Option { - if s.trim().is_empty() { - return None; - } - serde_json::from_str(s).ok() - } -} - /// Default thinking budget when thinking is enabled but no explicit budget set. /// 256 tokens is enough for the model to plan without overthinking — longer /// budgets waste decode throughput on reasoning that rarely improves output. const DEFAULT_THINKING_BUDGET: u32 = 256; impl ChatCompletionRequest { - /// Resolve thinking parameters from all supported request-body formats - /// into a single `(enable_thinking: bool, thinking_budget: Option)` - /// pair. The client's per-request choice always wins over the model - /// default; the model default (from MODEL.toml `[behavior].thinking_default`) - /// is used only when the client sends NO thinking parameter at all. - /// - /// The `--disable-thinking` CLI flag is a higher-priority kill switch - /// applied by the caller (api.rs / anthropic.rs) — this function does - /// not know about it. + /// Per-request override for the vLLM-anchored token-loop detector. + /// `None` = use the boot-global watchdog parameters. + pub fn repetition_detection(&self) -> Option { + self.repetition_detection + } + + /// Resolve the client's thinking intent from all supported + /// request-body formats into the neutral [`ThinkingDirective`]. + /// This is the OpenAI-edge half of thinking resolution; the model + /// default (MODEL.toml `[behavior].thinking_default`), the server + /// default directive, and the `--disable-thinking` kill switch are + /// folded in later by `api/chat/thinking.rs`, which never sees the + /// wire fields. /// /// Request-body priority (highest to lowest): /// 1. `thinking.budget_tokens` (Anthropic) — explicit budget /// 2. `thinking_token_budget` (vLLM PR) — explicit budget /// 3. `reasoning.effort` (OpenAI) — mapped to budget /// 4. `chat_template_kwargs` (vLLM stable) — enable/disable + optional budget - /// 5. `enable_thinking` (Atlas legacy) — boolean with default budget - /// 6. `model_default` argument (from MODEL.toml) — model-specific fallback + /// 5. `enable_thinking` (Atlas legacy) — boolean /// - /// Returns `true` if any of channels 1-5 carried an explicit thinking - /// intent from the client (i.e. the resolved value did NOT fall through - /// to `model_default`). Callers use this to decide whether a - /// server-side policy (e.g. `thinking_in_tools=false`) is allowed to - /// override the model default OR must respect the explicit request. - /// Per-request override for the vLLM-anchored token-loop detector. - /// `None` = use the boot-global watchdog parameters. - pub fn repetition_detection(&self) -> Option { - self.repetition_detection - } - - pub fn thinking_explicitly_requested(&self) -> bool { - if self.thinking.is_some() { - return true; - } - if self.thinking_token_budget.is_some() { - return true; - } - if let Some(ref rc) = self.reasoning - && rc.effort.is_some() - { - return true; - } - if let Some(ref kw) = self.chat_template_kwargs - && (kw.thinking_budget.is_some() || kw.enable_thinking.is_some()) - { - return true; - } - if self.enable_thinking { - return true; - } - false - } - - pub fn resolve_thinking(&self, model_default: bool) -> (bool, Option) { - // 1. Anthropic: thinking.budget_tokens + /// No channel present → [`ThinkingDirective::Unspecified`] (the old + /// `thinking_explicitly_requested() == false`). + pub fn client_thinking_directive(&self) -> ThinkingDirective { + // 1. Anthropic: thinking.budget_tokens / thinking.type if let Some(ref tc) = self.thinking { if let Some(ref t) = tc.thinking_type && t == "disabled" { - return (false, Some(0)); + return ThinkingDirective::Off; } if let Some(budget) = tc.budget_tokens { - return (true, Some(budget)); + return ThinkingDirective::On { + budget: Some(budget), + }; } // Anthropic "adaptive" / thinking object with no explicit budget // means "think as long as needed" — defer to the per-model - // `max_thinking_budget` (None), NOT the conservative 256-token - // DEFAULT_THINKING_BUDGET. A hard 256 cut force-injects - // mid-reasoning on agentic turns and wrecks tool selection. - // Mirrors the step-5 enable_thinking path below. - return (true, None); + // `max_thinking_budget` (budget: None), NOT the conservative + // 256-token DEFAULT_THINKING_BUDGET. A hard 256 cut force-injects + // mid-reasoning on agentic turns and wrecks tool + // selection. Mirrors the step-5 enable_thinking path below. + return ThinkingDirective::On { budget: None }; } // 2. vLLM PR: thinking_token_budget if let Some(budget) = self.thinking_token_budget { - return (budget > 0, Some(budget)); + return if budget > 0 { + ThinkingDirective::On { + budget: Some(budget), + } + } else { + ThinkingDirective::Off + }; } // 3. OpenAI: reasoning.effort @@ -394,47 +346,55 @@ impl ChatCompletionRequest { "xhigh" | "max" => 1024, _ => DEFAULT_THINKING_BUDGET, }; - return (budget > 0, Some(budget)); + return if budget > 0 { + ThinkingDirective::On { + budget: Some(budget), + } + } else { + ThinkingDirective::Off + }; } // 4. vLLM stable: chat_template_kwargs if let Some(ref kwargs) = self.chat_template_kwargs { if let Some(budget) = kwargs.thinking_budget { - return (budget > 0, Some(budget)); + return if budget > 0 { + ThinkingDirective::On { + budget: Some(budget), + } + } else { + ThinkingDirective::Off + }; } if let Some(enabled) = kwargs.enable_thinking { - // enable_thinking via chat_template_kwargs (incl. the server's - // --default-chat-template-kwargs '{"enable_thinking":true}'): - // defer the budget to the per-model max_thinking_budget (None) - // rather than the conservative 256-token DEFAULT — same rationale - // as the legacy/model-default branches below. Without this, that - // server default silently capped EVERY request's thinking at 256. - return (enabled, if enabled { None } else { Some(0) }); + // enable_thinking via chat_template_kwargs: defer the budget + // to the per-model max_thinking_budget (None) rather than the + // conservative 256-token DEFAULT — same rationale as the + // legacy branch below. Without this, a server default of + // '{"enable_thinking":true}' silently capped EVERY request's + // thinking at 256. + return if enabled { + ThinkingDirective::On { budget: None } + } else { + ThinkingDirective::Off + }; } } // 5. Atlas legacy: enable_thinking boolean in the request body. // Only honored when explicitly true — a false value (including the - // serde default when the field is absent) falls through to the - // MODEL.toml default so clients that don't know about this flag - // inherit the model's design intent instead of silently opting out. - // Returns `None` for the budget so `api/chat/thinking.rs` falls - // back to `state.behavior.max_thinking_budget` (the per-model - // MODEL.toml cap) instead of the conservative - // DEFAULT_THINKING_BUDGET — opencode-style clients otherwise - // hit a 256-token mid-sentence cut on thinking-tier models. + // serde default when the field is absent) falls through to + // Unspecified so clients that don't know about this flag inherit + // the model's design intent instead of silently opting out. + // `budget: None` so `api/chat/thinking.rs` falls back to the + // per-model MODEL.toml cap instead of the conservative + // DEFAULT_THINKING_BUDGET — opencode-style clients otherwise hit + // a 256-token mid-sentence cut on thinking-tier models. if self.enable_thinking { - return (true, None); + return ThinkingDirective::On { budget: None }; } - // 6. Model default from MODEL.toml [behavior].thinking_default. - // Same `None` rationale as step 5 — defer to the per-model - // `max_thinking_budget` rather than the conservative default. - if model_default { - (true, None) - } else { - (false, None) - } + ThinkingDirective::Unspecified } } diff --git a/crates/spark-server/src/openai/completions.rs b/crates/spark-server/src/openai/completions.rs index c17dc11e1..f72e58084 100644 --- a/crates/spark-server/src/openai/completions.rs +++ b/crates/spark-server/src/openai/completions.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use super::*; +use crate::api::inference_types::RepetitionDetectionParams; // ── Legacy /v1/completions types (OpenAI standard) ── diff --git a/crates/spark-server/src/openai/encode.rs b/crates/spark-server/src/openai/encode.rs new file mode 100644 index 000000000..ef8ae0211 --- /dev/null +++ b/crates/spark-server/src/openai/encode.rs @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Encoder: canonical `ir::ChatResponse` → OpenAI Chat Completions wire +// JSON (non-streaming). Owns everything wire-specific the core used to +// do at finalize time: id/`created` formatting, URL-citation +// annotations, `service_tier`/`metadata` echoes, `store: true` +// persistence, and the `--dump` response capture. + +use axum::response::{IntoResponse, Json, Response}; + +use crate::AppState; + +use super::{ + ChatChoice, ChatCompletionResponse, ChatMessage, ChoiceLogprobs, CompletionTokensDetails, + PromptTokensDetails, TokenLogprobInfo, TopLogprob, Usage, merged_annotations, +}; + +/// Serialize the response IR for the `/v1/chat/completions` surface. +pub(crate) fn encode_chat_response( + state: &AppState, + ir: crate::ir::ChatResponse, + echo: &crate::api::ResponseEcho, + dump_seq: Option, +) -> Response { + let usage = Usage { + prompt_tokens: ir.usage.prompt_tokens, + completion_tokens: ir.usage.completion_tokens, + total_tokens: ir.usage.prompt_tokens + ir.usage.completion_tokens, + prompt_tokens_details: Some(PromptTokensDetails { + cached_tokens: ir.usage.cached_prompt_tokens, + audio_tokens: 0, + }), + completion_tokens_details: Some(CompletionTokensDetails { + reasoning_tokens: ir.usage.reasoning_tokens, + audio_tokens: 0, + accepted_prediction_tokens: 0, + rejected_prediction_tokens: 0, + }), + time_to_first_token_ms: ir.usage.time_to_first_token_ms, + response_tokens_per_second: ir.usage.response_tokens_per_second, + }; + + let choices: Vec = ir + .choices + .into_iter() + .map(|c| { + let tool_calls = if c.tool_calls.is_empty() { + None + } else { + Some( + c.tool_calls + .into_iter() + .map(|tc| crate::tool_parser::ToolCall { + id: tc.id, + call_type: "function".to_string(), + function: crate::tool_parser::FunctionCall { + name: tc.name, + arguments: tc.arguments.to_string(), + }, + }) + .collect(), + ) + }; + // URL-citation annotations are derived from the FINAL + // content (post tool-XML strip / refusal null), so offsets + // always index into the text the client actually receives. + let annotations = c.content.as_deref().and_then(merged_annotations); + ChatChoice { + index: c.index, + message: ChatMessage { + role: "assistant".to_string(), + reasoning_content: c.reasoning, + content: c.content, + tool_calls, + annotations, + refusal: c.refusal, + }, + finish_reason: c.finish_reason.as_wire().to_string(), + logprobs: c.logprobs.map(encode_logprobs), + } + }) + .collect(); + + let completion_id = format!("chatcmpl-{}", ir.id); + let completion = ChatCompletionResponse { + id: completion_id.clone(), + object: "chat.completion".to_string(), + created: ir.created, + model: ir.model.clone(), + system_fingerprint: Some("fp_atlas".to_string()), + choices, + usage, + service_tier: echo.service_tier.clone(), + metadata: echo.metadata.clone(), + }; + + // Completion-storage backend: when `store: true`, persist the + // serialized body so a subsequent GET /v1/chat/completions/{id} + // can return it. Bounded LRU + TTL in response_store. + if echo.store + && let Ok(body) = serde_json::to_value(&completion) + { + state + .response_store + .insert(crate::response_store::StoredEntry { + id: completion_id, + kind: crate::response_store::StoredKind::ChatCompletion, + model: ir.model, + created_at: ir.created, + messages: Vec::new(), + body, + last_access: std::time::Instant::now(), + }); + } + + // --dump: record the non-streaming response body, correlated with + // the request via the shared seq number. + if let (Some(seq), Some(dump)) = (dump_seq, state.dump_writer.as_ref()) { + dump.dump_response("/v1/chat/completions", seq, &completion, false); + } + + Json(completion).into_response() +} + +fn encode_logprobs(lp: crate::ir::ChoiceLogprobs) -> ChoiceLogprobs { + ChoiceLogprobs { + content: lp + .content + .into_iter() + .map(|t| TokenLogprobInfo { + token: t.token, + logprob: t.logprob, + bytes: None, + top_logprobs: t + .top + .into_iter() + .map(|(token, logprob)| TopLogprob { + token, + logprob, + bytes: None, + }) + .collect(), + }) + .collect(), + } +} diff --git a/crates/spark-server/src/openai/encode_stream.rs b/crates/spark-server/src/openai/encode_stream.rs new file mode 100644 index 000000000..bbbd4a90c --- /dev/null +++ b/crates/spark-server/src/openai/encode_stream.rs @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Encoder: neutral streaming deltas (`ir::StreamDelta`) → OpenAI +// `chat.completion.chunk` SSE events. The single seam where the +// streaming chat core's provider-neutral output becomes OpenAI wire +// format, built with the exact same `ChatCompletionChunk` constructors +// the emission sites used to call — the wire bytes are unchanged. + +use axum::response::sse::Event; + +use crate::ir::StreamDelta; + +use super::{ChatCompletionChunk, CompletionTokensDetails, PromptTokensDetails, Usage}; + +/// Encode one neutral delta into its OpenAI SSE event(s). +/// +/// `include_usage` is the request's `stream_options.include_usage` +/// echo: the terminal framing decision (separate usage-only chunk +/// before a usage-less finish chunk vs a single finish chunk carrying +/// usage) belongs to this wire format, not to the generation core. +pub(crate) fn delta_to_chunk_events( + d: &StreamDelta, + model: &str, + id: &str, + include_usage: bool, +) -> Vec { + delta_to_payloads(d, model, id, include_usage) + .into_iter() + .map(|payload| Event::default().data(payload)) + .collect() +} + +/// The `data:` payload strings for one delta, in emission order. Split +/// from [`delta_to_chunk_events`] so tests can assert the exact chunk +/// JSON (axum's `Event` is write-only). +fn delta_to_payloads(d: &StreamDelta, model: &str, id: &str, include_usage: bool) -> Vec { + match d { + StreamDelta::Content { text, token_ids } => vec![chunk_json( + ChatCompletionChunk::content_chunk(model, id, text.clone()) + .with_token_ids(token_ids.clone()), + )], + StreamDelta::Reasoning { text, token_ids } => vec![chunk_json( + ChatCompletionChunk::reasoning_chunk(model, id, text.clone()) + .with_token_ids(token_ids.clone()), + )], + StreamDelta::ToolCallStart { + index, + id: call_id, + name, + } => { + // Rebuild the argument shape `tool_call_start_chunk` reads: + // `tc.id`, `tc.call_type`, `tc.function.name` (arguments are + // always emitted empty on the start chunk). Every core + // emission site produces `call_type == "function"` — the + // parser pipeline hardcodes it — so the delta carries only + // id + name. + let tc = crate::tool_parser::ToolCall { + id: call_id.clone(), + call_type: "function".to_string(), + function: crate::tool_parser::FunctionCall { + name: name.clone(), + arguments: String::new(), + }, + }; + vec![chunk_json(ChatCompletionChunk::tool_call_start_chunk( + model, id, &tc, *index, + ))] + } + StreamDelta::ToolCallArgs { + index, + fragment, + token_ids, + } => vec![chunk_json( + ChatCompletionChunk::tool_call_args_fragment(model, id, *index, fragment) + .with_token_ids(token_ids.clone()), + )], + StreamDelta::Refusal { text } => vec![chunk_json(ChatCompletionChunk::refusal_chunk( + model, + id, + text.clone(), + ))], + StreamDelta::Finish { + reason, + usage, + token_ids, + } => { + let wire_usage = wire_usage(usage); + if include_usage { + // OpenAI `stream_options.include_usage=true` framing: + // the usage-only chunk (`choices: []`) precedes the + // final `finish_reason` chunk (usage omitted). Residual + // token ids ride the final chunk, keeping + // Σ token_ids == completion_tokens. + vec![ + chunk_json(ChatCompletionChunk::usage_only_chunk(model, id, wire_usage)), + chunk_json( + ChatCompletionChunk::final_chunk_no_usage(model, id, reason.as_wire()) + .with_token_ids(token_ids.clone()), + ), + ] + } else { + vec![chunk_json( + ChatCompletionChunk::done_chunk(model, id, reason.as_wire(), wire_usage) + .with_token_ids(token_ids.clone()), + )] + } + } + // The core hands over a wire-ready error envelope; forward it + // verbatim as SSE data (matching the historical error arm). + StreamDelta::Error { message } => vec![message.clone()], + } +} + +fn chunk_json(chunk: ChatCompletionChunk) -> String { + serde_json::to_string(&chunk).unwrap_or_default() +} + +/// `ir::Usage` → OpenAI wire usage, field for field the construction +/// the streaming Done arm performed historically (`total_tokens` is +/// prompt + completion; audio/prediction counters pinned to 0). +fn wire_usage(u: &crate::ir::Usage) -> Usage { + Usage { + prompt_tokens: u.prompt_tokens, + completion_tokens: u.completion_tokens, + total_tokens: u.prompt_tokens + u.completion_tokens, + prompt_tokens_details: Some(PromptTokensDetails { + cached_tokens: u.cached_prompt_tokens, + audio_tokens: 0, + }), + completion_tokens_details: Some(CompletionTokensDetails { + reasoning_tokens: u.reasoning_tokens, + audio_tokens: 0, + accepted_prediction_tokens: 0, + rejected_prediction_tokens: 0, + }), + time_to_first_token_ms: u.time_to_first_token_ms, + response_tokens_per_second: u.response_tokens_per_second, + } +} + +/// Full OpenAI SSE response for the `/v1/chat/completions` surface: +/// role prologue + encoded deltas + `[DONE]`, with keep-alive. Mints +/// the wire chunk id (all chunks of one response share it). +pub(crate) fn encode_sse_response( + deltas: crate::ir::DeltaStream, + model: String, + include_usage: bool, +) -> axum::response::Response { + use axum::response::IntoResponse; + use axum::response::sse::{Event, KeepAlive, Sse}; + use futures::StreamExt; + + let chunk_id = super::new_chunk_id(); + let role_chunk = super::ChatCompletionChunk::role_chunk(&model, &chunk_id); + let role_json = serde_json::to_string(&role_chunk).unwrap_or_default(); + let role_event = futures::stream::once(async move { + Ok::<_, std::convert::Infallible>(Event::default().data(role_json)) + }); + let done_event = futures::stream::once(async { + Ok::<_, std::convert::Infallible>(Event::default().data("[DONE]")) + }); + let token_stream = deltas.flat_map(move |d| { + let events: Vec> = + delta_to_chunk_events(&d, &model, &chunk_id, include_usage) + .into_iter() + .map(Ok) + .collect(); + futures::stream::iter(events) + }); + let full_stream = role_event.chain(token_stream).chain(done_event); + Sse::new(full_stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::{FinishReason, StreamDelta}; + + /// Parse a chunk payload and zero the only wall-clock field so a + /// payload and its expected constructor twin compare equal even + /// when their `unix_timestamp()` calls straddle a second boundary. + fn norm(payload: &str) -> serde_json::Value { + let mut v: serde_json::Value = serde_json::from_str(payload).expect("valid chunk JSON"); + v["created"] = serde_json::json!(0); + v + } + + #[test] + fn content_delta_encodes_via_content_chunk() { + let d = StreamDelta::Content { + text: "hi".into(), + token_ids: Vec::new(), + }; + let p = delta_to_payloads(&d, "m", "chatcmpl-x", false); + assert_eq!(p.len(), 1); + assert!(p[0].contains("\"content\":\"hi\""), "payload: {}", p[0]); + assert!( + !p[0].contains("token_ids"), + "opt-out payload must omit token_ids: {}", + p[0] + ); + let expected = serde_json::to_string(&ChatCompletionChunk::content_chunk( + "m", + "chatcmpl-x", + "hi".to_string(), + )) + .unwrap(); + assert_eq!(norm(&p[0]), norm(&expected)); + + // Opted-in ids are stamped onto the chunk's first choice. + let d = StreamDelta::Content { + text: "hi".into(), + token_ids: vec![1, 2], + }; + let p = delta_to_payloads(&d, "m", "chatcmpl-x", false); + assert!(p[0].contains("\"token_ids\":[1,2]"), "payload: {}", p[0]); + } + + #[test] + fn reasoning_delta_encodes_via_reasoning_chunk() { + let d = StreamDelta::Reasoning { + text: "mull".into(), + token_ids: vec![9], + }; + let p = delta_to_payloads(&d, "m", "id-1", false); + assert_eq!(p.len(), 1); + assert!( + p[0].contains("\"reasoning_content\":\"mull\""), + "payload: {}", + p[0] + ); + let expected = serde_json::to_string( + &ChatCompletionChunk::reasoning_chunk("m", "id-1", "mull".to_string()) + .with_token_ids(vec![9]), + ) + .unwrap(); + assert_eq!(norm(&p[0]), norm(&expected)); + } + + #[test] + fn tool_call_start_and_args_deltas_encode_openai_tool_chunks() { + let start = StreamDelta::ToolCallStart { + index: 3, + id: "call_abc".into(), + name: "get_weather".into(), + }; + let p = delta_to_payloads(&start, "m", "id-1", false); + assert_eq!(p.len(), 1); + // Per OpenAI streaming spec the start chunk carries role + + // id/type/name with empty arguments, at the delta's tool slot. + assert!(p[0].contains("\"role\":\"assistant\""), "payload: {}", p[0]); + assert!( + p[0].contains( + "\"tool_calls\":[{\"index\":3,\"id\":\"call_abc\",\"type\":\"function\",\"function\":{\"name\":\"get_weather\",\"arguments\":\"\"}}]" + ), + "payload: {}", + p[0] + ); + + let args = StreamDelta::ToolCallArgs { + index: 3, + fragment: "{\"city\":".into(), + token_ids: Vec::new(), + }; + let p = delta_to_payloads(&args, "m", "id-1", false); + assert_eq!(p.len(), 1); + let expected = serde_json::to_string(&ChatCompletionChunk::tool_call_args_fragment( + "m", + "id-1", + 3, + "{\"city\":", + )) + .unwrap(); + assert_eq!(norm(&p[0]), norm(&expected)); + } + + #[test] + fn finish_framing_matches_include_usage_modes() { + let usage = crate::ir::Usage { + prompt_tokens: 10, + completion_tokens: 5, + cached_prompt_tokens: 2, + reasoning_tokens: 3, + time_to_first_token_ms: 12.5, + response_tokens_per_second: 40.0, + }; + let d = StreamDelta::Finish { + reason: FinishReason::Stop, + usage, + token_ids: vec![7], + }; + + // include_usage=true: usage-only chunk (`choices:[]`) FIRST, + // then the finish chunk with usage omitted; residual ids ride + // the final chunk. + let p = delta_to_payloads(&d, "m", "id-1", true); + assert_eq!(p.len(), 2); + assert!(p[0].contains("\"choices\":[]"), "payload: {}", p[0]); + assert!(p[0].contains("\"prompt_tokens\":10"), "payload: {}", p[0]); + assert!( + p[0].contains("\"completion_tokens\":5"), + "payload: {}", + p[0] + ); + assert!(p[0].contains("\"total_tokens\":15"), "payload: {}", p[0]); + assert!(p[0].contains("\"cached_tokens\":2"), "payload: {}", p[0]); + assert!(p[0].contains("\"reasoning_tokens\":3"), "payload: {}", p[0]); + assert!( + p[0].contains("\"time_to_first_token_ms\":12.5"), + "payload: {}", + p[0] + ); + assert!( + p[0].contains("\"response_token/s\":40.0"), + "payload: {}", + p[0] + ); + assert!( + p[1].contains("\"finish_reason\":\"stop\""), + "payload: {}", + p[1] + ); + assert!( + !p[1].contains("\"usage\""), + "final chunk must omit usage when include_usage=true: {}", + p[1] + ); + assert!(p[1].contains("\"token_ids\":[7]"), "payload: {}", p[1]); + + // include_usage=false: one done chunk carrying finish_reason + // AND usage, with the residual ids stamped on it. + let p = delta_to_payloads(&d, "m", "id-1", false); + assert_eq!(p.len(), 1); + assert!( + p[0].contains("\"finish_reason\":\"stop\""), + "payload: {}", + p[0] + ); + assert!(p[0].contains("\"usage\":{"), "payload: {}", p[0]); + assert!(p[0].contains("\"total_tokens\":15"), "payload: {}", p[0]); + assert!(p[0].contains("\"token_ids\":[7]"), "payload: {}", p[0]); + } + + #[test] + fn error_delta_payload_is_verbatim() { + let msg = r#"{"error":{"message":"boom","type":"server_error","code":500}}"#; + let d = StreamDelta::Error { + message: msg.to_string(), + }; + let p = delta_to_payloads(&d, "m", "id-1", false); + assert_eq!(p, vec![msg.to_string()]); + } +} diff --git a/crates/spark-server/src/openai/mod.rs b/crates/spark-server/src/openai/mod.rs index c036904f3..369d9e18f 100644 --- a/crates/spark-server/src/openai/mod.rs +++ b/crates/spark-server/src/openai/mod.rs @@ -7,9 +7,12 @@ mod chat_message; mod chat_request; mod chat_response; mod completions; +mod encode; +mod encode_stream; mod responses; mod responses_lowering; mod stream_chunk; +mod to_ir; #[cfg(test)] mod tests; @@ -19,66 +22,23 @@ pub use chat_message::*; pub use chat_request::*; pub use chat_response::*; pub use completions::*; +pub(crate) use encode::encode_chat_response; +pub(crate) use encode_stream::encode_sse_response; pub use responses::*; pub use responses_lowering::*; pub use stream_chunk::*; +// ID/timestamp primitives live in the provider-neutral `crate::ids`; +// the private import keeps bare `uuid_v4()` / `unix_timestamp()` calls +// in this module's children (via `use super::*`) working. +use crate::ids::{unix_timestamp, uuid_v4}; + /// Generate a new completion ID for SSE streaming. pub fn new_completion_id() -> String { format!("cmpl-{}", uuid_v4()) } -pub fn unix_timestamp() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs() -} - /// Generate a new chunk ID for SSE streaming. pub fn new_chunk_id() -> String { format!("chatcmpl-{}", uuid_v4()) } - -/// Read random bytes from the OS (Linux: getrandom syscall). -pub(crate) fn getrandom(buf: &mut [u8]) -> Result<(), ()> { - use std::fs::File; - use std::io::Read; - File::open("/dev/urandom") - .and_then(|mut f| f.read_exact(buf)) - .map_err(|_| ()) -} - -/// UUID v4 generation using OS randomness (no external crate needed). -pub(crate) fn uuid_v4() -> String { - let mut bytes = [0u8; 16]; - if let Ok(()) = getrandom(&mut bytes) { - bytes[6] = (bytes[6] & 0x0F) | 0x40; // version 4 - bytes[8] = (bytes[8] & 0x3F) | 0x80; // variant 1 - } else { - let t = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - bytes = t.to_le_bytes(); - } - format!( - "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", - bytes[0], - bytes[1], - bytes[2], - bytes[3], - bytes[4], - bytes[5], - bytes[6], - bytes[7], - bytes[8], - bytes[9], - bytes[10], - bytes[11], - bytes[12], - bytes[13], - bytes[14], - bytes[15], - ) -} diff --git a/crates/spark-server/src/openai/responses_lowering.rs b/crates/spark-server/src/openai/responses_lowering.rs index 533e92a32..6f0952a4d 100644 --- a/crates/spark-server/src/openai/responses_lowering.rs +++ b/crates/spark-server/src/openai/responses_lowering.rs @@ -229,40 +229,3 @@ pub fn lower_responses_to_chat( reasoning_effort: None, }) } - -/// UUID v4 generation using OS randomness (no external crate needed). -pub(crate) fn uuid_v4() -> String { - let mut bytes = [0u8; 16]; - // Use getrandom via std (available since Rust 1.36) - if let Ok(()) = getrandom(&mut bytes) { - // Set version 4 (bits 48-51) and variant 1 (bits 64-65) - bytes[6] = (bytes[6] & 0x0F) | 0x40; // version 4 - bytes[8] = (bytes[8] & 0x3F) | 0x80; // variant 1 - } else { - // Fallback: nanosecond timestamp (unique but not random) - let t = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos(); - bytes = t.to_le_bytes(); - } - format!( - "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", - bytes[0], - bytes[1], - bytes[2], - bytes[3], - bytes[4], - bytes[5], - bytes[6], - bytes[7], - bytes[8], - bytes[9], - bytes[10], - bytes[11], - bytes[12], - bytes[13], - bytes[14], - bytes[15], - ) -} diff --git a/crates/spark-server/src/openai/tests.rs b/crates/spark-server/src/openai/tests.rs index 23a59b941..bce83ba3e 100644 --- a/crates/spark-server/src/openai/tests.rs +++ b/crates/spark-server/src/openai/tests.rs @@ -195,6 +195,95 @@ fn markdown_link_with_parens_in_url_preserved() { assert_eq!(t, "Foo (bar)"); } +#[test] +fn responses_input_image_string_url_is_carried() { + // Regression: the Responses adapter used to drop images entirely + // (`images: Vec::new()`), silently losing multimodal input. + let item = serde_json::json!({ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAA"} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("message"); + assert_eq!(m.content.text, "what is this?"); + assert_eq!( + m.content.images, + vec!["data:image/png;base64,AAA".to_string()] + ); +} + +#[test] +fn responses_input_image_object_url_is_carried() { + // Some SDKs nest the url: `{image_url: {url: "..."}}`. + let item = serde_json::json!({ + "type": "message", + "role": "user", + "content": [ + {"type": "input_image", "image_url": {"url": "https://example.com/a.png"}} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("message"); + assert_eq!( + m.content.images, + vec!["https://example.com/a.png".to_string()] + ); + assert_eq!(m.content.text, ""); +} + +#[test] +fn responses_function_call_output_image_is_carried() { + // #165 parity for the Responses surface: a screenshot returned by a + // tool (`function_call_output` with structured output parts) must + // reach the vision encoder, exactly like Anthropic tool_result + // images and chat-completions role:"tool" array content. + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "output_text", "text": "screenshot follows"}, + {"type": "input_image", "image_url": "data:image/png;base64,BBB"} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.role, "tool"); + assert_eq!(m.tool_call_id.as_deref(), Some("call_1")); + assert_eq!(m.content.text, "screenshot follows"); + assert_eq!( + m.content.images, + vec!["data:image/png;base64,BBB".to_string()] + ); +} + +#[test] +fn responses_function_call_output_string_unchanged() { + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_2", + "output": "plain result" + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.content.text, "plain result"); + assert!(m.content.images.is_empty()); +} + +#[test] +fn responses_function_call_output_opaque_array_stringified() { + // Out-of-spec array (no recognizable parts) keeps the historical + // stringified-JSON behavior instead of silently emptying the result. + let opaque = serde_json::json!([{"weather": "sunny", "temp_c": 21}]); + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_3", + "output": opaque.clone() + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.content.text, opaque.to_string()); + assert!(m.content.images.is_empty()); +} + #[test] fn responses_in_progress_event_name() { let ev = ResponsesStreamEvent::InProgress { @@ -287,68 +376,167 @@ fn blocking_message_emits_only_reasoning_content() { ); } -// ── ChatTemplateKwargs ──────────────────────────────────────────── +// ── Thinking directive (client channels → ir::ThinkingDirective) ── -#[test] -fn chat_template_kwargs_parse() { - let kw = ChatTemplateKwargs::from_json(r#"{"enable_thinking":true,"thinking_budget":1024}"#) - .expect("should parse"); - assert_eq!(kw.enable_thinking, Some(true)); - assert_eq!(kw.thinking_budget, Some(1024)); +use crate::ir::ThinkingDirective; - assert!(ChatTemplateKwargs::from_json("").is_none()); +fn chat_req(body: serde_json::Value) -> ChatCompletionRequest { + serde_json::from_value(body).expect("valid chat request") } -fn empty_chat_request() -> ChatCompletionRequest { - serde_json::from_value(serde_json::json!({ +fn base_body() -> serde_json::Value { + serde_json::json!({ "model": "test", "messages": [{"role": "user", "content": "hi"}], - })) - .expect("valid chat request") + }) } #[test] -fn server_default_merged_when_request_silent() { - let mut req = empty_chat_request(); - assert!(req.chat_template_kwargs.is_none()); +fn silent_request_is_unspecified() { + let req = chat_req(base_body()); + assert_eq!( + req.client_thinking_directive(), + ThinkingDirective::Unspecified + ); + assert!(!req.client_thinking_directive().is_explicit()); +} - let server_kw = ChatTemplateKwargs { - enable_thinking: Some(true), - thinking_budget: None, - }; - if !req.thinking_explicitly_requested() { - req.chat_template_kwargs = Some(server_kw); +#[test] +fn anthropic_thinking_channel() { + // type=disabled wins even with a budget present. + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "disabled", "budget_tokens": 100}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); + + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "enabled", "budget_tokens": 512}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(512) } + ); + + // Adaptive / budget-less thinking object → think as long as needed + // (budget defers to the per-model max_thinking_budget). + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "adaptive"}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); +} + +#[test] +fn thinking_token_budget_channel() { + let mut b = base_body(); + b["thinking_token_budget"] = serde_json::json!(512); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(512) } + ); + + let mut b = base_body(); + b["thinking_token_budget"] = serde_json::json!(0); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); +} + +#[test] +fn reasoning_effort_channel() { + for (effort, expect) in [ + ("none", ThinkingDirective::Off), + ("minimal", ThinkingDirective::On { budget: Some(64) }), + ("low", ThinkingDirective::On { budget: Some(128) }), + ("medium", ThinkingDirective::On { budget: Some(256) }), + ("high", ThinkingDirective::On { budget: Some(512) }), + ("xhigh", ThinkingDirective::On { budget: Some(1024) }), + ("max", ThinkingDirective::On { budget: Some(1024) }), + // Unknown efforts fall back to the conservative default budget. + ("bogus", ThinkingDirective::On { budget: Some(256) }), + ] { + let mut b = base_body(); + b["reasoning"] = serde_json::json!({"effort": effort}); + assert_eq!( + chat_req(b).client_thinking_directive(), + expect, + "effort={effort}" + ); } - assert!(req.chat_template_kwargs.is_some()); +} + +#[test] +fn chat_template_kwargs_channel() { + // Struct still parses as a request-body wire field. + let kw: ChatTemplateKwargs = + serde_json::from_str(r#"{"enable_thinking":true,"thinking_budget":1024}"#) + .expect("should parse"); + assert_eq!(kw.enable_thinking, Some(true)); + assert_eq!(kw.thinking_budget, Some(1024)); + + // Budget rung wins over the enable flag. + let mut b = base_body(); + b["chat_template_kwargs"] = + serde_json::json!({"enable_thinking": false, "thinking_budget": 1024}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(1024) } + ); + + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"thinking_budget": 0}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); - let (enabled, budget) = req.resolve_thinking(false); - assert!(enabled); // enable_thinking with no explicit budget defers to the per-model - // max_thinking_budget (None), not the conservative 256-token default — - // a hard cut force-injects mid-reasoning and wrecks agentic - // tool selection (see resolve_thinking step 5). - assert!(budget.is_none()); + // max_thinking_budget (budget: None), not the conservative 256-token + // default — a hard cut force-injects mid-reasoning and + // wrecks agentic tool selection. + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": true}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); + + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": false}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); + + // Empty kwargs object carries no intent. + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Unspecified + ); } #[test] -fn server_default_not_merged_when_request_explicit() { - let mut req: ChatCompletionRequest = serde_json::from_value(serde_json::json!({ - "model": "test", - "messages": [{"role": "user", "content": "hi"}], - "enable_thinking": true, - })) - .expect("valid chat request"); - assert!(req.thinking_explicitly_requested()); +fn legacy_enable_thinking_channel() { + let mut b = base_body(); + b["enable_thinking"] = serde_json::json!(true); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); - let server_kw = ChatTemplateKwargs { - enable_thinking: Some(false), - thinking_budget: None, - }; - if !req.thinking_explicitly_requested() { - req.chat_template_kwargs = Some(server_kw); - } - assert!(req.chat_template_kwargs.is_none()); - assert!(req.resolve_thinking(false).0); + // false is the serde default — indistinguishable from absent, so it + // must NOT count as an explicit opt-out. + let mut b = base_body(); + b["enable_thinking"] = serde_json::json!(false); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Unspecified + ); } // ── Legacy /v1/completions echo + logprobs wire types ── diff --git a/crates/spark-server/src/openai/to_ir.rs b/crates/spark-server/src/openai/to_ir.rs new file mode 100644 index 000000000..b7ff19e17 --- /dev/null +++ b/crates/spark-server/src/openai/to_ir.rs @@ -0,0 +1,369 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// +// Adapter: OpenAI Chat Completions wire request → canonical chat IR. +// This is the OpenAI request-direction edge of the narrow waist: the +// wire message becomes `ir::Message`, and the whole wire request +// becomes the `ir::ChatRequest` envelope. Wire-only knobs (logit_bias +// string keys, the logprobs/top_logprobs pair, response_format "text", +// the five thinking channels) are resolved HERE so no internal module +// ever reads them. + +use super::{ChatCompletionRequest, IncomingMessage, ResponseFormat}; +use crate::ir; +use crate::ir::message::{ContentPart, ImageData, ImageSource, Message, Reasoning, Role, ToolCall}; + +impl From<&IncomingMessage> for Message { + fn from(m: &IncomingMessage) -> Self { + // Images first (matching the template's historical `[image*N, + // text]` content-array shape), then a single text part. The + // original wire interleaving was already flattened away by + // `ParsedContent`, and `build_msg_entries`/the template read + // `text()` and `image_count()` independently, so part order here + // does not affect rendering. + let mut content: Vec = Vec::new(); + for img in &m.content.images { + content.push(ContentPart::Image(ImageSource { + data: ImageData::from_uri(img.clone()), + })); + } + if !m.content.text.is_empty() { + content.push(ContentPart::Text(m.content.text.clone())); + } + + // Tool calls: parse the wire `arguments` string into structured + // JSON exactly as the historical path did + // (`from_str(..).unwrap_or(Object::default())`); a missing id + // becomes the empty string, also matching today. + let tool_calls: Vec = m + .tool_calls + .as_ref() + .map(|tcs| { + tcs.iter() + .map(|tc| ToolCall { + id: tc.id.clone().unwrap_or_default(), + name: tc.function.name.clone(), + arguments: serde_json::from_str(&tc.function.arguments) + .unwrap_or_else(|_| serde_json::Value::Object(Default::default())), + }) + .collect() + }) + .unwrap_or_default(); + + Message { + role: Role::from_wire_lossless(&m.role), + content, + tool_calls, + tool_call_id: m.tool_call_id.clone(), + name: m.name.clone(), + reasoning: m.reasoning_content.clone().map(|text| Reasoning { text }), + tool_error: false, + } + } +} + +impl ChatCompletionRequest { + /// Lower the parsed OpenAI wire request into the provider-agnostic + /// [`ir::ChatRequest`] envelope. Infallible: range validation + /// happens on the envelope (`chat_phases::validate_input`), and the + /// historical lenient parses (non-numeric logit_bias keys dropped, + /// malformed tool args → `{}`) are preserved. + /// + /// Echo-only fields (service_tier, store, metadata, + /// stream_options) are NOT lowered — the handler keeps the wire + /// request for encode-time echoes. + pub fn into_ir(self) -> ir::ChatRequest { + let thinking = self.client_thinking_directive(); + let top_logprobs = resolve_top_logprobs(self.logprobs, self.top_logprobs); + // Logit bias: OpenAI's string-keyed map → typed pairs. Keys that + // don't parse as token ids are dropped (historical behavior). + let logit_bias: Vec<(u32, f32)> = self.logit_bias.as_ref().map_or(Vec::new(), |map| { + map.iter() + .filter_map(|(k, &v)| k.parse::().ok().map(|id| (id, v))) + .collect() + }); + let response_format = match self.response_format { + // "text" is the wire spelling of "no constraint". + None | Some(ResponseFormat::Text) => None, + Some(ResponseFormat::JsonObject) => Some(ir::ResponseFormat::JsonObject), + Some(ResponseFormat::JsonSchema { json_schema }) => { + Some(ir::ResponseFormat::JsonSchema { + name: json_schema.name, + schema: json_schema.schema, + strict: json_schema.strict, + }) + } + }; + ir::ChatRequest { + model: self.model, + messages: self.messages.iter().map(Into::into).collect(), + tools: self.tools.unwrap_or_default(), + tool_choice: self.tool_choice, + sampling: ir::SamplingParams { + temperature: self.temperature, + top_k: self.top_k, + top_p: self.top_p, + top_n_sigma: self.top_n_sigma, + min_p: self.min_p, + repetition_penalty: self.repetition_penalty, + presence_penalty: self.presence_penalty, + frequency_penalty: self.frequency_penalty, + }, + max_tokens: self.max_tokens, + min_tokens: self.min_tokens, + stop: self.stop, + stream: self.stream, + n: self.n, + response_format, + thinking, + repetition_detection: self.repetition_detection, + logit_bias, + top_logprobs, + seed: self.seed, + timeout_secs: self.timeout, + return_token_ids: self.return_token_ids, + } + } +} + +/// Resolve chat logprobs params (OpenAI spec): an explicit +/// `top_logprobs` count wins (clamped 0-20); `logprobs: true` alone +/// enables sampled-token logprobs with no alternatives (count 0); +/// otherwise disabled. +pub(crate) fn resolve_top_logprobs(logprobs: Option, top_logprobs: Option) -> Option { + match (logprobs, top_logprobs) { + (_, Some(n)) => Some(n.min(20)), + (Some(true), None) => Some(0), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ir::message::{ContentPart, ImageData, ImageSource, Reasoning}; + use crate::openai::{IncomingMessage, ParsedContent}; + use crate::tool_parser::{IncomingFunction, IncomingToolCall}; + + fn msg(role: &str) -> IncomingMessage { + IncomingMessage { + role: role.to_string(), + content: ParsedContent::default(), + tool_calls: None, + tool_call_id: None, + name: None, + reasoning_content: None, + } + } + + #[test] + fn text_only_user_message() { + let mut m = msg("user"); + m.content.text = "hi".into(); + let ir: Message = (&m).into(); + assert_eq!(ir.role, Role::User); + assert_eq!(ir.content, vec![ContentPart::Text("hi".into())]); + assert_eq!(ir.image_count(), 0); + assert!(ir.tool_calls.is_empty()); + assert!(ir.reasoning.is_none()); + assert!(!ir.tool_error); + } + + #[test] + fn empty_content_yields_no_parts() { + let m = msg("user"); + let ir: Message = (&m).into(); + assert!(ir.content.is_empty()); + assert_eq!(ir.text(), ""); + } + + #[test] + fn images_precede_text_and_are_preserved_verbatim() { + let mut m = msg("user"); + m.content.text = "see".into(); + m.content.images = vec!["data:image/png;base64,AAA".into()]; + let ir: Message = (&m).into(); + // Images first (matches the template's [image*N, text] order), then text. + assert_eq!( + ir.content, + vec![ + ContentPart::Image(ImageSource { + data: ImageData::Base64("data:image/png;base64,AAA".into()), + }), + ContentPart::Text("see".into()), + ] + ); + assert_eq!(ir.image_count(), 1); + } + + #[test] + fn remote_url_image_classified_as_url_variant() { + // http(s) strings must become ImageData::Url so the pipeline can + // reject them explicitly — mislabeling them Base64 produced a + // confusing "base64 decode failed" from the vision preprocessor. + let mut m = msg("user"); + m.content.images = vec![ + "https://example.com/cat.png".into(), + "data:image/png;base64,AAA".into(), + ]; + let ir: Message = (&m).into(); + assert_eq!( + ir.content, + vec![ + ContentPart::Image(ImageSource { + data: ImageData::Url("https://example.com/cat.png".into()), + }), + ContentPart::Image(ImageSource { + data: ImageData::Base64("data:image/png;base64,AAA".into()), + }), + ] + ); + } + + #[test] + fn assistant_tool_calls_parse_arguments_to_json() { + let mut m = msg("assistant"); + m.tool_calls = Some(vec![IncomingToolCall { + id: Some("call_1".into()), + function: IncomingFunction { + name: "get_weather".into(), + arguments: r#"{"city":"SF"}"#.into(), + }, + }]); + let ir: Message = (&m).into(); + assert_eq!(ir.role, Role::Assistant); + assert_eq!(ir.tool_calls.len(), 1); + assert_eq!(ir.tool_calls[0].id, "call_1"); + assert_eq!(ir.tool_calls[0].name, "get_weather"); + assert_eq!( + ir.tool_calls[0].arguments, + serde_json::json!({"city": "SF"}) + ); + } + + #[test] + fn malformed_tool_args_default_to_empty_object() { + // Mirrors the historical `from_str(...).unwrap_or(Object::default())`. + let mut m = msg("assistant"); + m.tool_calls = Some(vec![IncomingToolCall { + id: None, + function: IncomingFunction { + name: "f".into(), + arguments: "not json".into(), + }, + }]); + let ir: Message = (&m).into(); + assert_eq!(ir.tool_calls[0].id, ""); // missing id → empty string (preserved) + assert_eq!(ir.tool_calls[0].arguments, serde_json::json!({})); + } + + #[test] + fn reasoning_content_maps_to_first_class_reasoning() { + let mut m = msg("assistant"); + m.reasoning_content = Some("ponder".into()); + let ir: Message = (&m).into(); + assert_eq!( + ir.reasoning, + Some(Reasoning { + text: "ponder".into() + }) + ); + + let m2 = msg("assistant"); + let ir2: Message = (&m2).into(); + assert!(ir2.reasoning.is_none()); + } + + #[test] + fn tool_message_preserves_call_id_and_name() { + let mut m = msg("tool"); + m.content.text = "exit 0".into(); + m.tool_call_id = Some("call_1".into()); + m.name = Some("bash".into()); + let ir: Message = (&m).into(); + assert_eq!(ir.role, Role::Tool); + assert_eq!(ir.tool_call_id.as_deref(), Some("call_1")); + assert_eq!(ir.name.as_deref(), Some("bash")); + assert_eq!(ir.text(), "exit 0"); + } + + #[test] + fn unknown_role_is_preserved_losslessly() { + let m = msg("developer"); + let ir: Message = (&m).into(); + assert_eq!(ir.role, Role::Other("developer".into())); + assert_eq!(ir.role.as_wire(), "developer"); + } + + // ── whole-request envelope lowering ── + + fn wire(body: serde_json::Value) -> ChatCompletionRequest { + serde_json::from_value(body).expect("valid chat request") + } + + #[test] + fn envelope_lowers_scalars_and_parses_logit_bias() { + let req = wire(serde_json::json!({ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 64, + "temperature": 0.5, + "logit_bias": {"42": 1.5, "not-a-token": -1.0}, + "logprobs": true, + "stop": ["END"], + "seed": 7, + "n": 2 + })); + let ir = req.into_ir(); + assert_eq!(ir.model, "m"); + assert_eq!(ir.messages.len(), 1); + assert_eq!(ir.max_tokens, 64); + assert_eq!(ir.sampling.temperature, Some(0.5)); + // Non-numeric keys dropped (historical behavior). + assert_eq!(ir.logit_bias, vec![(42, 1.5)]); + // logprobs:true alone → count 0 (sampled token only). + assert_eq!(ir.top_logprobs, Some(0)); + assert_eq!(ir.stop, vec!["END".to_string()]); + assert_eq!(ir.seed, Some(7)); + assert_eq!(ir.n, 2); + assert!(ir.tools.is_empty()); + assert!(ir.response_format.is_none()); + } + + #[test] + fn envelope_maps_text_response_format_to_none() { + let req = wire(serde_json::json!({ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "response_format": {"type": "text"} + })); + assert!(req.into_ir().response_format.is_none()); + + let req = wire(serde_json::json!({ + "model": "m", + "messages": [{"role": "user", "content": "hi"}], + "response_format": {"type": "json_schema", "json_schema": {"name": "s", "schema": {"type": "object"}}} + })); + match req.into_ir().response_format { + Some(ir::ResponseFormat::JsonSchema { + name, + schema, + strict, + }) => { + assert_eq!(name, "s"); + assert_eq!(schema, serde_json::json!({"type": "object"})); + assert!(strict); // default_true + } + other => panic!("expected JsonSchema, got {other:?}"), + } + } + + #[test] + fn resolve_top_logprobs_matrix() { + assert_eq!(resolve_top_logprobs(Some(true), None), Some(0)); + assert_eq!(resolve_top_logprobs(None, Some(5)), Some(5)); + assert_eq!(resolve_top_logprobs(Some(false), Some(3)), Some(3)); + assert_eq!(resolve_top_logprobs(Some(true), Some(99)), Some(20)); + assert_eq!(resolve_top_logprobs(None, None), None); + assert_eq!(resolve_top_logprobs(Some(false), None), None); + } +} diff --git a/crates/spark-server/src/scheduler/decode_logits_step.rs b/crates/spark-server/src/scheduler/decode_logits_step.rs index 644875edc..0c5a2e183 100644 --- a/crates/spark-server/src/scheduler/decode_logits_step.rs +++ b/crates/spark-server/src/scheduler/decode_logits_step.rs @@ -253,6 +253,9 @@ pub fn process_decode_logits( if a.inside_thinking { if think_end_token == Some(tok) { a.inside_thinking = false; + // Sticky: was THIS close force-injected? Read by the + // post-think EOS guard below. + a.think_force_closed = a.force_end_thinking; a.force_end_thinking = false; a.sentence_defer_count = 0; a.consecutive_confident = 0; @@ -541,8 +544,10 @@ pub fn process_decode_logits( // MTP-verify emit path (`emit_step.rs`) has no such guard, which is why // MTP-on stopped here while MTP-off leaked; this restores parity. let tools_armed = a.require_tool_call || a.tool_request; - let post_think_suppresses_eos = - tools_armed && a.think_ended && post_think_content_tokens < POST_THINK_MIN_CONTENT; + let post_think_suppresses_eos = tools_armed + && a.think_ended + && a.think_force_closed + && post_think_content_tokens < POST_THINK_MIN_CONTENT; let suppress_eos = grammar_suppresses_eos || legacy_suppresses_eos || min_tokens_suppresses diff --git a/crates/spark-server/src/scheduler/emit_step.rs b/crates/spark-server/src/scheduler/emit_step.rs index 42d5940d5..0490895e1 100644 --- a/crates/spark-server/src/scheduler/emit_step.rs +++ b/crates/spark-server/src/scheduler/emit_step.rs @@ -180,6 +180,9 @@ pub fn emit_token(a: &mut ActiveSeq, tok: u32, logprobs: Option bool { /// without per-request configuration are byte-identical to before. pub fn detect_thinking_token_loop_with( tokens: &[u32], - override_: Option, + override_: Option, ) -> bool { let (period_min, period_max, min_repeats) = match override_ { Some(p) => ( @@ -533,7 +533,7 @@ pub fn detect_content_token_loop(tokens: &[u32]) -> bool { /// constants. See [`detect_thinking_token_loop_with`] for rationale. pub fn detect_content_token_loop_with( tokens: &[u32], - override_: Option, + override_: Option, ) -> bool { let (period_min, period_max, min_repeats) = match override_ { Some(p) => ( @@ -580,7 +580,7 @@ pub fn detect_content_token_loop_normalized(tokens: &[u32], mask: &[bool]) -> bo pub fn detect_content_token_loop_normalized_with( tokens: &[u32], mask: &[bool], - override_: Option, + override_: Option, ) -> bool { let n = tokens.len(); if n < CONTENT_LOOP_MIN_TOKENS as usize { diff --git a/crates/spark-server/src/scheduler/helpers_tests.rs b/crates/spark-server/src/scheduler/helpers_tests.rs index 88feca213..96a23c876 100644 --- a/crates/spark-server/src/scheduler/helpers_tests.rs +++ b/crates/spark-server/src/scheduler/helpers_tests.rs @@ -348,7 +348,7 @@ fn override_loosens_content_loop_threshold() { ); // Override path: min_count=4 ⇒ 3 repeats are insufficient. - let strict = crate::openai::RepetitionDetectionParams { + let strict = crate::api::inference_types::RepetitionDetectionParams { min_pattern_size: 2, max_pattern_size: 64, min_count: 4, @@ -371,7 +371,7 @@ fn override_tightens_content_loop_threshold() { for _ in 0..5 { tokens.extend(pat.iter()); } - let permissive = crate::openai::RepetitionDetectionParams { + let permissive = crate::api::inference_types::RepetitionDetectionParams { min_pattern_size: 5, max_pattern_size: 5, min_count: 3, @@ -396,7 +396,7 @@ fn override_applies_to_thinking_loop() { "default thinking-loop thresholds must still fire on 4× period-10" ); // Override demanding 6 repeats ⇒ 4 is insufficient ⇒ must not fire. - let strict = crate::openai::RepetitionDetectionParams { + let strict = crate::api::inference_types::RepetitionDetectionParams { min_pattern_size: 4, max_pattern_size: 20, min_count: 6, diff --git a/crates/spark-server/src/scheduler/lifecycle.rs b/crates/spark-server/src/scheduler/lifecycle.rs index 0118d3fc4..e440ff602 100644 --- a/crates/spark-server/src/scheduler/lifecycle.rs +++ b/crates/spark-server/src/scheduler/lifecycle.rs @@ -231,6 +231,7 @@ pub fn swap_out_sequence( content_tokens: a.content_tokens, prose_tokens_since_last_tool: a.prose_tokens_since_last_tool, think_watchdog_fires: a.think_watchdog_fires, + think_force_closed: a.think_force_closed, rollback_count: a.rollback_count, tool_call_start_token: a.tool_call_start_token, tool_call_opened: a.tool_call_opened, @@ -318,6 +319,7 @@ pub fn resume_swapped_seq( content_tokens: 0, prose_tokens_since_last_tool: 0, think_watchdog_fires: s.think_watchdog_fires, + think_force_closed: s.think_force_closed, rollback_count: s.rollback_count, // Decode-rollback SSM snapshots are GPU-resident and not part of // the disk swap image — a resumed sequence starts with an empty diff --git a/crates/spark-server/src/scheduler/phase_promote_prefills.rs b/crates/spark-server/src/scheduler/phase_promote_prefills.rs index 0d4311be3..a79d42552 100644 --- a/crates/spark-server/src/scheduler/phase_promote_prefills.rs +++ b/crates/spark-server/src/scheduler/phase_promote_prefills.rs @@ -171,6 +171,7 @@ fn build_active_seq_from_prefill( thinking_tokens: 0, cached_prompt_tokens: cached_prompt_tok, force_end_thinking: false, + think_force_closed: false, sentence_defer_count: 0, consecutive_confident: 0, in_code_fence: false, diff --git a/crates/spark-server/src/scheduler/prefill_a_step.rs b/crates/spark-server/src/scheduler/prefill_a_step.rs index 69d6867fb..9a712646c 100644 --- a/crates/spark-server/src/scheduler/prefill_a_step.rs +++ b/crates/spark-server/src/scheduler/prefill_a_step.rs @@ -409,6 +409,7 @@ pub fn start_chunked_prefill( thinking_tokens: 0, cached_prompt_tokens: cached_prompt_tok, force_end_thinking: false, + think_force_closed: false, sentence_defer_count: 0, consecutive_confident: 0, in_code_fence: false, @@ -494,6 +495,7 @@ pub fn start_chunked_prefill( thinking_tokens: 0, cached_prompt_tokens: cached_prompt_tok, force_end_thinking: false, + think_force_closed: false, sentence_defer_count: 0, consecutive_confident: 0, in_code_fence: false, diff --git a/crates/spark-server/src/scheduler/prefill_a_step_params.rs b/crates/spark-server/src/scheduler/prefill_a_step_params.rs index 2d518083b..30fb0ef76 100644 --- a/crates/spark-server/src/scheduler/prefill_a_step_params.rs +++ b/crates/spark-server/src/scheduler/prefill_a_step_params.rs @@ -13,8 +13,8 @@ use std::time::Instant; use spark_model::traits::SequenceState; +use crate::api::inference_types::RepetitionDetectionParams; use crate::grammar::GrammarState; -use crate::openai::RepetitionDetectionParams; use super::types::{PrefillInProgress, ResponseSink}; diff --git a/crates/spark-server/src/scheduler/prefill_b_step.rs b/crates/spark-server/src/scheduler/prefill_b_step.rs index d5614d799..02ab122c2 100644 --- a/crates/spark-server/src/scheduler/prefill_b_step.rs +++ b/crates/spark-server/src/scheduler/prefill_b_step.rs @@ -242,6 +242,7 @@ pub fn prefill_request( thinking_tokens: 0, cached_prompt_tokens: cached_prompt_tok, force_end_thinking: false, + think_force_closed: false, sentence_defer_count: 0, consecutive_confident: 0, in_code_fence: false, @@ -323,6 +324,7 @@ pub fn prefill_request( thinking_tokens: 0, cached_prompt_tokens: cached_prompt_tok, force_end_thinking: false, + think_force_closed: false, sentence_defer_count: 0, consecutive_confident: 0, in_code_fence: false, diff --git a/crates/spark-server/src/scheduler/types.rs b/crates/spark-server/src/scheduler/types.rs index 63980e46c..ac75dbb1b 100644 --- a/crates/spark-server/src/scheduler/types.rs +++ b/crates/spark-server/src/scheduler/types.rs @@ -13,9 +13,9 @@ use std::time::Instant; use anyhow::Result; use spark_model::traits::SequenceState; +use crate::api::inference_types::RepetitionDetectionParams; use crate::api::{InferenceRequest, InferenceResponse, StreamEvent}; use crate::grammar::GrammarState; -use crate::openai::RepetitionDetectionParams; /// Shared queue between receiver thread and scheduler. pub(super) struct PendingQueue { @@ -157,6 +157,13 @@ pub(super) struct ActiveSeq { pub thinking_tokens: u32, /// When true, the next decode step must produce the `` token. pub force_end_thinking: bool, + /// Whether the `` that closed the current thinking span was + /// FORCE-INJECTED (budget exhaustion or thinking-loop watchdog) rather + /// than emitted by the model. Captured at the `` commit; read + /// by the post-think EOS guard, which must only fire on watchdog + /// recoveries — a naturally closed think followed by a short answer + /// ends the turn like vLLM does. + pub think_force_closed: bool, /// Decode-step counter incremented while `force_end_thinking` is /// armed but the injection is deferred (waiting for a sentence- /// boundary token or fence close). Reset to 0 on the false→true @@ -385,6 +392,7 @@ pub(super) struct SwappedSeq { pub spontaneous_think_budget: u32, pub thinking_tokens: u32, pub force_end_thinking: bool, + pub think_force_closed: bool, pub sentence_defer_count: u32, pub consecutive_confident: u32, pub in_code_fence: bool, diff --git a/scripts/test-qwen36-tool-image.sh b/scripts/test-qwen36-tool-image.sh new file mode 100755 index 000000000..602501321 --- /dev/null +++ b/scripts/test-qwen36-tool-image.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only +# +# Download + serve + smoke-test the Qwen3.6-35B-A3B (vision) model on Atlas, +# exercising the issue-#165 fix: an image attached to a *tool result* must +# reach the vision encoder (it used to be silently dropped). +# +# Run this on the GB10 box, from the repo root, with a binary built from the +# `feat/canonical-chat-ir` branch (the tool-result-image fix is NOT in the +# published Docker image yet). +# +# ./scripts/test-qwen36-tool-image.sh +# +# Override any setting via env, e.g.: +# HF_REPO=Sehyo/Qwen3.6-35B-A3B-NVFP4 PORT=8889 ./scripts/test-qwen36-tool-image.sh +set -euo pipefail + +# ─────────────────────────── config (edit / override via env) ─────────────── +# HuggingFace checkpoint to download. The in-tree kernel target +# kernels/gb10/qwen3.6-35b-a3b/MODEL.toml declares hf_id="Qwen/Qwen3.6-35B-A3B-FP8" +# (FP8 weights + NVFP4 lm-head). If you have a pre-quantized NVFP4 weights repo, +# set HF_REPO to it instead. +HF_REPO="${HF_REPO:-Qwen/Qwen3.6-35B-A3B-FP8}" +# Name the server advertises (so `@atlas/` in your client matches). +SERVED_NAME="${SERVED_NAME:-qwen3.6-35b-a3b-nvfp4}" + +HOST="${HOST:-127.0.0.1}" +PORT="${PORT:-8888}" +BASE_URL="http://${HOST}:${PORT}" + +SPARK_BIN="${SPARK_BIN:-./target/release/spark}" +KV_DTYPE="${KV_DTYPE:-nvfp4}" +MAX_SEQ_LEN="${MAX_SEQ_LEN:-8192}" +GPU_MEM_UTIL="${GPU_MEM_UTIL:-0.88}" +SCHED_POLICY="${SCHED_POLICY:-slai}" + +# auto = start a server only if one isn't already answering on $PORT. +# yes = always start one. no = never (test an already-running server). +START_SERVER="${START_SERVER:-auto}" +READY_TIMEOUT="${READY_TIMEOUT:-900}" # seconds to wait for weights to load +# This model defaults thinking ON (768-tok budget) — that would eat a small +# max_tokens before any content. Disable it for deterministic, fast tests. +DISABLE_THINKING="${DISABLE_THINKING:-1}" # 1 = pass --disable-thinking +# Dir holding libnccl.so / libnccl.so.2 for a source-built binary (optional). +# Atlas links NCCL; if your shell doesn't already resolve it, point this at a +# dir containing the symlinks and it's prepended to LD_LIBRARY_PATH. +NCCL_DIR="${NCCL_DIR:-}" + +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/atlas-qwen36-test.XXXXXX")" +SERVER_LOG="${WORKDIR}/server.log" +SERVER_PID="" +STARTED_SERVER=0 +# ───────────────────────────────────────────────────────────────────────────── + +# Progress goes to stderr so it never pollutes a captured stdout (e.g. the +# image data URL or a chat reply). +log() { printf '\033[1;34m[%s]\033[0m %s\n' "$(date +%H:%M:%S)" "$*" >&2; } +ok() { printf '\033[1;32m ✓ %s\033[0m\n' "$*" >&2; } +warn() { printf '\033[1;33m ! %s\033[0m\n' "$*" >&2; } +die() { printf '\033[1;31m[FATAL] %s\033[0m\n' "$*" >&2; exit 1; } +need() { command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"; } + +cleanup() { + if [[ "${STARTED_SERVER}" == "1" && -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + log "stopping server (pid ${SERVER_PID})" + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +need curl +need python3 +export SERVED_NAME # consumed by the inline python request builders +# huggingface-cli / hf is checked (with fallback) inside download_model. + +# ── a small, recognizable test image (white bg, red circle, "ATLAS") ───────── +# Returns a `data:image/png;base64,...` URL on stdout. Uses Pillow if present, +# else falls back to a pure-stdlib solid-red PNG (still describable as "red"). +make_image_data_url() { + local png="${WORKDIR}/test.png" + if python3 -c 'import PIL' >/dev/null 2>&1; then + python3 - "$png" <<'PY' +import sys +from PIL import Image, ImageDraw +img = Image.new("RGB", (256, 256), "white") +d = ImageDraw.Draw(img) +d.ellipse([48, 48, 208, 208], fill="red", outline="black", width=5) +d.text((96, 120), "ATLAS", fill="white") +img.save(sys.argv[1], "PNG") +PY + else + warn "Pillow not installed — using a plain red square (model should still say 'red')" + python3 - "$png" <<'PY' +import sys, zlib, struct +w = h = 128 +raw = bytearray() +for _ in range(h): + raw.append(0) # filter byte per scanline + raw.extend((220, 30, 30) * w) # solid red RGB +def chunk(tag, data): + c = tag + data + return struct.pack(">I", len(data)) + c + struct.pack(">I", zlib.crc32(c) & 0xffffffff) +png = b"\x89PNG\r\n\x1a\n" +png += chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) +png += chunk(b"IDAT", zlib.compress(bytes(raw), 9)) +png += chunk(b"IEND", b"") +open(sys.argv[1], "wb").write(png) +PY + fi + printf 'data:image/png;base64,%s' "$(base64 -w0 < "$png" 2>/dev/null || base64 < "$png" | tr -d '\n')" +} + +server_up() { curl -fsS -m 3 "${BASE_URL}/health" >/dev/null 2>&1; } + +download_model() { + if command -v huggingface-cli >/dev/null 2>&1; then + log "downloading ${HF_REPO} into the HF cache (large; resumes if interrupted)…" + huggingface-cli download "${HF_REPO}" --revision main + elif command -v hf >/dev/null 2>&1; then + log "downloading ${HF_REPO} into the HF cache (large; resumes if interrupted)…" + hf download "${HF_REPO}" --revision main + else + warn "no 'huggingface-cli'/'hf' found — skipping download; assuming ${HF_REPO} is" + warn "already in the HF cache (the server errors clearly if it isn't)." + return 0 + fi + ok "checkpoint present in cache" +} + +start_server() { + [[ -x "${SPARK_BIN}" ]] || die "spark binary not found/executable at '${SPARK_BIN}'. +Build it from the feat/canonical-chat-ir branch first. Atlas links NCCL, so the +linker needs a dir containing libnccl.so on LIBRARY_PATH (NCCL_DIR below is only +the RUNTIME path): + LIBRARY_PATH=\$NCCL_DIR ATLAS_TARGET_MODEL=qwen3.6-35b-a3b \\ + cargo build --release -p spark-server +(then re-run, or set SPARK_BIN=/path/to/spark)" + + if [[ -n "${NCCL_DIR}" ]]; then + export LIBRARY_PATH="${NCCL_DIR}:${LIBRARY_PATH:-}" + export LD_LIBRARY_PATH="${NCCL_DIR}:${LD_LIBRARY_PATH:-}" + fi + + local extra=() + [[ "${DISABLE_THINKING}" == "1" ]] && extra+=(--disable-thinking) + + log "starting server: ${SPARK_BIN} serve ${HF_REPO} (logs → ${SERVER_LOG})" + "${SPARK_BIN}" serve "${HF_REPO}" \ + --model-name "${SERVED_NAME}" \ + --bind "${HOST}" \ + --port "${PORT}" \ + --max-seq-len "${MAX_SEQ_LEN}" \ + --kv-cache-dtype "${KV_DTYPE}" \ + --gpu-memory-utilization "${GPU_MEM_UTIL}" \ + --scheduling-policy "${SCHED_POLICY}" \ + "${extra[@]}" \ + >"${SERVER_LOG}" 2>&1 & + SERVER_PID=$! + STARTED_SERVER=1 + + log "waiting up to ${READY_TIMEOUT}s for weights to load…" + local deadline=$(( $(date +%s) + READY_TIMEOUT )) + while ! server_up; do + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + tail -n 40 "${SERVER_LOG}" >&2 || true + die "server process exited before becoming ready (see ${SERVER_LOG})" + fi + [[ $(date +%s) -lt ${deadline} ]] || { tail -n 40 "${SERVER_LOG}" >&2; die "timed out waiting for readiness"; } + sleep 3 + done + ok "server ready" +} + +# chat → prints assistant content (and stashes raw json) +chat() { + local body_file="$1" raw="${WORKDIR}/resp.json" + curl -fsS -m 180 "${BASE_URL}/v1/chat/completions" \ + -H 'Content-Type: application/json' --data @"${body_file}" >"${raw}" + python3 - "${raw}" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +msg = (d.get("choices") or [{}])[0].get("message", {}) +print((msg.get("content") or "").strip()) +PY +} + +# ─────────────────────────────── run ──────────────────────────────────────── +log "Atlas Qwen3.6-35B-A3B vision / tool-image test" +log "repo=${HF_REPO} served-as=${SERVED_NAME} endpoint=${BASE_URL}" + +download_model + +case "${START_SERVER}" in + no) server_up || die "START_SERVER=no but nothing is answering ${BASE_URL}/health" ;; + yes) start_server ;; + auto) if server_up; then ok "reusing server already listening on ${PORT}"; else start_server; fi ;; + *) die "START_SERVER must be auto|yes|no" ;; +esac + +log "building test image…" +DATA_URL="$(make_image_data_url)"; export DATA_URL +ok "image ready (${#DATA_URL} base64 chars)" + +# 1) smoke — plain text round-trip +cat >"${WORKDIR}/req_smoke.json" <"${WORKDIR}/req_anthropic.json" <"${WORKDIR}/anthropic.json" +python3 - "${WORKDIR}/anthropic.json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +assert d["type"] == "message" and d["role"] == "assistant", d +text = "".join(b.get("text", "") for b in d["content"] if b["type"] == "text") +assert "pong" in text.lower(), f"unexpected content: {text!r}" +u = d["usage"] +assert u["input_tokens"] > 0, f"input_tokens must be > 0: {u}" +assert u["output_tokens"] > 0, f"output_tokens must be > 0: {u}" +assert "cache_read_input_tokens" in u, f"cache accounting missing: {u}" +print(f" input_tokens={u['input_tokens']} output_tokens={u['output_tokens']} cache_read={u['cache_read_input_tokens']}") +PY +ok "/v1/messages non-streaming passed (usage + content)" + +# 5) /v1/messages streaming: event order + non-zero input_tokens in the +# final message_delta (the B1 fix — used to always report 0). +log "test 5/8 — /v1/messages streaming (event framing + input_tokens)" +cat >"${WORKDIR}/req_anthropic_stream.json" <"${WORKDIR}/anthropic_stream.txt" +python3 - "${WORKDIR}/anthropic_stream.txt" <<'PY' +import json, sys +events = [] +for line in open(sys.argv[1]): + line = line.strip() + if line.startswith("event:"): + events.append({"name": line.split(":", 1)[1].strip()}) + elif line.startswith("data:") and events: + try: + events[-1]["data"] = json.loads(line.split(":", 1)[1].strip()) + except json.JSONDecodeError: + pass +names = [e["name"] for e in events] +assert names[0] == "message_start", names +assert "content_block_start" in names and "content_block_delta" in names, names +assert names[-2:] == ["message_delta", "message_stop"], names[-4:] +md = next(e for e in reversed(events) if e["name"] == "message_delta") +u = md["data"]["usage"] +assert u["input_tokens"] > 0, f"streaming input_tokens must be > 0 (B1): {u}" +assert u["output_tokens"] > 0, f"streaming output_tokens must be > 0: {u}" +print(f" events={len(events)} final usage={u}") +PY +ok "/v1/messages streaming passed (framing + input_tokens > 0)" + +# 6) /v1/messages/count_tokens: must be consistent with what serving reports +# (shared prompt pipeline — the old divergent count path is gone). +log "test 6/8 — /v1/messages/count_tokens vs served input_tokens" +curl -fsS -m 60 "${BASE_URL}/v1/messages/count_tokens" -H 'Content-Type: application/json' \ + --data @"${WORKDIR}/req_anthropic.json" >"${WORKDIR}/count.json" +python3 - "${WORKDIR}/count.json" "${WORKDIR}/anthropic.json" <<'PY' +import json, sys +count = json.load(open(sys.argv[1]))["input_tokens"] +served = json.load(open(sys.argv[2]))["usage"]["input_tokens"] +assert count > 0, count +drift = abs(count - served) / max(served, 1) +assert drift <= 0.10, f"count_tokens {count} vs served {served} drifts {drift:.0%} (>10%)" +print(f" count_tokens={count} served input_tokens={served} drift={drift:.1%}") +PY +ok "count_tokens matches served usage (<=10% drift)" + +# 7) /v1/responses — image on a function_call_output (the #165 parity fix +# for the Responses surface: it used to drop tool-result images). +python3 - <"${WORKDIR}/responses.json" +python3 - "${WORKDIR}/responses.json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +assert d.get("status") == "completed", d.get("status") +text = "" +for item in d.get("output", []): + if item.get("type") == "message": + for part in item.get("content", []): + text += part.get("text", "") +assert "red" in text.lower(), f"model did not describe the tool image as red: {text!r}" +print(f" model said: {text.strip()!r}") +PY +ok "/v1/responses carried the function_call_output image (saw 'red')" + +# 8) /v1/responses streaming smoke: completes with usage +log "test 8/8 — /v1/responses streaming smoke" +python3 - <"${WORKDIR}/responses_stream.txt" +python3 - "${WORKDIR}/responses_stream.txt" <<'PY' +import json, sys +completed = None +for line in open(sys.argv[1]): + line = line.strip() + if line.startswith("data:"): + try: + d = json.loads(line.split(":", 1)[1].strip()) + except json.JSONDecodeError: + continue + if d.get("type") == "response.completed": + completed = d +assert completed is not None, "no response.completed event seen" +usage = completed["response"].get("usage") or {} +assert usage.get("input_tokens", 0) > 0, f"usage missing/zero: {usage}" +print(f" completed with usage={usage}") +PY +ok "/v1/responses streaming completed with usage" + +log "done. artifacts in ${WORKDIR}" From 6d37182a725170e98cbe829fc72daba8ba45e085 Mon Sep 17 00:00:00 2001 From: Sercan Degirmenci Date: Tue, 14 Jul 2026 10:59:50 +0300 Subject: [PATCH 2/4] feat(server): split large files --- crates/spark-server/src/citation.rs | 312 +-------- .../spark-server/src/citation_structured.rs | 319 +++++++++ crates/spark-server/src/main.rs | 1 + crates/spark-server/src/openai/tests.rs | 635 +----------------- .../src/openai/tests/annotations.rs | 114 ++++ .../src/openai/tests/chat_wire.rs | 82 +++ .../src/openai/tests/completions.rs | 95 +++ .../src/openai/tests/responses.rs | 195 ++++++ .../spark-server/src/openai/tests/thinking.rs | 165 +++++ 9 files changed, 982 insertions(+), 936 deletions(-) create mode 100644 crates/spark-server/src/citation_structured.rs create mode 100644 crates/spark-server/src/openai/tests/annotations.rs create mode 100644 crates/spark-server/src/openai/tests/chat_wire.rs create mode 100644 crates/spark-server/src/openai/tests/completions.rs create mode 100644 crates/spark-server/src/openai/tests/responses.rs create mode 100644 crates/spark-server/src/openai/tests/thinking.rs diff --git a/crates/spark-server/src/citation.rs b/crates/spark-server/src/citation.rs index 8968d34f4..eb3cdf4f0 100644 --- a/crates/spark-server/src/citation.rs +++ b/crates/spark-server/src/citation.rs @@ -7,7 +7,7 @@ //! * [`extract_url_citations`] finds bare + markdown-link URLs (skipping //! code spans so `curl https://…` examples don't become citations). //! * [`extract`] recognizes three common "model-emitted" structured -//! citation patterns: +//! citation patterns (parsers live in [`crate::citation_structured`]): //! //! 1. Markdown footnotes //! ```text @@ -45,6 +45,10 @@ //! pattern we recognize". The shape clients receive is identical to //! what a real web-search backend would produce. +use crate::citation_structured::{ + footnote_citations, numeric_ref_citations, sources_block_citations, +}; + /// A URL citation found in assistant text, with byte offsets into the /// content it was extracted from. Provider-neutral. #[derive(Debug, Clone, PartialEq, Eq)] @@ -197,249 +201,6 @@ pub fn extract_url_citations(content: &str) -> Vec { out } -/// Parse markdown footnote references: every `[^label]` in text paired -/// with a `[^label]: url [title]` definition. -fn footnote_citations(content: &str) -> Vec { - let defs = collect_footnote_defs(content); - if defs.is_empty() { - return Vec::new(); - } - let mut out = Vec::new(); - let mut i = 0; - let bytes = content.as_bytes(); - while i < bytes.len() { - if let Some(rel) = content[i..].find("[^") { - let start = i + rel; - // Skip footnote definitions (`[^x]:` at line start). - let after_br = &content[start..]; - if let Some(close_rel) = after_br.find(']') { - let close = start + close_rel; - // Reject if this is a definition line (next char is ':'). - if content.as_bytes().get(close + 1) == Some(&b':') { - i = close + 1; - continue; - } - let label = &content[start + 2..close]; - if let Some((url, title)) = defs.get(label) { - out.push(Citation { - start_index: start, - end_index: close + 1, - url: url.clone(), - title: title.clone().unwrap_or_else(|| label.to_string()), - }); - } - i = close + 1; - } else { - break; - } - } else { - break; - } - } - out -} - -/// Walk each line; return `{label: (url, optional_title)}` for lines -/// matching `[^label]: url [title]`. -fn collect_footnote_defs( - content: &str, -) -> std::collections::HashMap)> { - let mut map = std::collections::HashMap::new(); - for line in content.lines() { - let trimmed = line.trim_start(); - let Some(rest) = trimmed.strip_prefix("[^") else { - continue; - }; - let Some(close_rel) = rest.find(']') else { - continue; - }; - let label = &rest[..close_rel]; - let after = &rest[close_rel + 1..]; - let Some(body) = after.strip_prefix(':') else { - continue; - }; - let body = body.trim_start(); - let Some((url, rest)) = split_url(body) else { - continue; - }; - let title = rest.trim(); - let title_opt = if title.is_empty() { - None - } else { - Some(title.to_string()) - }; - map.insert(label.to_string(), (url, title_opt)); - } - map -} - -/// Parse `[N]` refs paired with a `[N] url` definition line. -fn numeric_ref_citations(content: &str) -> Vec { - let defs = collect_numeric_defs(content); - if defs.is_empty() { - return Vec::new(); - } - let mut out = Vec::new(); - let mut i = 0; - while i < content.len() { - let Some(rel) = content[i..].find('[') else { - break; - }; - let start = i + rel; - let after = &content[start + 1..]; - let Some(close_rel) = after.find(']') else { - break; - }; - let label = &after[..close_rel]; - if label.bytes().all(|b| b.is_ascii_digit()) && !label.is_empty() { - let close = start + 1 + close_rel; - // Skip definition sites. - let at_line_start = start == 0 || content.as_bytes().get(start - 1) == Some(&b'\n'); - let next_is_url_hint = { - let next = content.get(close + 1..).unwrap_or(""); - next.trim_start().starts_with("http") - }; - let is_definition = at_line_start && next_is_url_hint; - if !is_definition && let Some((url, title)) = defs.get(label) { - out.push(Citation { - start_index: start, - end_index: close + 1, - url: url.clone(), - title: title.clone().unwrap_or_else(|| label.to_string()), - }); - } - i = close + 1; - } else { - i = start + 1; - } - } - out -} - -fn collect_numeric_defs( - content: &str, -) -> std::collections::HashMap)> { - let mut map = std::collections::HashMap::new(); - for line in content.lines() { - let trimmed = line.trim_start(); - let Some(rest) = trimmed.strip_prefix('[') else { - continue; - }; - let Some(close_rel) = rest.find(']') else { - continue; - }; - let label = &rest[..close_rel]; - if label.is_empty() || !label.bytes().all(|b| b.is_ascii_digit()) { - continue; - } - let mut after = &rest[close_rel + 1..]; - if let Some(stripped) = after.strip_prefix(':') { - after = stripped; - } - let body = after.trim_start(); - let Some((url, rest)) = split_url(body) else { - continue; - }; - let title = rest.trim(); - let title_opt = if title.is_empty() { - None - } else { - Some(title.to_string()) - }; - map.insert(label.to_string(), (url, title_opt)); - } - map -} - -/// Find `Sources:` or `References:` heading and emit a citation for -/// each bullet line containing an http(s) URL until a blank line or a -/// non-bullet line breaks the section. -fn sources_block_citations(content: &str) -> Vec { - let lower = content.to_ascii_lowercase(); - let markers = ["sources:", "references:", "citations:"]; - let mut out = Vec::new(); - for marker in &markers { - let mut search_from = 0usize; - while let Some(rel) = lower[search_from..].find(marker) { - let heading_start = search_from + rel; - // Must be at a line start (previous char is \n or BOF). - let at_line_start = - heading_start == 0 || content.as_bytes().get(heading_start - 1) == Some(&b'\n'); - if !at_line_start { - search_from = heading_start + marker.len(); - continue; - } - let after = heading_start + marker.len(); - let rest = &content[after..]; - // Walk lines after the heading. Skip the (typically empty) - // line fragment between the heading's colon and the first - // newline so the first real line is the first bullet. - let mut cursor = after; - let mut seen_content = false; - for line in rest.lines() { - let line_start = cursor; - cursor += line.len(); - if content.as_bytes().get(cursor) == Some(&b'\n') { - cursor += 1; - } - let trimmed = line.trim_start(); - if trimmed.is_empty() { - if seen_content { - break; - } - continue; - } - seen_content = true; - // Accept `- `, `* `, `• `, or a bare line as bullet. - let body = trimmed - .strip_prefix("- ") - .or_else(|| trimmed.strip_prefix("* ")) - .or_else(|| trimmed.strip_prefix("• ")) - .unwrap_or(trimmed); - let Some((url, rest_after)) = split_url(body) else { - break; - }; - // Compute absolute offset of the URL in `content`. - let body_offset_in_line = line.len() - body.len(); - let url_abs_start = line_start + body_offset_in_line; - let url_abs_end = url_abs_start + url.len(); - let title = rest_after - .trim() - .trim_matches(|c: char| c == '-' || c.is_whitespace()); - let title_out = if title.is_empty() { - url.clone() - } else { - title.to_string() - }; - out.push(Citation { - start_index: url_abs_start, - end_index: url_abs_end, - url, - title: title_out, - }); - } - search_from = after; - } - } - out -} - -/// Split `s` into `(url, rest)` where `url` is the first http(s) token. -/// Returns `None` when the string doesn't begin with a URL. -fn split_url(s: &str) -> Option<(String, &str)> { - if !(s.starts_with("http://") || s.starts_with("https://")) { - return None; - } - let end = s - .find(|c: char| c.is_whitespace() || matches!(c, ')' | ']' | '>' | '`')) - .unwrap_or(s.len()); - let url = s[..end].trim_end_matches(['.', ',', ';', ':', '!', '?']); - if url.len() <= "https://".len() { - return None; - } - Some((url.to_string(), &s[end..])) -} - /// Merge two citation lists, deduping by URL (keeping the first hit /// in document order). Used to combine the bare-URL extractor's output /// with structured citations without emitting the same URL twice. @@ -551,69 +312,6 @@ fn mask_code_spans(content: &str) -> String { mod tests { use super::*; - fn urls(cits: &[Citation]) -> Vec<(&str, &str, usize, usize)> { - cits.iter() - .map(|c| (c.url.as_str(), c.title.as_str(), c.start_index, c.end_index)) - .collect() - } - - #[test] - fn footnote_ref_resolved() { - let input = "See the docs[^1] for more.\n\n[^1]: https://example.com/api The API reference"; - let got = extract(input); - let u = urls(&got); - assert_eq!(u.len(), 1); - assert_eq!(u[0].0, "https://example.com/api"); - assert_eq!(u[0].1, "The API reference"); - } - - #[test] - fn footnote_without_title_falls_back_to_label() { - let input = "Cite me[^src].\n[^src]: https://example.com"; - let got = extract(input); - let u = urls(&got); - assert_eq!(u.len(), 1); - assert_eq!(u[0].0, "https://example.com"); - assert_eq!(u[0].1, "src"); - } - - #[test] - fn numeric_refs_resolved() { - let input = - "See [1] and [2].\n\n[1] https://a.example.com\n[2] https://b.example.com Example B"; - let got = extract(input); - let u = urls(&got); - assert_eq!(u.len(), 2); - assert_eq!(u[0].0, "https://a.example.com"); - assert_eq!(u[1].0, "https://b.example.com"); - assert_eq!(u[1].1, "Example B"); - } - - #[test] - fn numeric_ref_ignores_non_refs() { - let input = "array[0] is [abc].\nNo definitions here."; - let got = extract(input); - assert!(got.is_empty()); - } - - #[test] - fn sources_block_extracted() { - let input = "Some answer.\n\nSources:\n- https://a.example.com\n- https://b.example.com short description\n\nOther text."; - let got = extract(input); - let u = urls(&got); - assert_eq!(u.len(), 2); - assert_eq!(u[0].0, "https://a.example.com"); - assert_eq!(u[1].0, "https://b.example.com"); - assert_eq!(u[1].1, "short description"); - } - - #[test] - fn references_heading_also_matches() { - let input = "Stuff.\n\nReferences:\n- https://x.example.com\n"; - let got = extract(input); - assert_eq!(got.len(), 1); - } - #[test] fn no_citations_returns_empty() { let got = extract("plain text with no citations"); diff --git a/crates/spark-server/src/citation_structured.rs b/crates/spark-server/src/citation_structured.rs new file mode 100644 index 000000000..085b04f74 --- /dev/null +++ b/crates/spark-server/src/citation_structured.rs @@ -0,0 +1,319 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Structured "model-emitted" citation extractors: markdown footnotes, +//! numeric bracket refs, and fenced `Sources:` sections. See the +//! [`crate::citation`] module docs for the recognized shapes; these are +//! the pattern parsers behind [`crate::citation::extract`]. + +use crate::citation::Citation; + +/// Parse markdown footnote references: every `[^label]` in text paired +/// with a `[^label]: url [title]` definition. +pub(crate) fn footnote_citations(content: &str) -> Vec { + let defs = collect_footnote_defs(content); + if defs.is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + let mut i = 0; + let bytes = content.as_bytes(); + while i < bytes.len() { + if let Some(rel) = content[i..].find("[^") { + let start = i + rel; + // Skip footnote definitions (`[^x]:` at line start). + let after_br = &content[start..]; + if let Some(close_rel) = after_br.find(']') { + let close = start + close_rel; + // Reject if this is a definition line (next char is ':'). + if content.as_bytes().get(close + 1) == Some(&b':') { + i = close + 1; + continue; + } + let label = &content[start + 2..close]; + if let Some((url, title)) = defs.get(label) { + out.push(Citation { + start_index: start, + end_index: close + 1, + url: url.clone(), + title: title.clone().unwrap_or_else(|| label.to_string()), + }); + } + i = close + 1; + } else { + break; + } + } else { + break; + } + } + out +} + +/// Walk each line; return `{label: (url, optional_title)}` for lines +/// matching `[^label]: url [title]`. +fn collect_footnote_defs( + content: &str, +) -> std::collections::HashMap)> { + let mut map = std::collections::HashMap::new(); + for line in content.lines() { + let trimmed = line.trim_start(); + let Some(rest) = trimmed.strip_prefix("[^") else { + continue; + }; + let Some(close_rel) = rest.find(']') else { + continue; + }; + let label = &rest[..close_rel]; + let after = &rest[close_rel + 1..]; + let Some(body) = after.strip_prefix(':') else { + continue; + }; + let body = body.trim_start(); + let Some((url, rest)) = split_url(body) else { + continue; + }; + let title = rest.trim(); + let title_opt = if title.is_empty() { + None + } else { + Some(title.to_string()) + }; + map.insert(label.to_string(), (url, title_opt)); + } + map +} + +/// Parse `[N]` refs paired with a `[N] url` definition line. +pub(crate) fn numeric_ref_citations(content: &str) -> Vec { + let defs = collect_numeric_defs(content); + if defs.is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + let mut i = 0; + while i < content.len() { + let Some(rel) = content[i..].find('[') else { + break; + }; + let start = i + rel; + let after = &content[start + 1..]; + let Some(close_rel) = after.find(']') else { + break; + }; + let label = &after[..close_rel]; + if label.bytes().all(|b| b.is_ascii_digit()) && !label.is_empty() { + let close = start + 1 + close_rel; + // Skip definition sites. + let at_line_start = start == 0 || content.as_bytes().get(start - 1) == Some(&b'\n'); + let next_is_url_hint = { + let next = content.get(close + 1..).unwrap_or(""); + next.trim_start().starts_with("http") + }; + let is_definition = at_line_start && next_is_url_hint; + if !is_definition && let Some((url, title)) = defs.get(label) { + out.push(Citation { + start_index: start, + end_index: close + 1, + url: url.clone(), + title: title.clone().unwrap_or_else(|| label.to_string()), + }); + } + i = close + 1; + } else { + i = start + 1; + } + } + out +} + +fn collect_numeric_defs( + content: &str, +) -> std::collections::HashMap)> { + let mut map = std::collections::HashMap::new(); + for line in content.lines() { + let trimmed = line.trim_start(); + let Some(rest) = trimmed.strip_prefix('[') else { + continue; + }; + let Some(close_rel) = rest.find(']') else { + continue; + }; + let label = &rest[..close_rel]; + if label.is_empty() || !label.bytes().all(|b| b.is_ascii_digit()) { + continue; + } + let mut after = &rest[close_rel + 1..]; + if let Some(stripped) = after.strip_prefix(':') { + after = stripped; + } + let body = after.trim_start(); + let Some((url, rest)) = split_url(body) else { + continue; + }; + let title = rest.trim(); + let title_opt = if title.is_empty() { + None + } else { + Some(title.to_string()) + }; + map.insert(label.to_string(), (url, title_opt)); + } + map +} + +/// Find `Sources:` or `References:` heading and emit a citation for +/// each bullet line containing an http(s) URL until a blank line or a +/// non-bullet line breaks the section. +pub(crate) fn sources_block_citations(content: &str) -> Vec { + let lower = content.to_ascii_lowercase(); + let markers = ["sources:", "references:", "citations:"]; + let mut out = Vec::new(); + for marker in &markers { + let mut search_from = 0usize; + while let Some(rel) = lower[search_from..].find(marker) { + let heading_start = search_from + rel; + // Must be at a line start (previous char is \n or BOF). + let at_line_start = + heading_start == 0 || content.as_bytes().get(heading_start - 1) == Some(&b'\n'); + if !at_line_start { + search_from = heading_start + marker.len(); + continue; + } + let after = heading_start + marker.len(); + let rest = &content[after..]; + // Walk lines after the heading. Skip the (typically empty) + // line fragment between the heading's colon and the first + // newline so the first real line is the first bullet. + let mut cursor = after; + let mut seen_content = false; + for line in rest.lines() { + let line_start = cursor; + cursor += line.len(); + if content.as_bytes().get(cursor) == Some(&b'\n') { + cursor += 1; + } + let trimmed = line.trim_start(); + if trimmed.is_empty() { + if seen_content { + break; + } + continue; + } + seen_content = true; + // Accept `- `, `* `, `• `, or a bare line as bullet. + let body = trimmed + .strip_prefix("- ") + .or_else(|| trimmed.strip_prefix("* ")) + .or_else(|| trimmed.strip_prefix("• ")) + .unwrap_or(trimmed); + let Some((url, rest_after)) = split_url(body) else { + break; + }; + // Compute absolute offset of the URL in `content`. + let body_offset_in_line = line.len() - body.len(); + let url_abs_start = line_start + body_offset_in_line; + let url_abs_end = url_abs_start + url.len(); + let title = rest_after + .trim() + .trim_matches(|c: char| c == '-' || c.is_whitespace()); + let title_out = if title.is_empty() { + url.clone() + } else { + title.to_string() + }; + out.push(Citation { + start_index: url_abs_start, + end_index: url_abs_end, + url, + title: title_out, + }); + } + search_from = after; + } + } + out +} + +/// Split `s` into `(url, rest)` where `url` is the first http(s) token. +/// Returns `None` when the string doesn't begin with a URL. +fn split_url(s: &str) -> Option<(String, &str)> { + if !(s.starts_with("http://") || s.starts_with("https://")) { + return None; + } + let end = s + .find(|c: char| c.is_whitespace() || matches!(c, ')' | ']' | '>' | '`')) + .unwrap_or(s.len()); + let url = s[..end].trim_end_matches(['.', ',', ';', ':', '!', '?']); + if url.len() <= "https://".len() { + return None; + } + Some((url.to_string(), &s[end..])) +} + +#[cfg(test)] +mod tests { + use crate::citation::{Citation, extract}; + + fn urls(cits: &[Citation]) -> Vec<(&str, &str, usize, usize)> { + cits.iter() + .map(|c| (c.url.as_str(), c.title.as_str(), c.start_index, c.end_index)) + .collect() + } + + #[test] + fn footnote_ref_resolved() { + let input = "See the docs[^1] for more.\n\n[^1]: https://example.com/api The API reference"; + let got = extract(input); + let u = urls(&got); + assert_eq!(u.len(), 1); + assert_eq!(u[0].0, "https://example.com/api"); + assert_eq!(u[0].1, "The API reference"); + } + + #[test] + fn footnote_without_title_falls_back_to_label() { + let input = "Cite me[^src].\n[^src]: https://example.com"; + let got = extract(input); + let u = urls(&got); + assert_eq!(u.len(), 1); + assert_eq!(u[0].0, "https://example.com"); + assert_eq!(u[0].1, "src"); + } + + #[test] + fn numeric_refs_resolved() { + let input = + "See [1] and [2].\n\n[1] https://a.example.com\n[2] https://b.example.com Example B"; + let got = extract(input); + let u = urls(&got); + assert_eq!(u.len(), 2); + assert_eq!(u[0].0, "https://a.example.com"); + assert_eq!(u[1].0, "https://b.example.com"); + assert_eq!(u[1].1, "Example B"); + } + + #[test] + fn numeric_ref_ignores_non_refs() { + let input = "array[0] is [abc].\nNo definitions here."; + let got = extract(input); + assert!(got.is_empty()); + } + + #[test] + fn sources_block_extracted() { + let input = "Some answer.\n\nSources:\n- https://a.example.com\n- https://b.example.com short description\n\nOther text."; + let got = extract(input); + let u = urls(&got); + assert_eq!(u.len(), 2); + assert_eq!(u[0].0, "https://a.example.com"); + assert_eq!(u[1].0, "https://b.example.com"); + assert_eq!(u[1].1, "short description"); + } + + #[test] + fn references_heading_also_matches() { + let input = "Stuff.\n\nReferences:\n- https://x.example.com\n"; + let got = extract(input); + assert_eq!(got.len(), 1); + } +} diff --git a/crates/spark-server/src/main.rs b/crates/spark-server/src/main.rs index 944a81d24..fe86f0b96 100644 --- a/crates/spark-server/src/main.rs +++ b/crates/spark-server/src/main.rs @@ -26,6 +26,7 @@ mod anthropic; mod api; mod auth; mod citation; +mod citation_structured; mod cli; mod conversation_store; pub mod grammar; diff --git a/crates/spark-server/src/openai/tests.rs b/crates/spark-server/src/openai/tests.rs index bce83ba3e..1faaa88da 100644 --- a/crates/spark-server/src/openai/tests.rs +++ b/crates/spark-server/src/openai/tests.rs @@ -1,632 +1,9 @@ // SPDX-License-Identifier: AGPL-3.0-only -use super::*; +//! OpenAI wire-type tests, grouped by API surface. -fn url_of(a: &Annotation) -> (usize, usize, &str, &str) { - match a { - Annotation::UrlCitation { - url_citation: - UrlCitation { - start_index, - end_index, - url, - title, - }, - } => (*start_index, *end_index, url.as_str(), title.as_str()), - } -} - -#[test] -fn bare_url_extracted() { - let got = extract_url_annotations("see https://example.com/foo for more").unwrap(); - assert_eq!(got.len(), 1); - let (s, e, u, t) = url_of(&got[0]); - assert_eq!(u, "https://example.com/foo"); - assert_eq!(t, "https://example.com/foo"); - assert_eq!(s, 4); - assert_eq!(e, 4 + "https://example.com/foo".len()); -} - -#[test] -fn trailing_sentence_punct_stripped() { - let got = extract_url_annotations("go to https://example.com.").unwrap(); - let (_, _, u, _) = url_of(&got[0]); - assert_eq!(u, "https://example.com"); -} - -#[test] -fn wikipedia_parens_preserved() { - let got = extract_url_annotations("see https://en.wikipedia.org/wiki/Foo_(bar) now").unwrap(); - let (_, _, u, _) = url_of(&got[0]); - assert_eq!(u, "https://en.wikipedia.org/wiki/Foo_(bar)"); -} - -#[test] -fn markdown_link_uses_title() { - let got = extract_url_annotations("read [the docs](https://example.com/api) today").unwrap(); - assert_eq!(got.len(), 1); - let (_, _, u, t) = url_of(&got[0]); - assert_eq!(u, "https://example.com/api"); - assert_eq!(t, "the docs"); -} - -#[test] -fn url_in_fenced_code_skipped() { - let input = "run this:\n```bash\ncurl https://example.com\n```\ndone"; - assert!(extract_url_annotations(input).is_none()); -} - -#[test] -fn url_in_inline_code_skipped() { - let input = "use `curl https://example.com` to fetch"; - assert!(extract_url_annotations(input).is_none()); -} - -#[test] -fn multiple_urls_sorted_by_position() { - let input = "first https://a.example.com and [second](https://b.example.com)"; - let got = extract_url_annotations(input).unwrap(); - assert_eq!(got.len(), 2); - let (s0, _, u0, _) = url_of(&got[0]); - let (s1, _, u1, _) = url_of(&got[1]); - assert!(s0 < s1); - assert_eq!(u0, "https://a.example.com"); - assert_eq!(u1, "https://b.example.com"); -} - -#[test] -fn non_http_ignored() { - assert!(extract_url_annotations("ftp://example.com not a citation").is_none()); -} - -#[test] -fn empty_input_returns_none() { - assert!(extract_url_annotations("").is_none()); - assert!(extract_url_annotations("no URLs here").is_none()); -} - -#[test] -fn query_and_fragment_preserved() { - let got = extract_url_annotations("see https://example.com/p?q=1&r=2#frag here").unwrap(); - let (_, _, u, _) = url_of(&got[0]); - assert_eq!(u, "https://example.com/p?q=1&r=2#frag"); -} - -// TODO: `mask_code_spans` was an internal helper that no longer exists -// after the URL-annotations refactor. The remaining call to -// `extract_url_annotations` is exercised by the other tests in this file; -// re-add a UTF-8 boundary test once the new internal mask helper has a -// stable name. - -fn lower_with_tools( - tools: serde_json::Value, -) -> Result { - let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ - "model": "test-model", - "input": "ping", - "tools": tools, - })) - .expect("ResponsesRequest deserialize"); - lower_responses_to_chat(req, |_| None) -} - -#[test] -fn responses_flat_function_tool_accepted() { - // OpenAI's official Python SDK sends function tools in the flat - // shape `{type, name, description, parameters}` — no nested - // `function` object. Atlas must accept both shapes. - let chat = lower_with_tools(serde_json::json!([ - { - "type": "function", - "name": "get_weather", - "description": "look up weather", - "parameters": {"type": "object", "properties": {"loc": {"type": "string"}}, "required": ["loc"]} - } - ])).expect("flat-form function tool should parse"); - let tools = chat.tools.expect("tools present"); - assert_eq!(tools.len(), 1); - assert_eq!(tools[0].tool_type, "function"); - assert_eq!(tools[0].function.name, "get_weather"); -} - -#[test] -fn responses_nested_function_tool_still_accepted() { - // Backwards-compat: chat-completions-style `{type, function:{...}}` - // must keep working since older clients send it. - let chat = lower_with_tools(serde_json::json!([ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object"} - } - } - ])) - .expect("nested-form function tool should parse"); - let tools = chat.tools.expect("tools present"); - assert_eq!(tools.len(), 1); - assert_eq!(tools[0].function.name, "get_weather"); -} - -#[test] -fn responses_flat_tool_choice_accepted() { - let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ - "model": "test", - "input": "ping", - "tool_choice": {"type": "function", "name": "get_weather"}, - })) - .unwrap(); - let chat = lower_responses_to_chat(req, |_| None).expect("flat tool_choice"); - match chat.tool_choice { - Some(crate::tool_parser::ToolChoice::Specific { function }) => { - assert_eq!(function.name, "get_weather"); - } - other => panic!("expected Specific tool_choice, got {other:?}"), - } -} - -#[test] -fn responses_string_tool_choice_accepted() { - let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ - "model": "test", - "input": "ping", - "tool_choice": "required", - })) - .unwrap(); - let chat = lower_responses_to_chat(req, |_| None).expect("string tool_choice"); - match chat.tool_choice { - Some(crate::tool_parser::ToolChoice::Mode(s)) => { - assert_eq!(s, "required"); - } - other => panic!("expected Mode tool_choice, got {other:?}"), - } -} - -#[test] -fn markdown_link_with_parens_in_url_preserved() { - // Wikipedia URLs contain `(...)` which the bare `find(')')` would - // truncate. Verify the balanced-paren scan keeps the full URL. - let got = - extract_url_annotations("see [Foo (bar)](https://en.wikipedia.org/wiki/Foo_(bar)) here") - .unwrap(); - assert_eq!(got.len(), 1); - let (_, _, u, t) = url_of(&got[0]); - assert_eq!(u, "https://en.wikipedia.org/wiki/Foo_(bar)"); - assert_eq!(t, "Foo (bar)"); -} - -#[test] -fn responses_input_image_string_url_is_carried() { - // Regression: the Responses adapter used to drop images entirely - // (`images: Vec::new()`), silently losing multimodal input. - let item = serde_json::json!({ - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "what is this?"}, - {"type": "input_image", "image_url": "data:image/png;base64,AAA"} - ] - }); - let m = IncomingMessage::from_responses_input_item(&item).expect("message"); - assert_eq!(m.content.text, "what is this?"); - assert_eq!( - m.content.images, - vec!["data:image/png;base64,AAA".to_string()] - ); -} - -#[test] -fn responses_input_image_object_url_is_carried() { - // Some SDKs nest the url: `{image_url: {url: "..."}}`. - let item = serde_json::json!({ - "type": "message", - "role": "user", - "content": [ - {"type": "input_image", "image_url": {"url": "https://example.com/a.png"}} - ] - }); - let m = IncomingMessage::from_responses_input_item(&item).expect("message"); - assert_eq!( - m.content.images, - vec!["https://example.com/a.png".to_string()] - ); - assert_eq!(m.content.text, ""); -} - -#[test] -fn responses_function_call_output_image_is_carried() { - // #165 parity for the Responses surface: a screenshot returned by a - // tool (`function_call_output` with structured output parts) must - // reach the vision encoder, exactly like Anthropic tool_result - // images and chat-completions role:"tool" array content. - let item = serde_json::json!({ - "type": "function_call_output", - "call_id": "call_1", - "output": [ - {"type": "output_text", "text": "screenshot follows"}, - {"type": "input_image", "image_url": "data:image/png;base64,BBB"} - ] - }); - let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); - assert_eq!(m.role, "tool"); - assert_eq!(m.tool_call_id.as_deref(), Some("call_1")); - assert_eq!(m.content.text, "screenshot follows"); - assert_eq!( - m.content.images, - vec!["data:image/png;base64,BBB".to_string()] - ); -} - -#[test] -fn responses_function_call_output_string_unchanged() { - let item = serde_json::json!({ - "type": "function_call_output", - "call_id": "call_2", - "output": "plain result" - }); - let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); - assert_eq!(m.content.text, "plain result"); - assert!(m.content.images.is_empty()); -} - -#[test] -fn responses_function_call_output_opaque_array_stringified() { - // Out-of-spec array (no recognizable parts) keeps the historical - // stringified-JSON behavior instead of silently emptying the result. - let opaque = serde_json::json!([{"weather": "sunny", "temp_c": 21}]); - let item = serde_json::json!({ - "type": "function_call_output", - "call_id": "call_3", - "output": opaque.clone() - }); - let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); - assert_eq!(m.content.text, opaque.to_string()); - assert!(m.content.images.is_empty()); -} - -#[test] -fn responses_in_progress_event_name() { - let ev = ResponsesStreamEvent::InProgress { - sequence_number: 1, - response: ResponsesStreamEnvelope { - id: "resp_test".into(), - object: "response", - created_at: 0, - model: "m".into(), - status: "in_progress", - metadata: None, - }, - }; - assert_eq!(responses_event_name(&ev), "response.in_progress"); -} - -// ── return_token_ids wire format ──────────────────────────────────── - -#[test] -fn token_ids_absent_by_default_keeps_wire_byte_identical() { - // PCND: a client that did not opt in must see no `token_ids` key. - let chunk = ChatCompletionChunk::content_chunk("m", "id", "hi".into()); - let json = serde_json::to_string(&chunk).unwrap(); - assert!(!json.contains("token_ids"), "default wire changed: {json}"); - // Empty `with_token_ids` is a no-op (still absent). - let chunk = - ChatCompletionChunk::content_chunk("m", "id", "hi".into()).with_token_ids(Vec::new()); - let json = serde_json::to_string(&chunk).unwrap(); - assert!(!json.contains("token_ids")); -} - -#[test] -fn with_token_ids_stamps_first_choice() { - let chunk = - ChatCompletionChunk::content_chunk("m", "id", "hi".into()).with_token_ids(vec![10, 20, 30]); - assert_eq!(chunk.choices[0].token_ids, vec![10, 20, 30]); - let json = serde_json::to_string(&chunk).unwrap(); - assert!(json.contains("\"token_ids\":[10,20,30]"), "{json}"); - // No choices (usage-only chunk) → no panic, no-op. - let usage = Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - prompt_tokens_details: None, - completion_tokens_details: None, - time_to_first_token_ms: 0.0, - response_tokens_per_second: 0.0, - }; - let chunk = ChatCompletionChunk::usage_only_chunk("m", "id", usage).with_token_ids(vec![1, 2]); - assert!(chunk.choices.is_empty()); -} - -// ── reasoning wire format: exactly one field ──────────────────────── -// A response carrying BOTH `reasoning_content` and a `reasoning` mirror is -// rejected by strict OpenAI-compatible clients (they assert exactly one). -// Atlas emits only `reasoning_content` — these lock that contract in. - -#[test] -fn reasoning_delta_emits_only_reasoning_content() { - let chunk = ChatCompletionChunk::reasoning_chunk("m", "id", "thinking".into()); - let json = serde_json::to_string(&chunk).unwrap(); - assert!( - json.contains("\"reasoning_content\":\"thinking\""), - "reasoning_content missing: {json}" - ); - assert!( - !json.contains("\"reasoning\":"), - "mirror `reasoning` field leaked into stream delta: {json}" - ); -} - -#[test] -fn blocking_message_emits_only_reasoning_content() { - let msg = ChatMessage { - role: "assistant".into(), - reasoning_content: Some("thinking".into()), - content: Some("hi".into()), - tool_calls: None, - annotations: None, - refusal: None, - }; - let json = serde_json::to_string(&msg).unwrap(); - assert!( - json.contains("\"reasoning_content\":\"thinking\""), - "reasoning_content missing: {json}" - ); - assert!( - !json.contains("\"reasoning\":"), - "mirror `reasoning` field leaked into message: {json}" - ); -} - -// ── Thinking directive (client channels → ir::ThinkingDirective) ── - -use crate::ir::ThinkingDirective; - -fn chat_req(body: serde_json::Value) -> ChatCompletionRequest { - serde_json::from_value(body).expect("valid chat request") -} - -fn base_body() -> serde_json::Value { - serde_json::json!({ - "model": "test", - "messages": [{"role": "user", "content": "hi"}], - }) -} - -#[test] -fn silent_request_is_unspecified() { - let req = chat_req(base_body()); - assert_eq!( - req.client_thinking_directive(), - ThinkingDirective::Unspecified - ); - assert!(!req.client_thinking_directive().is_explicit()); -} - -#[test] -fn anthropic_thinking_channel() { - // type=disabled wins even with a budget present. - let mut b = base_body(); - b["thinking"] = serde_json::json!({"type": "disabled", "budget_tokens": 100}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Off - ); - - let mut b = base_body(); - b["thinking"] = serde_json::json!({"type": "enabled", "budget_tokens": 512}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: Some(512) } - ); - - // Adaptive / budget-less thinking object → think as long as needed - // (budget defers to the per-model max_thinking_budget). - let mut b = base_body(); - b["thinking"] = serde_json::json!({"type": "adaptive"}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: None } - ); -} - -#[test] -fn thinking_token_budget_channel() { - let mut b = base_body(); - b["thinking_token_budget"] = serde_json::json!(512); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: Some(512) } - ); - - let mut b = base_body(); - b["thinking_token_budget"] = serde_json::json!(0); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Off - ); -} - -#[test] -fn reasoning_effort_channel() { - for (effort, expect) in [ - ("none", ThinkingDirective::Off), - ("minimal", ThinkingDirective::On { budget: Some(64) }), - ("low", ThinkingDirective::On { budget: Some(128) }), - ("medium", ThinkingDirective::On { budget: Some(256) }), - ("high", ThinkingDirective::On { budget: Some(512) }), - ("xhigh", ThinkingDirective::On { budget: Some(1024) }), - ("max", ThinkingDirective::On { budget: Some(1024) }), - // Unknown efforts fall back to the conservative default budget. - ("bogus", ThinkingDirective::On { budget: Some(256) }), - ] { - let mut b = base_body(); - b["reasoning"] = serde_json::json!({"effort": effort}); - assert_eq!( - chat_req(b).client_thinking_directive(), - expect, - "effort={effort}" - ); - } -} - -#[test] -fn chat_template_kwargs_channel() { - // Struct still parses as a request-body wire field. - let kw: ChatTemplateKwargs = - serde_json::from_str(r#"{"enable_thinking":true,"thinking_budget":1024}"#) - .expect("should parse"); - assert_eq!(kw.enable_thinking, Some(true)); - assert_eq!(kw.thinking_budget, Some(1024)); - - // Budget rung wins over the enable flag. - let mut b = base_body(); - b["chat_template_kwargs"] = - serde_json::json!({"enable_thinking": false, "thinking_budget": 1024}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: Some(1024) } - ); - - let mut b = base_body(); - b["chat_template_kwargs"] = serde_json::json!({"thinking_budget": 0}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Off - ); - - // enable_thinking with no explicit budget defers to the per-model - // max_thinking_budget (budget: None), not the conservative 256-token - // default — a hard cut force-injects mid-reasoning and - // wrecks agentic tool selection. - let mut b = base_body(); - b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": true}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: None } - ); - - let mut b = base_body(); - b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": false}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Off - ); - - // Empty kwargs object carries no intent. - let mut b = base_body(); - b["chat_template_kwargs"] = serde_json::json!({}); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Unspecified - ); -} - -#[test] -fn legacy_enable_thinking_channel() { - let mut b = base_body(); - b["enable_thinking"] = serde_json::json!(true); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::On { budget: None } - ); - - // false is the serde default — indistinguishable from absent, so it - // must NOT count as an explicit opt-out. - let mut b = base_body(); - b["enable_thinking"] = serde_json::json!(false); - assert_eq!( - chat_req(b).client_thinking_directive(), - ThinkingDirective::Unspecified - ); -} - -// ── Legacy /v1/completions echo + logprobs wire types ── - -#[test] -fn completion_request_echo_logprobs_n_deser() { - let req: CompletionRequest = serde_json::from_value(serde_json::json!({ - "model": "test", - "prompt": "hello", - "echo": true, - "logprobs": 3, - "n": 2, - "max_tokens": 0, - "stream_options": {"include_usage": true}, - "user": "eval-harness", - "suffix": "tail", - "best_of": 2, - })) - .expect("valid completion request"); - assert!(req.echo); - assert_eq!(req.logprobs, Some(3)); - assert_eq!(req.n, 2); - assert_eq!(req.max_tokens, 0); - assert!(req.stream_options.expect("stream_options").include_usage); -} - -#[test] -fn completion_request_defaults_match_openai_spec() { - // echo=false, n=1, logprobs absent — the spec defaults; a request - // that names none of the new fields must behave exactly as before. - let req: CompletionRequest = serde_json::from_value(serde_json::json!({ - "model": "test", - "prompt": "hello", - })) - .expect("valid completion request"); - assert!(!req.echo); - assert_eq!(req.n, 1); - assert!(req.logprobs.is_none()); - assert!(req.stream_options.is_none()); -} - -#[test] -fn completion_logprobs_serializes_four_parallel_arrays_with_nulls() { - let lp = CompletionLogprobs { - tokens: vec!["He".into(), "llo".into()], - token_logprobs: vec![None, Some(-0.5)], - top_logprobs: vec![ - None, - Some(std::collections::HashMap::from([( - "llo".to_string(), - -0.5f32, - )])), - ], - text_offset: vec![0, 2], - }; - let v = serde_json::to_value(&lp).expect("serialize"); - // Legacy shape: 4 parallel arrays, JSON null for the first echoed token. - assert!(v["token_logprobs"][0].is_null()); - assert!(v["top_logprobs"][0].is_null()); - assert_eq!(v["tokens"].as_array().map(Vec::len), Some(2)); - assert_eq!(v["text_offset"][1], 2); -} - -#[test] -fn completion_response_carries_system_fingerprint_and_optional_logprobs() { - let usage = Usage { - prompt_tokens: 1, - completion_tokens: 1, - total_tokens: 2, - prompt_tokens_details: None, - completion_tokens_details: None, - time_to_first_token_ms: 0.0, - response_tokens_per_second: 0.0, - }; - let resp = CompletionResponse::new("m", "hi".into(), usage, "stop"); - let v = serde_json::to_value(&resp).expect("serialize"); - assert_eq!(v["system_fingerprint"], "fp_atlas"); - // logprobs must be ABSENT (not null) when not requested — some - // clients treat an explicit null as a malformed logprobs block. - assert!(v["choices"][0].get("logprobs").is_none()); -} - -#[test] -fn completion_request_n_bounds_are_handler_enforced() { - // serde accepts any usize; the HANDLER rejects n==0 and n>128 with a - // 400 (OpenAI spec bound). This test locks the parse side: values - // arrive intact for the handler check (no silent serde clamping). - let req: CompletionRequest = serde_json::from_value(serde_json::json!({ - "model": "m", "prompt": "p", "n": 4096, - })) - .expect("parse"); - assert_eq!(req.n, 4096); -} +mod annotations; +mod chat_wire; +mod completions; +mod responses; +mod thinking; diff --git a/crates/spark-server/src/openai/tests/annotations.rs b/crates/spark-server/src/openai/tests/annotations.rs new file mode 100644 index 000000000..4fa5b9983 --- /dev/null +++ b/crates/spark-server/src/openai/tests/annotations.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! URL-annotation extraction (`extract_url_annotations`) wire tests. + +use crate::openai::*; + +fn url_of(a: &Annotation) -> (usize, usize, &str, &str) { + match a { + Annotation::UrlCitation { + url_citation: + UrlCitation { + start_index, + end_index, + url, + title, + }, + } => (*start_index, *end_index, url.as_str(), title.as_str()), + } +} + +#[test] +fn bare_url_extracted() { + let got = extract_url_annotations("see https://example.com/foo for more").unwrap(); + assert_eq!(got.len(), 1); + let (s, e, u, t) = url_of(&got[0]); + assert_eq!(u, "https://example.com/foo"); + assert_eq!(t, "https://example.com/foo"); + assert_eq!(s, 4); + assert_eq!(e, 4 + "https://example.com/foo".len()); +} + +#[test] +fn trailing_sentence_punct_stripped() { + let got = extract_url_annotations("go to https://example.com.").unwrap(); + let (_, _, u, _) = url_of(&got[0]); + assert_eq!(u, "https://example.com"); +} + +#[test] +fn wikipedia_parens_preserved() { + let got = extract_url_annotations("see https://en.wikipedia.org/wiki/Foo_(bar) now").unwrap(); + let (_, _, u, _) = url_of(&got[0]); + assert_eq!(u, "https://en.wikipedia.org/wiki/Foo_(bar)"); +} + +#[test] +fn markdown_link_uses_title() { + let got = extract_url_annotations("read [the docs](https://example.com/api) today").unwrap(); + assert_eq!(got.len(), 1); + let (_, _, u, t) = url_of(&got[0]); + assert_eq!(u, "https://example.com/api"); + assert_eq!(t, "the docs"); +} + +#[test] +fn markdown_link_with_parens_in_url_preserved() { + // Wikipedia URLs contain `(...)` which the bare `find(')')` would + // truncate. Verify the balanced-paren scan keeps the full URL. + let got = + extract_url_annotations("see [Foo (bar)](https://en.wikipedia.org/wiki/Foo_(bar)) here") + .unwrap(); + assert_eq!(got.len(), 1); + let (_, _, u, t) = url_of(&got[0]); + assert_eq!(u, "https://en.wikipedia.org/wiki/Foo_(bar)"); + assert_eq!(t, "Foo (bar)"); +} + +#[test] +fn url_in_fenced_code_skipped() { + let input = "run this:\n```bash\ncurl https://example.com\n```\ndone"; + assert!(extract_url_annotations(input).is_none()); +} + +#[test] +fn url_in_inline_code_skipped() { + let input = "use `curl https://example.com` to fetch"; + assert!(extract_url_annotations(input).is_none()); +} + +#[test] +fn multiple_urls_sorted_by_position() { + let input = "first https://a.example.com and [second](https://b.example.com)"; + let got = extract_url_annotations(input).unwrap(); + assert_eq!(got.len(), 2); + let (s0, _, u0, _) = url_of(&got[0]); + let (s1, _, u1, _) = url_of(&got[1]); + assert!(s0 < s1); + assert_eq!(u0, "https://a.example.com"); + assert_eq!(u1, "https://b.example.com"); +} + +#[test] +fn non_http_ignored() { + assert!(extract_url_annotations("ftp://example.com not a citation").is_none()); +} + +#[test] +fn empty_input_returns_none() { + assert!(extract_url_annotations("").is_none()); + assert!(extract_url_annotations("no URLs here").is_none()); +} + +#[test] +fn query_and_fragment_preserved() { + let got = extract_url_annotations("see https://example.com/p?q=1&r=2#frag here").unwrap(); + let (_, _, u, _) = url_of(&got[0]); + assert_eq!(u, "https://example.com/p?q=1&r=2#frag"); +} + +// TODO: `mask_code_spans` was an internal helper that no longer exists +// after the URL-annotations refactor. The remaining call to +// `extract_url_annotations` is exercised by the other tests in this file; +// re-add a UTF-8 boundary test once the new internal mask helper has a +// stable name. diff --git a/crates/spark-server/src/openai/tests/chat_wire.rs b/crates/spark-server/src/openai/tests/chat_wire.rs new file mode 100644 index 000000000..9bdebc932 --- /dev/null +++ b/crates/spark-server/src/openai/tests/chat_wire.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Chat-completions wire-format contracts: `token_ids` opt-in and the +//! reasoning single-field rule. + +use crate::openai::*; + +// ── return_token_ids wire format ──────────────────────────────────── + +#[test] +fn token_ids_absent_by_default_keeps_wire_byte_identical() { + // PCND: a client that did not opt in must see no `token_ids` key. + let chunk = ChatCompletionChunk::content_chunk("m", "id", "hi".into()); + let json = serde_json::to_string(&chunk).unwrap(); + assert!(!json.contains("token_ids"), "default wire changed: {json}"); + // Empty `with_token_ids` is a no-op (still absent). + let chunk = + ChatCompletionChunk::content_chunk("m", "id", "hi".into()).with_token_ids(Vec::new()); + let json = serde_json::to_string(&chunk).unwrap(); + assert!(!json.contains("token_ids")); +} + +#[test] +fn with_token_ids_stamps_first_choice() { + let chunk = + ChatCompletionChunk::content_chunk("m", "id", "hi".into()).with_token_ids(vec![10, 20, 30]); + assert_eq!(chunk.choices[0].token_ids, vec![10, 20, 30]); + let json = serde_json::to_string(&chunk).unwrap(); + assert!(json.contains("\"token_ids\":[10,20,30]"), "{json}"); + // No choices (usage-only chunk) → no panic, no-op. + let usage = Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + prompt_tokens_details: None, + completion_tokens_details: None, + time_to_first_token_ms: 0.0, + response_tokens_per_second: 0.0, + }; + let chunk = ChatCompletionChunk::usage_only_chunk("m", "id", usage).with_token_ids(vec![1, 2]); + assert!(chunk.choices.is_empty()); +} + +// ── reasoning wire format: exactly one field ──────────────────────── +// A response carrying BOTH `reasoning_content` and a `reasoning` mirror is +// rejected by strict OpenAI-compatible clients (they assert exactly one). +// Atlas emits only `reasoning_content` — these lock that contract in. + +#[test] +fn reasoning_delta_emits_only_reasoning_content() { + let chunk = ChatCompletionChunk::reasoning_chunk("m", "id", "thinking".into()); + let json = serde_json::to_string(&chunk).unwrap(); + assert!( + json.contains("\"reasoning_content\":\"thinking\""), + "reasoning_content missing: {json}" + ); + assert!( + !json.contains("\"reasoning\":"), + "mirror `reasoning` field leaked into stream delta: {json}" + ); +} + +#[test] +fn blocking_message_emits_only_reasoning_content() { + let msg = ChatMessage { + role: "assistant".into(), + reasoning_content: Some("thinking".into()), + content: Some("hi".into()), + tool_calls: None, + annotations: None, + refusal: None, + }; + let json = serde_json::to_string(&msg).unwrap(); + assert!( + json.contains("\"reasoning_content\":\"thinking\""), + "reasoning_content missing: {json}" + ); + assert!( + !json.contains("\"reasoning\":"), + "mirror `reasoning` field leaked into message: {json}" + ); +} diff --git a/crates/spark-server/src/openai/tests/completions.rs b/crates/spark-server/src/openai/tests/completions.rs new file mode 100644 index 000000000..4439b5965 --- /dev/null +++ b/crates/spark-server/src/openai/tests/completions.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Legacy /v1/completions echo + logprobs wire types. + +use crate::openai::*; + +#[test] +fn completion_request_echo_logprobs_n_deser() { + let req: CompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "test", + "prompt": "hello", + "echo": true, + "logprobs": 3, + "n": 2, + "max_tokens": 0, + "stream_options": {"include_usage": true}, + "user": "eval-harness", + "suffix": "tail", + "best_of": 2, + })) + .expect("valid completion request"); + assert!(req.echo); + assert_eq!(req.logprobs, Some(3)); + assert_eq!(req.n, 2); + assert_eq!(req.max_tokens, 0); + assert!(req.stream_options.expect("stream_options").include_usage); +} + +#[test] +fn completion_request_defaults_match_openai_spec() { + // echo=false, n=1, logprobs absent — the spec defaults; a request + // that names none of the new fields must behave exactly as before. + let req: CompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "test", + "prompt": "hello", + })) + .expect("valid completion request"); + assert!(!req.echo); + assert_eq!(req.n, 1); + assert!(req.logprobs.is_none()); + assert!(req.stream_options.is_none()); +} + +#[test] +fn completion_logprobs_serializes_four_parallel_arrays_with_nulls() { + let lp = CompletionLogprobs { + tokens: vec!["He".into(), "llo".into()], + token_logprobs: vec![None, Some(-0.5)], + top_logprobs: vec![ + None, + Some(std::collections::HashMap::from([( + "llo".to_string(), + -0.5f32, + )])), + ], + text_offset: vec![0, 2], + }; + let v = serde_json::to_value(&lp).expect("serialize"); + // Legacy shape: 4 parallel arrays, JSON null for the first echoed token. + assert!(v["token_logprobs"][0].is_null()); + assert!(v["top_logprobs"][0].is_null()); + assert_eq!(v["tokens"].as_array().map(Vec::len), Some(2)); + assert_eq!(v["text_offset"][1], 2); +} + +#[test] +fn completion_response_carries_system_fingerprint_and_optional_logprobs() { + let usage = Usage { + prompt_tokens: 1, + completion_tokens: 1, + total_tokens: 2, + prompt_tokens_details: None, + completion_tokens_details: None, + time_to_first_token_ms: 0.0, + response_tokens_per_second: 0.0, + }; + let resp = CompletionResponse::new("m", "hi".into(), usage, "stop"); + let v = serde_json::to_value(&resp).expect("serialize"); + assert_eq!(v["system_fingerprint"], "fp_atlas"); + // logprobs must be ABSENT (not null) when not requested — some + // clients treat an explicit null as a malformed logprobs block. + assert!(v["choices"][0].get("logprobs").is_none()); +} + +#[test] +fn completion_request_n_bounds_are_handler_enforced() { + // serde accepts any usize; the HANDLER rejects n==0 and n>128 with a + // 400 (OpenAI spec bound). This test locks the parse side: values + // arrive intact for the handler check (no silent serde clamping). + let req: CompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "m", "prompt": "p", "n": 4096, + })) + .expect("parse"); + assert_eq!(req.n, 4096); +} diff --git a/crates/spark-server/src/openai/tests/responses.rs b/crates/spark-server/src/openai/tests/responses.rs new file mode 100644 index 000000000..304910238 --- /dev/null +++ b/crates/spark-server/src/openai/tests/responses.rs @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Responses-API lowering tests: tool shapes, tool_choice, input items, +//! and stream event names. + +use crate::openai::*; + +fn lower_with_tools( + tools: serde_json::Value, +) -> Result { + let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ + "model": "test-model", + "input": "ping", + "tools": tools, + })) + .expect("ResponsesRequest deserialize"); + lower_responses_to_chat(req, |_| None) +} + +#[test] +fn responses_flat_function_tool_accepted() { + // OpenAI's official Python SDK sends function tools in the flat + // shape `{type, name, description, parameters}` — no nested + // `function` object. Atlas must accept both shapes. + let chat = lower_with_tools(serde_json::json!([ + { + "type": "function", + "name": "get_weather", + "description": "look up weather", + "parameters": {"type": "object", "properties": {"loc": {"type": "string"}}, "required": ["loc"]} + } + ])).expect("flat-form function tool should parse"); + let tools = chat.tools.expect("tools present"); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].tool_type, "function"); + assert_eq!(tools[0].function.name, "get_weather"); +} + +#[test] +fn responses_nested_function_tool_still_accepted() { + // Backwards-compat: chat-completions-style `{type, function:{...}}` + // must keep working since older clients send it. + let chat = lower_with_tools(serde_json::json!([ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object"} + } + } + ])) + .expect("nested-form function tool should parse"); + let tools = chat.tools.expect("tools present"); + assert_eq!(tools.len(), 1); + assert_eq!(tools[0].function.name, "get_weather"); +} + +#[test] +fn responses_flat_tool_choice_accepted() { + let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ + "model": "test", + "input": "ping", + "tool_choice": {"type": "function", "name": "get_weather"}, + })) + .unwrap(); + let chat = lower_responses_to_chat(req, |_| None).expect("flat tool_choice"); + match chat.tool_choice { + Some(crate::tool_parser::ToolChoice::Specific { function }) => { + assert_eq!(function.name, "get_weather"); + } + other => panic!("expected Specific tool_choice, got {other:?}"), + } +} + +#[test] +fn responses_string_tool_choice_accepted() { + let req: ResponsesRequest = serde_json::from_value(serde_json::json!({ + "model": "test", + "input": "ping", + "tool_choice": "required", + })) + .unwrap(); + let chat = lower_responses_to_chat(req, |_| None).expect("string tool_choice"); + match chat.tool_choice { + Some(crate::tool_parser::ToolChoice::Mode(s)) => { + assert_eq!(s, "required"); + } + other => panic!("expected Mode tool_choice, got {other:?}"), + } +} + +#[test] +fn responses_input_image_string_url_is_carried() { + // Regression: the Responses adapter used to drop images entirely + // (`images: Vec::new()`), silently losing multimodal input. + let item = serde_json::json!({ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "what is this?"}, + {"type": "input_image", "image_url": "data:image/png;base64,AAA"} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("message"); + assert_eq!(m.content.text, "what is this?"); + assert_eq!( + m.content.images, + vec!["data:image/png;base64,AAA".to_string()] + ); +} + +#[test] +fn responses_input_image_object_url_is_carried() { + // Some SDKs nest the url: `{image_url: {url: "..."}}`. + let item = serde_json::json!({ + "type": "message", + "role": "user", + "content": [ + {"type": "input_image", "image_url": {"url": "https://example.com/a.png"}} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("message"); + assert_eq!( + m.content.images, + vec!["https://example.com/a.png".to_string()] + ); + assert_eq!(m.content.text, ""); +} + +#[test] +fn responses_function_call_output_image_is_carried() { + // #165 parity for the Responses surface: a screenshot returned by a + // tool (`function_call_output` with structured output parts) must + // reach the vision encoder, exactly like Anthropic tool_result + // images and chat-completions role:"tool" array content. + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_1", + "output": [ + {"type": "output_text", "text": "screenshot follows"}, + {"type": "input_image", "image_url": "data:image/png;base64,BBB"} + ] + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.role, "tool"); + assert_eq!(m.tool_call_id.as_deref(), Some("call_1")); + assert_eq!(m.content.text, "screenshot follows"); + assert_eq!( + m.content.images, + vec!["data:image/png;base64,BBB".to_string()] + ); +} + +#[test] +fn responses_function_call_output_string_unchanged() { + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_2", + "output": "plain result" + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.content.text, "plain result"); + assert!(m.content.images.is_empty()); +} + +#[test] +fn responses_function_call_output_opaque_array_stringified() { + // Out-of-spec array (no recognizable parts) keeps the historical + // stringified-JSON behavior instead of silently emptying the result. + let opaque = serde_json::json!([{"weather": "sunny", "temp_c": 21}]); + let item = serde_json::json!({ + "type": "function_call_output", + "call_id": "call_3", + "output": opaque.clone() + }); + let m = IncomingMessage::from_responses_input_item(&item).expect("tool message"); + assert_eq!(m.content.text, opaque.to_string()); + assert!(m.content.images.is_empty()); +} + +#[test] +fn responses_in_progress_event_name() { + let ev = ResponsesStreamEvent::InProgress { + sequence_number: 1, + response: ResponsesStreamEnvelope { + id: "resp_test".into(), + object: "response", + created_at: 0, + model: "m".into(), + status: "in_progress", + metadata: None, + }, + }; + assert_eq!(responses_event_name(&ev), "response.in_progress"); +} diff --git a/crates/spark-server/src/openai/tests/thinking.rs b/crates/spark-server/src/openai/tests/thinking.rs new file mode 100644 index 000000000..9f8409939 --- /dev/null +++ b/crates/spark-server/src/openai/tests/thinking.rs @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +//! Thinking directive tests: client channels → `ir::ThinkingDirective`. + +use crate::ir::ThinkingDirective; +use crate::openai::*; + +fn chat_req(body: serde_json::Value) -> ChatCompletionRequest { + serde_json::from_value(body).expect("valid chat request") +} + +fn base_body() -> serde_json::Value { + serde_json::json!({ + "model": "test", + "messages": [{"role": "user", "content": "hi"}], + }) +} + +#[test] +fn silent_request_is_unspecified() { + let req = chat_req(base_body()); + assert_eq!( + req.client_thinking_directive(), + ThinkingDirective::Unspecified + ); + assert!(!req.client_thinking_directive().is_explicit()); +} + +#[test] +fn anthropic_thinking_channel() { + // type=disabled wins even with a budget present. + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "disabled", "budget_tokens": 100}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); + + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "enabled", "budget_tokens": 512}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(512) } + ); + + // Adaptive / budget-less thinking object → think as long as needed + // (budget defers to the per-model max_thinking_budget). + let mut b = base_body(); + b["thinking"] = serde_json::json!({"type": "adaptive"}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); +} + +#[test] +fn thinking_token_budget_channel() { + let mut b = base_body(); + b["thinking_token_budget"] = serde_json::json!(512); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(512) } + ); + + let mut b = base_body(); + b["thinking_token_budget"] = serde_json::json!(0); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); +} + +#[test] +fn reasoning_effort_channel() { + for (effort, expect) in [ + ("none", ThinkingDirective::Off), + ("minimal", ThinkingDirective::On { budget: Some(64) }), + ("low", ThinkingDirective::On { budget: Some(128) }), + ("medium", ThinkingDirective::On { budget: Some(256) }), + ("high", ThinkingDirective::On { budget: Some(512) }), + ("xhigh", ThinkingDirective::On { budget: Some(1024) }), + ("max", ThinkingDirective::On { budget: Some(1024) }), + // Unknown efforts fall back to the conservative default budget. + ("bogus", ThinkingDirective::On { budget: Some(256) }), + ] { + let mut b = base_body(); + b["reasoning"] = serde_json::json!({"effort": effort}); + assert_eq!( + chat_req(b).client_thinking_directive(), + expect, + "effort={effort}" + ); + } +} + +#[test] +fn chat_template_kwargs_channel() { + // Struct still parses as a request-body wire field. + let kw: ChatTemplateKwargs = + serde_json::from_str(r#"{"enable_thinking":true,"thinking_budget":1024}"#) + .expect("should parse"); + assert_eq!(kw.enable_thinking, Some(true)); + assert_eq!(kw.thinking_budget, Some(1024)); + + // Budget rung wins over the enable flag. + let mut b = base_body(); + b["chat_template_kwargs"] = + serde_json::json!({"enable_thinking": false, "thinking_budget": 1024}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: Some(1024) } + ); + + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"thinking_budget": 0}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); + + // enable_thinking with no explicit budget defers to the per-model + // max_thinking_budget (budget: None), not the conservative 256-token + // default — a hard cut force-injects mid-reasoning and + // wrecks agentic tool selection. + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": true}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); + + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({"enable_thinking": false}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Off + ); + + // Empty kwargs object carries no intent. + let mut b = base_body(); + b["chat_template_kwargs"] = serde_json::json!({}); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Unspecified + ); +} + +#[test] +fn legacy_enable_thinking_channel() { + let mut b = base_body(); + b["enable_thinking"] = serde_json::json!(true); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::On { budget: None } + ); + + // false is the serde default — indistinguishable from absent, so it + // must NOT count as an explicit opt-out. + let mut b = base_body(); + b["enable_thinking"] = serde_json::json!(false); + assert_eq!( + chat_req(b).client_thinking_directive(), + ThinkingDirective::Unspecified + ); +} From b5e359da46f86d8d3b4925f8bd8d62387b0c5ee0 Mon Sep 17 00:00:00 2001 From: Sercan Degirmenci Date: Wed, 15 Jul 2026 18:45:45 +0300 Subject: [PATCH 3/4] feat(server): resolve review comments and follow Rust conventions --- crates/spark-server/src/anthropic/handlers.rs | 24 ++--- .../src/anthropic/handlers_stream.rs | 43 ++++++-- crates/spark-server/src/anthropic/helpers.rs | 41 +++---- .../src/anthropic/tests/ir_carry.rs | 19 ++-- crates/spark-server/src/anthropic/to_ir.rs | 41 +++---- .../spark-server/src/anthropic/translate.rs | 102 +++++++++--------- .../spark-server/src/anthropic/translator.rs | 14 +-- crates/spark-server/src/anthropic/types.rs | 2 +- crates/spark-server/src/api/chat/echo.rs | 4 +- crates/spark-server/src/api/chat/mod.rs | 4 +- crates/spark-server/src/api/chat_blocking.rs | 2 +- .../src/api/chat_stream/handle_done.rs | 2 +- crates/spark-server/src/api/responses.rs | 39 +------ .../spark-server/src/api/responses_stream.rs | 2 +- crates/spark-server/src/ir/message.rs | 6 +- crates/spark-server/src/ir/response.rs | 6 +- .../spark-server/src/openai/chat_message.rs | 87 ++++++++++----- .../spark-server/src/openai/chat_request.rs | 6 -- crates/spark-server/src/openai/to_ir.rs | 62 +++++------ 19 files changed, 267 insertions(+), 239 deletions(-) diff --git a/crates/spark-server/src/anthropic/handlers.rs b/crates/spark-server/src/anthropic/handlers.rs index e6d5e2137..c8859ced9 100644 --- a/crates/spark-server/src/anthropic/handlers.rs +++ b/crates/spark-server/src/anthropic/handlers.rs @@ -11,7 +11,6 @@ use crate::AppState; use super::handlers_stream::*; use super::helpers::*; -use super::translate::*; use super::types::*; // ── Handler ── @@ -19,7 +18,7 @@ use super::types::*; /// POST /v1/messages — Anthropic Messages API. /// /// Lowers the request directly into the canonical chat IR -/// (`MessagesRequest::into_ir`), dispatches through +/// (`From for ir::ChatRequest`), dispatches through /// `api::chat_completions_inner` (which runs every fix12-23 /// sanitization, salvage, watchdog, and dump path), and translates the /// response back into Anthropic format. The Anthropic-specific surface @@ -77,26 +76,25 @@ pub async fn messages(State(state): State>, body: axum::body::Byte // sampling preset, and prompt mutation logic lives there. // Anthropic has no echo-only wire fields (service_tier / store / // stream_options), so the echo context stays default. - let outcome = - crate::api::chat_completions_inner(state.clone(), None, req.into_ir(), None).await; + let outcome = crate::api::chat_completions_inner(state.clone(), None, req.into(), None).await; // 5. Encode back to Anthropic shape. Non-streaming success arrives // as the response IR — no body re-parse; a malformed inner body // is now structurally impossible. let chat_resp = match outcome { crate::api::ChatOutcome::Blocking(ir) => { - let messages_resp = ir_to_anthropic_response(*ir); + let messages_resp = MessagesResponse::from(*ir); if let (Some(seq), Some(dump)) = (dump_seq, state.dump_writer.as_ref()) { dump.dump_response("/v1/messages", seq, &messages_resp, false); } return Json(messages_resp).into_response(); } crate::api::ChatOutcome::Streaming(deltas) => { - // Note: streaming dump from the Anthropic side is - // best-effort; Anthropic-shape response capture is a - // follow-up. - let _ = dump_seq; - return anthropic_sse_from_deltas(deltas, model_echo); + // --dump: the encoder aggregates the typed Anthropic events + // and writes them under the same seq as the request entry. + let dump = + dump_seq.and_then(|seq| state.dump_writer.as_ref().map(|d| (seq, d.clone()))); + return anthropic_sse_from_deltas(deltas, model_echo, dump); } crate::api::ChatOutcome::Http(r) => r, }; @@ -156,13 +154,13 @@ pub async fn count_tokens( }; // Count against the EXACT prompt the serving path renders: same - // adapter (into_ir), same tool-prompt injection / hint / cwd / - // thinking resolution / Jinja variant (prepare_chat_prompt). The + // adapter (the IR `From` impl), same tool-prompt injection / hint / + // cwd / thinking resolution / Jinja variant (prepare_chat_prompt). The // old path was a third, divergent lowering — it dropped images and // thinking blocks, force-prepended an empty `` wrapper, and // rendered through the non-openai Jinja variant, so counts drifted // from real usage. - let mut ir_req = req.into_ir(); + let mut ir_req = crate::ir::ChatRequest::from(req); // Counting must not require the vision encoder: strip image parts // (they contributed 0 tokens in the old count too — pad expansion // needs real pixel grids, which a count endpoint can't produce). diff --git a/crates/spark-server/src/anthropic/handlers_stream.rs b/crates/spark-server/src/anthropic/handlers_stream.rs index cc3a28771..93195a2f0 100644 --- a/crates/spark-server/src/anthropic/handlers_stream.rs +++ b/crates/spark-server/src/anthropic/handlers_stream.rs @@ -12,9 +12,14 @@ use super::translator::*; /// is processed. (This replaces the old wrapper that re-parsed the /// serialized OpenAI SSE bytes chunk-by-chunk — one fewer serialize/ /// parse round-trip and one fewer protocol to drift against.) +/// +/// `dump` (--dump seq + writer handle) captures the response in +/// Anthropic shape: the typed events are aggregated as they are sent +/// and written as one `stream: true` entry when the stream ends. pub(super) fn anthropic_sse_from_deltas( deltas: crate::ir::DeltaStream, req_model: String, + dump: Option<(u64, crate::request_dumper::DumpHandle)>, ) -> Response { // Match the chat_stream sizing — see chat_stream/mod.rs for rationale. let (tx, rx) = tokio::sync::mpsc::channel::>(1024); @@ -23,34 +28,52 @@ pub(super) fn anthropic_sse_from_deltas( let mut translator = AnthropicTranslator::new(req_model); let mut pending: Vec = Vec::new(); let mut deltas = deltas; + // --dump: aggregate the typed events (event name + JSON data) + // so the dump entry records the exact Anthropic wire framing. + let mut captured: Option> = dump.as_ref().map(|_| Vec::new()); // Drain `pending` into the channel, converting each typed - // `SseEvent` to an axum wire event. Returns `false` when the - // receiver has hung up — caller should abort. + // `SseEvent` to an axum wire event (and capturing it for the + // --dump entry when enabled). Returns `false` when the receiver + // has hung up — caller should abort. async fn flush( tx: &tokio::sync::mpsc::Sender>, pending: &mut Vec, + captured: &mut Option>, ) -> bool { for ev in pending.drain(..) { - if tx.send(Ok(to_axum_event(&ev))).await.is_err() { + if let Some(events) = captured { + events.push(serde_json::json!({"event": ev.event, "data": ev.data})); + } + if tx.send(Ok(ev.to_axum_event())).await.is_err() { return false; } } true } + let mut aborted = false; while let Some(delta) = deltas.next().await { translator.on_delta(&delta, &mut pending); - if !flush(&tx, &mut pending).await { - return; + if !flush(&tx, &mut pending, &mut captured).await { + aborted = true; + break; + } + } + + if !aborted { + // Ensure message_stop fires even if the stream ended without + // a Finish delta. + translator.finalize(&mut pending); + if !flush(&tx, &mut pending, &mut captured).await { + tracing::warn!("anthropic stream: final flush failed (receiver dropped)"); } } - // Ensure message_stop fires even if the stream ended without a - // Finish delta. - translator.finalize(&mut pending); - if !flush(&tx, &mut pending).await { - tracing::warn!("anthropic stream: final flush failed (receiver dropped)"); + // --dump: write the aggregated Anthropic event list under the + // request's seq (partial when the client hung up mid-stream). + if let (Some((seq, writer)), Some(events)) = (dump, captured) { + writer.dump_response("/v1/messages", seq, &events, true); } }); diff --git a/crates/spark-server/src/anthropic/helpers.rs b/crates/spark-server/src/anthropic/helpers.rs index 006dae90a..8fd218291 100644 --- a/crates/spark-server/src/anthropic/helpers.rs +++ b/crates/spark-server/src/anthropic/helpers.rs @@ -20,37 +20,38 @@ pub(super) fn anthropic_error(status: StatusCode, error_type: &str, message: Str // ── Conversion helpers ── -/// Convert Anthropic tools to OpenAI-compatible tool definitions. -pub(super) fn convert_tools(tools: &[AnthropicTool]) -> Vec { - tools - .iter() - .map(|t| tool_parser::ToolDefinition { +impl From<&AnthropicTool> for tool_parser::ToolDefinition { + /// Convert an Anthropic tool to an OpenAI-compatible tool definition. + fn from(t: &AnthropicTool) -> Self { + tool_parser::ToolDefinition { tool_type: "function".to_string(), function: tool_parser::FunctionDefinition { name: t.name.clone(), description: t.description.clone(), parameters: Some(t.input_schema.clone()), }, - }) - .collect() + } + } } -/// Convert Anthropic tool_choice to OpenAI-compatible tool_choice. -pub(super) fn convert_tool_choice(tc: &AnthropicToolChoice) -> tool_parser::ToolChoice { - match tc.choice_type.as_str() { - "any" => tool_parser::ToolChoice::Mode("required".to_string()), - "auto" => tool_parser::ToolChoice::Mode("auto".to_string()), - "none" => tool_parser::ToolChoice::Mode("none".to_string()), - "tool" => { - if let Some(ref name) = tc.name { - tool_parser::ToolChoice::Specific { - function: tool_parser::ToolChoiceFunction { name: name.clone() }, +impl From<&AnthropicToolChoice> for tool_parser::ToolChoice { + /// Convert an Anthropic tool_choice to an OpenAI-compatible tool_choice. + fn from(tc: &AnthropicToolChoice) -> Self { + match tc.choice_type.as_str() { + "any" => tool_parser::ToolChoice::Mode("required".to_string()), + "auto" => tool_parser::ToolChoice::Mode("auto".to_string()), + "none" => tool_parser::ToolChoice::Mode("none".to_string()), + "tool" => { + if let Some(ref name) = tc.name { + tool_parser::ToolChoice::Specific { + function: tool_parser::ToolChoiceFunction { name: name.clone() }, + } + } else { + tool_parser::ToolChoice::Mode("auto".to_string()) } - } else { - tool_parser::ToolChoice::Mode("auto".to_string()) } + _ => tool_parser::ToolChoice::Mode("auto".to_string()), } - _ => tool_parser::ToolChoice::Mode("auto".to_string()), } } diff --git a/crates/spark-server/src/anthropic/tests/ir_carry.rs b/crates/spark-server/src/anthropic/tests/ir_carry.rs index 98619acb8..6460d3f7d 100644 --- a/crates/spark-server/src/anthropic/tests/ir_carry.rs +++ b/crates/spark-server/src/anthropic/tests/ir_carry.rs @@ -5,8 +5,7 @@ // calls, and the tool_result error flag as typed structure (issue // #165 + IR migration). -use super::super::translate::ir_to_anthropic_response; -use super::super::types::MessagesRequest; +use super::super::types::{MessagesRequest, MessagesResponse}; use crate::ir::message::ImageSource; use crate::ir::{ChatRequest, ContentPart, ImageData, Role, ThinkingDirective}; @@ -14,7 +13,7 @@ use crate::ir::{ChatRequest, ContentPart, ImageData, Role, ThinkingDirective}; /// `handlers::messages` takes). fn lower(req_json: serde_json::Value) -> ChatRequest { let req: MessagesRequest = serde_json::from_value(req_json).expect("MessagesRequest"); - req.into_ir() + req.into() } fn text_of(parts: &[ContentPart]) -> String { @@ -305,7 +304,7 @@ fn wire_surfaces_lower_to_identical_ir_core() { ] })) .expect("chat wire parses"); - let chat_ir = chat.into_ir(); + let chat_ir = ChatRequest::from(chat); assert_eq!(anth.messages, chat_ir.messages); assert_eq!(anth.thinking, chat_ir.thinking); @@ -322,7 +321,7 @@ fn wire_surfaces_lower_to_identical_ir_core() { ); } -// ── response direction (ir_to_anthropic_response) ── +// ── response direction (IR → MessagesResponse) ── fn ir_response(choice: crate::ir::Choice) -> crate::ir::ChatResponse { crate::ir::ChatResponse { @@ -364,7 +363,7 @@ fn response_maps_reasoning_text_tools_and_usage() { arguments: serde_json::json!({"x": 1}), }]; c.finish_reason = crate::ir::FinishReason::ToolCalls; - let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + let json = serde_json::to_value(MessagesResponse::from(ir_response(c))).unwrap(); assert_eq!(json["id"], "msg_abc"); assert_eq!(json["model"], "m"); assert_eq!(json["stop_reason"], "tool_use"); @@ -386,19 +385,19 @@ fn response_stop_reason_mapping() { // content_filter → refusal (Anthropic's dedicated stop reason). let mut c = choice(); c.finish_reason = crate::ir::FinishReason::ContentFilter; - let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + let json = serde_json::to_value(MessagesResponse::from(ir_response(c))).unwrap(); assert_eq!(json["stop_reason"], "refusal"); // length → max_tokens. let mut c = choice(); c.finish_reason = crate::ir::FinishReason::Length; - let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + let json = serde_json::to_value(MessagesResponse::from(ir_response(c))).unwrap(); assert_eq!(json["stop_reason"], "max_tokens"); // A client stop sequence gets stop_sequence + the matched echo. let mut c = choice(); c.matched_stop = Some("END".into()); - let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + let json = serde_json::to_value(MessagesResponse::from(ir_response(c))).unwrap(); assert_eq!(json["stop_reason"], "stop_sequence"); assert_eq!(json["stop_sequence"], "END"); } @@ -407,7 +406,7 @@ fn response_stop_reason_mapping() { fn response_empty_content_omits_text_block() { let mut c = choice(); c.content = Some(String::new()); - let json = serde_json::to_value(ir_to_anthropic_response(ir_response(c))).unwrap(); + let json = serde_json::to_value(MessagesResponse::from(ir_response(c))).unwrap(); assert_eq!(json["content"].as_array().unwrap().len(), 0); assert_eq!(json["stop_reason"], "end_turn"); } diff --git a/crates/spark-server/src/anthropic/to_ir.rs b/crates/spark-server/src/anthropic/to_ir.rs index 9d63c513d..ce4ce9c89 100644 --- a/crates/spark-server/src/anthropic/to_ir.rs +++ b/crates/spark-server/src/anthropic/to_ir.rs @@ -12,12 +12,11 @@ use crate::ir; use crate::ir::message::{ImageSource, Reasoning, ToolCall}; use crate::ir::{ContentPart, ImageData, Message, Role, ThinkingDirective}; -use super::helpers::{convert_tool_choice, convert_tools}; use super::types::{ AnthropicContent, ContentBlock, MessagesRequest, SystemContent, ToolResultContent, }; -impl MessagesRequest { +impl From for ir::ChatRequest { /// Lower the parsed Anthropic wire request into the /// provider-agnostic [`ir::ChatRequest`] envelope. /// @@ -37,11 +36,11 @@ impl MessagesRequest { /// the text here. /// * unknown roles collapse to user (Anthropic wire only defines /// user/assistant). - pub fn into_ir(self) -> ir::ChatRequest { - let mut messages: Vec = Vec::with_capacity(self.messages.len() + 1); + fn from(req: MessagesRequest) -> Self { + let mut messages: Vec = Vec::with_capacity(req.messages.len() + 1); // System message (filter x-anthropic- billing/config blocks). - if let Some(sys) = &self.system { + if let Some(sys) = &req.system { let sys_text = match sys { SystemContent::Text(s) => s.clone(), SystemContent::Blocks(blocks) => blocks @@ -60,7 +59,7 @@ impl MessagesRequest { } // Conversation history. - for m in self.messages { + for m in req.messages { let role = match m.role.as_str() { "assistant" => Role::Assistant, _ => Role::User, @@ -93,7 +92,7 @@ impl MessagesRequest { match b { ContentBlock::Text { text } => text_parts.push(text), ContentBlock::Image { source } => { - if let Some(uri) = source.to_image_uri() { + if let Some(uri) = source.maybe_get_image_uri() { images.push(uri); } } @@ -120,7 +119,9 @@ impl MessagesRequest { Some(ToolResultContent::Blocks(inner)) => inner .iter() .filter_map(|ib| match ib { - ContentBlock::Image { source } => source.to_image_uri(), + ContentBlock::Image { source } => { + source.maybe_get_image_uri() + } _ => None, }) .collect(), @@ -192,7 +193,7 @@ impl MessagesRequest { // OpenAI ladder applies to its `thinking` channel: disabled wins, // then an explicit budget, then budget-less enabled/adaptive // (think as long as needed — defer to the per-model cap). - let thinking = match &self.thinking { + let thinking = match &req.thinking { None => ThinkingDirective::Unspecified, Some(tc) if tc.thinking_type == "disabled" => ThinkingDirective::Off, Some(tc) => match tc.budget_tokens { @@ -204,20 +205,24 @@ impl MessagesRequest { }; ir::ChatRequest { - model: self.model, + model: req.model, messages, - tools: self.tools.as_deref().map(convert_tools).unwrap_or_default(), - tool_choice: self.tool_choice.as_ref().map(convert_tool_choice), + tools: req + .tools + .as_deref() + .map(|ts| ts.iter().map(Into::into).collect()) + .unwrap_or_default(), + tool_choice: req.tool_choice.as_ref().map(Into::into), sampling: ir::SamplingParams { - temperature: self.temperature, - top_k: self.top_k, - top_p: self.top_p, + temperature: req.temperature, + top_k: req.top_k, + top_p: req.top_p, ..Default::default() }, - max_tokens: self.max_tokens, + max_tokens: req.max_tokens, min_tokens: 0, - stop: self.stop_sequences, - stream: self.stream, + stop: req.stop_sequences, + stream: req.stream, n: 1, response_format: None, thinking, diff --git a/crates/spark-server/src/anthropic/translate.rs b/crates/spark-server/src/anthropic/translate.rs index 7d1e5617e..2c810efdc 100644 --- a/crates/spark-server/src/anthropic/translate.rs +++ b/crates/spark-server/src/anthropic/translate.rs @@ -2,58 +2,60 @@ use super::types::*; -/// Serialize the canonical [`crate::ir::ChatResponse`] into Anthropic's -/// `MessagesResponse` wire shape (non-streaming). Reads the first -/// choice — the Anthropic adapter pins `n = 1`. -pub(super) fn ir_to_anthropic_response(ir: crate::ir::ChatResponse) -> MessagesResponse { - let id = format!("msg_{}", ir.id); - let choice = ir.choices.into_iter().next(); +impl From for MessagesResponse { + /// Serialize the canonical [`crate::ir::ChatResponse`] into + /// Anthropic's `MessagesResponse` wire shape (non-streaming). Reads + /// the first choice — the Anthropic adapter pins `n = 1`. + fn from(ir: crate::ir::ChatResponse) -> Self { + let id = format!("msg_{}", ir.id); + let choice = ir.choices.into_iter().next(); - let mut content: Vec = Vec::new(); - let mut stop_reason = "end_turn"; - let mut stop_sequence: Option = None; - if let Some(c) = choice { - if let Some(r) = c.reasoning { - content.push(ResponseBlock::Thinking { thinking: r }); + let mut content: Vec = Vec::new(); + let mut stop_reason = "end_turn"; + let mut stop_sequence: Option = None; + if let Some(c) = choice { + if let Some(r) = c.reasoning { + content.push(ResponseBlock::Thinking { thinking: r }); + } + if let Some(text) = c.content + && !text.is_empty() + { + content.push(ResponseBlock::Text { text }); + } + for tc in c.tool_calls { + content.push(ResponseBlock::ToolUse { + id: tc.id, + name: tc.name, + input: tc.arguments, + }); + } + stop_reason = match c.finish_reason { + // A client stop sequence gets its dedicated reason + echo. + crate::ir::FinishReason::Stop if c.matched_stop.is_some() => "stop_sequence", + crate::ir::FinishReason::Stop => "end_turn", + crate::ir::FinishReason::ToolCalls => "tool_use", + crate::ir::FinishReason::Length => "max_tokens", + // Safety-filtered output maps to Anthropic's dedicated + // refusal stop reason (2025-05 API), not a normal end_turn. + crate::ir::FinishReason::ContentFilter => "refusal", + crate::ir::FinishReason::Other(_) => "end_turn", + }; + stop_sequence = c.matched_stop; } - if let Some(text) = c.content - && !text.is_empty() - { - content.push(ResponseBlock::Text { text }); - } - for tc in c.tool_calls { - content.push(ResponseBlock::ToolUse { - id: tc.id, - name: tc.name, - input: tc.arguments, - }); - } - stop_reason = match c.finish_reason { - // A client stop sequence gets its dedicated reason + echo. - crate::ir::FinishReason::Stop if c.matched_stop.is_some() => "stop_sequence", - crate::ir::FinishReason::Stop => "end_turn", - crate::ir::FinishReason::ToolCalls => "tool_use", - crate::ir::FinishReason::Length => "max_tokens", - // Safety-filtered output maps to Anthropic's dedicated - // refusal stop reason (2025-05 API), not a normal end_turn. - crate::ir::FinishReason::ContentFilter => "refusal", - crate::ir::FinishReason::Other(_) => "end_turn", - }; - stop_sequence = c.matched_stop; - } - MessagesResponse { - id, - response_type: "message".to_string(), - role: "assistant".to_string(), - content, - model: ir.model, - stop_reason: Some(stop_reason.to_string()), - stop_sequence, - usage: AnthropicUsage { - input_tokens: ir.usage.prompt_tokens, - output_tokens: ir.usage.completion_tokens, - cache_read_input_tokens: ir.usage.cached_prompt_tokens, - }, + MessagesResponse { + id, + response_type: "message".to_string(), + role: "assistant".to_string(), + content, + model: ir.model, + stop_reason: Some(stop_reason.to_string()), + stop_sequence, + usage: AnthropicUsage { + input_tokens: ir.usage.prompt_tokens, + output_tokens: ir.usage.completion_tokens, + cache_read_input_tokens: ir.usage.cached_prompt_tokens, + }, + } } } diff --git a/crates/spark-server/src/anthropic/translator.rs b/crates/spark-server/src/anthropic/translator.rs index b6d323785..780b738ce 100644 --- a/crates/spark-server/src/anthropic/translator.rs +++ b/crates/spark-server/src/anthropic/translator.rs @@ -6,18 +6,20 @@ use super::helpers::*; /// A typed Anthropic SSE event (event name + JSON data). Testable, unlike /// axum's opaque `Event`; converted to an axum event at the handler edge -/// by [`to_axum_event`]. +/// by [`Self::to_axum_event`]. #[derive(Debug, Clone, PartialEq)] pub(super) struct SseEvent { pub(super) event: String, pub(super) data: serde_json::Value, } -/// Convert a typed [`SseEvent`] to an axum SSE event for the wire. -pub(super) fn to_axum_event(e: &SseEvent) -> Event { - Event::default() - .event(&e.event) - .data(serde_json::to_string(&e.data).unwrap_or_default()) +impl SseEvent { + /// Convert a reference to self to an axum SSE event for the wire. + pub(super) fn to_axum_event(&self) -> Event { + Event::default() + .event(&self.event) + .data(serde_json::to_string(&self.data).unwrap_or_default()) + } } /// Open-block tracker for the streaming translator's state machine. diff --git a/crates/spark-server/src/anthropic/types.rs b/crates/spark-server/src/anthropic/types.rs index a4a514627..cfb001bd0 100644 --- a/crates/spark-server/src/anthropic/types.rs +++ b/crates/spark-server/src/anthropic/types.rs @@ -138,7 +138,7 @@ impl ImageSourceBlock { /// Build the string the vision encoder consumes: a `data:` URI for /// base64 sources, or the raw URL for url sources. `None` when the /// required fields are missing. - pub(super) fn to_image_uri(&self) -> Option { + pub(super) fn maybe_get_image_uri(&self) -> Option { match self.source_type.as_str() { "base64" => { let data = self.data.as_ref()?; diff --git a/crates/spark-server/src/api/chat/echo.rs b/crates/spark-server/src/api/chat/echo.rs index 2dbf72a98..62a7547a9 100644 --- a/crates/spark-server/src/api/chat/echo.rs +++ b/crates/spark-server/src/api/chat/echo.rs @@ -20,9 +20,9 @@ pub(crate) struct ResponseEcho { pub(crate) include_usage: bool, } -impl ResponseEcho { +impl<'a> From<&'a crate::openai::ChatCompletionRequest> for ResponseEcho { /// Capture the echo fields from a parsed OpenAI wire request. - pub(crate) fn from_wire(req: &crate::openai::ChatCompletionRequest) -> Self { + fn from(req: &'a crate::openai::ChatCompletionRequest) -> Self { ResponseEcho { service_tier: req.service_tier.clone(), metadata: req.metadata.clone(), diff --git a/crates/spark-server/src/api/chat/mod.rs b/crates/spark-server/src/api/chat/mod.rs index 50740527e..e4f86f773 100644 --- a/crates/spark-server/src/api/chat/mod.rs +++ b/crates/spark-server/src/api/chat/mod.rs @@ -104,8 +104,8 @@ pub async fn chat_completions( // Wire → IR at the edge: echo-only fields peel off beside the // envelope; everything downstream reads only the IR. - let echo = ResponseEcho::from_wire(&req); - match chat_completions_inner(state.clone(), req_ctx, req.into_ir(), dump_seq).await { + let echo = ResponseEcho::from(&req); + match chat_completions_inner(state.clone(), req_ctx, req.into(), dump_seq).await { ChatOutcome::Blocking(ir) => { crate::openai::encode_chat_response(&state, *ir, &echo, dump_seq) } diff --git a/crates/spark-server/src/api/chat_blocking.rs b/crates/spark-server/src/api/chat_blocking.rs index 24231aa68..fe79bb55a 100644 --- a/crates/spark-server/src/api/chat_blocking.rs +++ b/crates/spark-server/src/api/chat_blocking.rs @@ -413,7 +413,7 @@ async fn build_choice_message( reasoning: reasoning_content, tool_calls, refusal: msg_refusal, - finish_reason: ir::FinishReason::from_wire(&finish_reason_i), + finish_reason: ir::FinishReason::from(finish_reason_i.as_str()), matched_stop: None, // caller fills logprobs: None, // caller fills } diff --git a/crates/spark-server/src/api/chat_stream/handle_done.rs b/crates/spark-server/src/api/chat_stream/handle_done.rs index d3312b4bc..17a241060 100644 --- a/crates/spark-server/src/api/chat_stream/handle_done.rs +++ b/crates/spark-server/src/api/chat_stream/handle_done.rs @@ -223,7 +223,7 @@ pub(super) fn handle_done( // and never rode a content delta — ride the Finish delta so // Σ token_ids == completion_tokens exactly. deltas.push(StreamDelta::Finish { - reason: crate::ir::FinishReason::from_wire(fr), + reason: crate::ir::FinishReason::from(fr), usage, token_ids: state.take_ids_if(ctx.req_return_token_ids), }); diff --git a/crates/spark-server/src/api/responses.rs b/crates/spark-server/src/api/responses.rs index c191d895c..588009f37 100644 --- a/crates/spark-server/src/api/responses.rs +++ b/crates/spark-server/src/api/responses.rs @@ -85,7 +85,7 @@ pub async fn responses_endpoint( Some(snap) => snap .items .iter() - .filter_map(conversation_item_to_message) + .filter_map(crate::openai::IncomingMessage::from_conversation_item) .collect(), None => { return openai_error_response_with_param( @@ -144,7 +144,7 @@ pub async fn responses_endpoint( // request. Use the _inner variant because we already have a parsed // struct (no raw bytes available to dump at this layer; the Responses // handler dumps at its own entry point if --dump is enabled). - let resp = chat_completions_inner(state.0.clone(), None, chat_req.into_ir(), None).await; + let resp = chat_completions_inner(state.0.clone(), None, chat_req.into(), None).await; let conv_pair = conversation_id.map(|cid| (state.conversation_store.clone(), cid)); translate_chat_response_to_responses( resp, @@ -156,38 +156,3 @@ pub async fn responses_endpoint( ) .await } - -/// Convert a conversation item into an IncomingMessage for pipeline -/// replay. Items we don't recognize (tool outputs in exotic shapes) -/// are silently dropped — they wouldn't contribute to the text -/// context anyway. -pub(super) fn conversation_item_to_message( - item: &serde_json::Value, -) -> Option { - let role = item.get("role").and_then(|v| v.as_str())?; - let content = item.get("content"); - let text = match content { - Some(serde_json::Value::String(s)) => s.clone(), - Some(serde_json::Value::Array(parts)) => parts - .iter() - .filter_map(|p| { - p.get("text") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()) - }) - .collect::>() - .join(""), - _ => String::new(), - }; - Some(crate::openai::IncomingMessage { - role: role.to_string(), - content: crate::openai::ParsedContent { - text, - images: Vec::new(), - }, - tool_calls: None, - tool_call_id: None, - name: None, - reasoning_content: None, - }) -} diff --git a/crates/spark-server/src/api/responses_stream.rs b/crates/spark-server/src/api/responses_stream.rs index 154f6e6f1..90ba0970a 100644 --- a/crates/spark-server/src/api/responses_stream.rs +++ b/crates/spark-server/src/api/responses_stream.rs @@ -87,7 +87,7 @@ pub(super) async fn responses_endpoint_stream( // Run the chat-completions streaming handler (re-using the full // pipeline: scheduler, tool detection, thinking, logprobs, ...). - let deltas = match chat_completions_inner(state.0, None, chat_req.into_ir(), None).await { + let deltas = match chat_completions_inner(state.0, None, chat_req.into(), None).await { super::chat::ChatOutcome::Streaming(d) => d, // Error envelope — forward unchanged. super::chat::ChatOutcome::Http(r) => return r, diff --git a/crates/spark-server/src/ir/message.rs b/crates/spark-server/src/ir/message.rs index 96a249b28..4086b0c30 100644 --- a/crates/spark-server/src/ir/message.rs +++ b/crates/spark-server/src/ir/message.rs @@ -35,7 +35,7 @@ impl Role { /// Parse a known wire role. Returns `None` for anything unknown — /// callers decide the fallback explicitly (PCND: no silent default). - /// Use [`Role::from_wire_lossless`] when an unknown role must be + /// Use the [`From<&str>`] impl when an unknown role must be /// preserved rather than rejected. pub fn from_wire(s: &str) -> Option { match s { @@ -46,11 +46,13 @@ impl Role { _ => None, } } +} +impl From<&str> for Role { /// Map any wire role string to a `Role`, preserving unknown roles as /// [`Role::Other`] (lossless — the template still decides what to do /// with them). - pub fn from_wire_lossless(s: &str) -> Self { + fn from(s: &str) -> Self { Self::from_wire(s).unwrap_or_else(|| Role::Other(s.to_string())) } } diff --git a/crates/spark-server/src/ir/response.rs b/crates/spark-server/src/ir/response.rs index e31093caa..abf3ab0ae 100644 --- a/crates/spark-server/src/ir/response.rs +++ b/crates/spark-server/src/ir/response.rs @@ -90,10 +90,10 @@ pub enum FinishReason { Other(String), } -impl FinishReason { +impl From<&str> for FinishReason { /// Map the engine's finish-reason string (the scheduler's internal /// vocabulary, which happens to match OpenAI's wire strings). - pub fn from_wire(s: &str) -> Self { + fn from(s: &str) -> Self { match s { "stop" => FinishReason::Stop, "length" => FinishReason::Length, @@ -102,7 +102,9 @@ impl FinishReason { other => FinishReason::Other(other.to_string()), } } +} +impl FinishReason { /// The canonical wire string (OpenAI-compatible surfaces emit it /// verbatim; other surfaces map per their own vocabulary). pub fn as_wire(&self) -> &str { diff --git a/crates/spark-server/src/openai/chat_message.rs b/crates/spark-server/src/openai/chat_message.rs index b489c7c55..f86bba5d6 100644 --- a/crates/spark-server/src/openai/chat_message.rs +++ b/crates/spark-server/src/openai/chat_message.rs @@ -69,6 +69,39 @@ impl IncomingMessage { } } + /// Convert a stored conversation item into a message for pipeline + /// replay. Items we don't recognize (tool outputs in exotic shapes) + /// are silently dropped — they wouldn't contribute to the text + /// context anyway. + pub fn from_conversation_item(item: &serde_json::Value) -> Option { + let role = item.get("role").and_then(|v| v.as_str())?; + let content = item.get("content"); + let text = match content { + Some(serde_json::Value::String(s)) => s.clone(), + Some(serde_json::Value::Array(parts)) => parts + .iter() + .filter_map(|p| { + p.get("text") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }) + .collect::>() + .join(""), + _ => String::new(), + }; + Some(Self { + role: role.to_string(), + content: ParsedContent { + text, + images: Vec::new(), + }, + tool_calls: None, + tool_call_id: None, + name: None, + reasoning_content: None, + }) + } + /// Translate a Responses-API `input` array item into a chat-completions /// message. Returns `None` for items the adapter doesn't understand (they /// are silently skipped so the request still runs). @@ -93,7 +126,7 @@ impl IncomingMessage { let content_val = obj.get("content")?; Some(Self { role, - content: responses_content_to_parsed(content_val), + content: ParsedContent::from_responses_content(content_val), tool_calls: None, tool_call_id: None, name: None, @@ -153,7 +186,7 @@ impl IncomingMessage { // no recognizable parts keep the old stringified-JSON // behavior so out-of-spec payloads still reach the model. Some(arr @ serde_json::Value::Array(_)) => { - let parsed = responses_content_to_parsed(arr); + let parsed = ParsedContent::from_responses_content(arr); if parsed.text.is_empty() && parsed.images.is_empty() { ParsedContent { text: arr.to_string(), @@ -194,35 +227,37 @@ impl IncomingMessage { } } -/// Flatten a Responses-API content value (string, or array of -/// `input_text`/`output_text`/`input_image`/… parts) into text + image -/// lists. Shared by `message` items and `function_call_output` items so -/// images are carried on both — the pipeline collects them into the -/// vision encoder. -fn responses_content_to_parsed(v: &serde_json::Value) -> ParsedContent { - let mut text = String::new(); - let mut images: Vec = Vec::new(); - match v { - serde_json::Value::String(s) => text.push_str(s), - serde_json::Value::Array(parts) => { - for part in parts { - if let Some(po) = part.as_object() { - let part_kind = po.get("type").and_then(|t| t.as_str()).unwrap_or(""); - if matches!(part_kind, "input_text" | "output_text" | "text") - && let Some(t) = po.get("text").and_then(|t| t.as_str()) - { - text.push_str(t); - } else if matches!(part_kind, "input_image" | "image_url" | "image") - && let Some(url) = responses_image_url(po) - { - images.push(url); +impl ParsedContent { + /// Flatten a Responses-API content value (string, or array of + /// `input_text`/`output_text`/`input_image`/… parts) into text + + /// image lists. Shared by `message` items and `function_call_output` + /// items so images are carried on both — the pipeline collects them + /// into the vision encoder. + fn from_responses_content(v: &serde_json::Value) -> Self { + let mut text = String::new(); + let mut images: Vec = Vec::new(); + match v { + serde_json::Value::String(s) => text.push_str(s), + serde_json::Value::Array(parts) => { + for part in parts { + if let Some(po) = part.as_object() { + let part_kind = po.get("type").and_then(|t| t.as_str()).unwrap_or(""); + if matches!(part_kind, "input_text" | "output_text" | "text") + && let Some(t) = po.get("text").and_then(|t| t.as_str()) + { + text.push_str(t); + } else if matches!(part_kind, "input_image" | "image_url" | "image") + && let Some(url) = responses_image_url(po) + { + images.push(url); + } } } } + _ => {} } - _ => {} + ParsedContent { text, images } } - ParsedContent { text, images } } /// Extract the image URL / data-URI from a Responses `input_image` diff --git a/crates/spark-server/src/openai/chat_request.rs b/crates/spark-server/src/openai/chat_request.rs index 2092f217b..288e6540a 100644 --- a/crates/spark-server/src/openai/chat_request.rs +++ b/crates/spark-server/src/openai/chat_request.rs @@ -277,12 +277,6 @@ pub struct ChatTemplateKwargs { const DEFAULT_THINKING_BUDGET: u32 = 256; impl ChatCompletionRequest { - /// Per-request override for the vLLM-anchored token-loop detector. - /// `None` = use the boot-global watchdog parameters. - pub fn repetition_detection(&self) -> Option { - self.repetition_detection - } - /// Resolve the client's thinking intent from all supported /// request-body formats into the neutral [`ThinkingDirective`]. /// This is the OpenAI-edge half of thinking resolution; the model diff --git a/crates/spark-server/src/openai/to_ir.rs b/crates/spark-server/src/openai/to_ir.rs index b7ff19e17..c1aada594 100644 --- a/crates/spark-server/src/openai/to_ir.rs +++ b/crates/spark-server/src/openai/to_ir.rs @@ -50,7 +50,7 @@ impl From<&IncomingMessage> for Message { .unwrap_or_default(); Message { - role: Role::from_wire_lossless(&m.role), + role: Role::from(m.role.as_str()), content, tool_calls, tool_call_id: m.tool_call_id.clone(), @@ -61,7 +61,7 @@ impl From<&IncomingMessage> for Message { } } -impl ChatCompletionRequest { +impl From for ir::ChatRequest { /// Lower the parsed OpenAI wire request into the provider-agnostic /// [`ir::ChatRequest`] envelope. Infallible: range validation /// happens on the envelope (`chat_phases::validate_input`), and the @@ -71,17 +71,17 @@ impl ChatCompletionRequest { /// Echo-only fields (service_tier, store, metadata, /// stream_options) are NOT lowered — the handler keeps the wire /// request for encode-time echoes. - pub fn into_ir(self) -> ir::ChatRequest { - let thinking = self.client_thinking_directive(); - let top_logprobs = resolve_top_logprobs(self.logprobs, self.top_logprobs); + fn from(req: ChatCompletionRequest) -> Self { + let thinking = req.client_thinking_directive(); + let top_logprobs = resolve_top_logprobs(req.logprobs, req.top_logprobs); // Logit bias: OpenAI's string-keyed map → typed pairs. Keys that // don't parse as token ids are dropped (historical behavior). - let logit_bias: Vec<(u32, f32)> = self.logit_bias.as_ref().map_or(Vec::new(), |map| { + let logit_bias: Vec<(u32, f32)> = req.logit_bias.as_ref().map_or(Vec::new(), |map| { map.iter() .filter_map(|(k, &v)| k.parse::().ok().map(|id| (id, v))) .collect() }); - let response_format = match self.response_format { + let response_format = match req.response_format { // "text" is the wire spelling of "no constraint". None | Some(ResponseFormat::Text) => None, Some(ResponseFormat::JsonObject) => Some(ir::ResponseFormat::JsonObject), @@ -94,33 +94,33 @@ impl ChatCompletionRequest { } }; ir::ChatRequest { - model: self.model, - messages: self.messages.iter().map(Into::into).collect(), - tools: self.tools.unwrap_or_default(), - tool_choice: self.tool_choice, + model: req.model, + messages: req.messages.iter().map(Into::into).collect(), + tools: req.tools.unwrap_or_default(), + tool_choice: req.tool_choice, sampling: ir::SamplingParams { - temperature: self.temperature, - top_k: self.top_k, - top_p: self.top_p, - top_n_sigma: self.top_n_sigma, - min_p: self.min_p, - repetition_penalty: self.repetition_penalty, - presence_penalty: self.presence_penalty, - frequency_penalty: self.frequency_penalty, + temperature: req.temperature, + top_k: req.top_k, + top_p: req.top_p, + top_n_sigma: req.top_n_sigma, + min_p: req.min_p, + repetition_penalty: req.repetition_penalty, + presence_penalty: req.presence_penalty, + frequency_penalty: req.frequency_penalty, }, - max_tokens: self.max_tokens, - min_tokens: self.min_tokens, - stop: self.stop, - stream: self.stream, - n: self.n, + max_tokens: req.max_tokens, + min_tokens: req.min_tokens, + stop: req.stop, + stream: req.stream, + n: req.n, response_format, thinking, - repetition_detection: self.repetition_detection, + repetition_detection: req.repetition_detection, logit_bias, top_logprobs, - seed: self.seed, - timeout_secs: self.timeout, - return_token_ids: self.return_token_ids, + seed: req.seed, + timeout_secs: req.timeout, + return_token_ids: req.return_token_ids, } } } @@ -313,7 +313,7 @@ mod tests { "seed": 7, "n": 2 })); - let ir = req.into_ir(); + let ir = ir::ChatRequest::from(req); assert_eq!(ir.model, "m"); assert_eq!(ir.messages.len(), 1); assert_eq!(ir.max_tokens, 64); @@ -336,14 +336,14 @@ mod tests { "messages": [{"role": "user", "content": "hi"}], "response_format": {"type": "text"} })); - assert!(req.into_ir().response_format.is_none()); + assert!(ir::ChatRequest::from(req).response_format.is_none()); let req = wire(serde_json::json!({ "model": "m", "messages": [{"role": "user", "content": "hi"}], "response_format": {"type": "json_schema", "json_schema": {"name": "s", "schema": {"type": "object"}}} })); - match req.into_ir().response_format { + match ir::ChatRequest::from(req).response_format { Some(ir::ResponseFormat::JsonSchema { name, schema, From 6102bea260e8878c54f2ac9146c2a2ee15c0b5c6 Mon Sep 17 00:00:00 2001 From: Sercan Degirmenci Date: Thu, 16 Jul 2026 15:53:21 +0300 Subject: [PATCH 4/4] feat(server): fix failing 500 loc check --- crates/spark-server/src/api/chat/msg_entry.rs | 105 ------------------ .../src/api/chat/msg_entry_tests.rs | 105 ++++++++++++++++++ 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/crates/spark-server/src/api/chat/msg_entry.rs b/crates/spark-server/src/api/chat/msg_entry.rs index cbded18a5..1dd5e4596 100644 --- a/crates/spark-server/src/api/chat/msg_entry.rs +++ b/crates/spark-server/src/api/chat/msg_entry.rs @@ -450,108 +450,3 @@ fn duplicate_error_masks(tool_results: &[(usize, String)]) -> Vec<(usize, String #[cfg(test)] #[path = "msg_entry_tests.rs"] mod msg_entry_tests; - -#[cfg(test)] -mod build_tests { - use super::build_msg_entries; - use crate::ir::message::{ContentPart, ImageData, ImageSource, Message, Role}; - use axum::http::StatusCode; - - fn assert_bad_request(msgs: &[Message], tools_active: bool) { - match build_msg_entries(None, None, msgs, tools_active) { - Ok(_) => panic!("expected 400, got Ok"), - Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), - } - } - - fn text(role: Role, t: &str) -> Message { - Message { - role, - content: vec![ContentPart::Text(t.into())], - tool_calls: Vec::new(), - tool_call_id: None, - name: None, - reasoning: None, - tool_error: false, - } - } - - fn image(role: Role) -> Message { - Message { - role, - content: vec![ - ContentPart::Image(ImageSource { - data: ImageData::Base64("data:image/png;base64,AAA".into()), - }), - ContentPart::Text("result".into()), - ], - tool_calls: Vec::new(), - tool_call_id: Some("c1".into()), - name: None, - reasoning: None, - tool_error: false, - } - } - - #[test] - fn text_only_builds_without_vision_config() { - let msgs = vec![text(Role::User, "hello")]; - let out = build_msg_entries(None, None, &msgs, false).expect("text-only ok"); - assert_eq!(out.messages.len(), 1); - assert_eq!(out.messages[0].image_count, 0); - } - - #[test] - fn user_image_without_vision_config_is_rejected() { - // Previously: silently dropped (200). Now: fail fast. - assert_bad_request(&[text(Role::User, "hi"), image(Role::User)], false); - } - - #[test] - fn tool_result_image_is_collected_and_rejected_without_vision() { - // Proves the tool branch now COUNTS + COLLECTS images (it used to - // hardcode image_count: 0 and `continue` before collection): the - // fail-fast only fires when an image was actually collected. - assert_bad_request(&[text(Role::User, "look"), image(Role::Tool)], true); - } - - #[test] - fn developer_role_normalized_to_system_at_build() { - // Was previously mapped only at JSON render time, so `developer` - // messages bypassed the cwd-hint / vacuous-system / CWD-injection - // scans that string-compare on "system". - let msgs = vec![text(Role::Other("developer".into()), "be terse")]; - let out = build_msg_entries(None, None, &msgs, false).expect("ok"); - assert_eq!(out.messages[0].role, "system"); - - // Other unknown roles still pass through verbatim for the - // template to handle. - let msgs = vec![text(Role::Other("critic".into()), "hm")]; - let out = build_msg_entries(None, None, &msgs, false).expect("ok"); - assert_eq!(out.messages[0].role, "critic"); - } - - #[test] - fn remote_url_image_rejected_with_clear_reason() { - // A https URL used to be mislabeled as base64 and die later in - // the vision preprocessor with "base64 decode failed". It must - // now be rejected up front with the real reason — even when the - // model HAS no vision config (the URL check fires first, at - // collection time). - let url_msg = Message { - role: Role::User, - content: vec![ContentPart::Image(ImageSource { - data: ImageData::Url("https://example.com/cat.png".into()), - })], - tool_calls: Vec::new(), - tool_call_id: None, - name: None, - reasoning: None, - tool_error: false, - }; - match build_msg_entries(None, None, &[url_msg], false) { - Ok(_) => panic!("expected 400, got Ok"), - Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), - } - } -} diff --git a/crates/spark-server/src/api/chat/msg_entry_tests.rs b/crates/spark-server/src/api/chat/msg_entry_tests.rs index 38439b2a4..ac563158c 100644 --- a/crates/spark-server/src/api/chat/msg_entry_tests.rs +++ b/crates/spark-server/src/api/chat/msg_entry_tests.rs @@ -108,3 +108,108 @@ mod vacuous_system_tests { )); } } + +#[cfg(test)] +mod build_tests { + use super::super::build_msg_entries; + use crate::ir::message::{ContentPart, ImageData, ImageSource, Message, Role}; + use axum::http::StatusCode; + + fn assert_bad_request(msgs: &[Message], tools_active: bool) { + match build_msg_entries(None, None, msgs, tools_active) { + Ok(_) => panic!("expected 400, got Ok"), + Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), + } + } + + fn text(role: Role, t: &str) -> Message { + Message { + role, + content: vec![ContentPart::Text(t.into())], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + } + } + + fn image(role: Role) -> Message { + Message { + role, + content: vec![ + ContentPart::Image(ImageSource { + data: ImageData::Base64("data:image/png;base64,AAA".into()), + }), + ContentPart::Text("result".into()), + ], + tool_calls: Vec::new(), + tool_call_id: Some("c1".into()), + name: None, + reasoning: None, + tool_error: false, + } + } + + #[test] + fn text_only_builds_without_vision_config() { + let msgs = vec![text(Role::User, "hello")]; + let out = build_msg_entries(None, None, &msgs, false).expect("text-only ok"); + assert_eq!(out.messages.len(), 1); + assert_eq!(out.messages[0].image_count, 0); + } + + #[test] + fn user_image_without_vision_config_is_rejected() { + // Previously: silently dropped (200). Now: fail fast. + assert_bad_request(&[text(Role::User, "hi"), image(Role::User)], false); + } + + #[test] + fn tool_result_image_is_collected_and_rejected_without_vision() { + // Proves the tool branch now COUNTS + COLLECTS images (it used to + // hardcode image_count: 0 and `continue` before collection): the + // fail-fast only fires when an image was actually collected. + assert_bad_request(&[text(Role::User, "look"), image(Role::Tool)], true); + } + + #[test] + fn developer_role_normalized_to_system_at_build() { + // Was previously mapped only at JSON render time, so `developer` + // messages bypassed the cwd-hint / vacuous-system / CWD-injection + // scans that string-compare on "system". + let msgs = vec![text(Role::Other("developer".into()), "be terse")]; + let out = build_msg_entries(None, None, &msgs, false).expect("ok"); + assert_eq!(out.messages[0].role, "system"); + + // Other unknown roles still pass through verbatim for the + // template to handle. + let msgs = vec![text(Role::Other("critic".into()), "hm")]; + let out = build_msg_entries(None, None, &msgs, false).expect("ok"); + assert_eq!(out.messages[0].role, "critic"); + } + + #[test] + fn remote_url_image_rejected_with_clear_reason() { + // A https URL used to be mislabeled as base64 and die later in + // the vision preprocessor with "base64 decode failed". It must + // now be rejected up front with the real reason — even when the + // model HAS no vision config (the URL check fires first, at + // collection time). + let url_msg = Message { + role: Role::User, + content: vec![ContentPart::Image(ImageSource { + data: ImageData::Url("https://example.com/cat.png".into()), + })], + tool_calls: Vec::new(), + tool_call_id: None, + name: None, + reasoning: None, + tool_error: false, + }; + match build_msg_entries(None, None, &[url_msg], false) { + Ok(_) => panic!("expected 400, got Ok"), + Err(resp) => assert_eq!(resp.status(), StatusCode::BAD_REQUEST), + } + } +}