From eee047ecb53b194da8d44c23ad9b4fb2f9f0de12 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Thu, 4 Jun 2026 11:45:00 -0400 Subject: [PATCH 01/10] feat(ai): execute a turn's tool calls concurrently via a shared ToolCallExecutor Agent delegations are ordinary tool calls, so whether parallel delegations issued in a single model turn actually run concurrently was decided by each provider's tool-call loop, and those loops were inconsistent: Anthropic and Google ran them in parallel while the OpenAI Response API and Ollama ran them sequentially. As a result, parallel delegations ran one-after-another on the sequential providers. Introduce an injectable `ToolCallExecutor` service in @theia/ai-core that executes the tool calls of a single model turn concurrently (Promise.all) with uniform error handling: tool-not-found and thrown handlers become error results instead of rejections, so one failing call never short-circuits its siblings, and results are returned in input order for deterministic message threading. The concurrency contract ("tool calls in one turn run concurrently; the model serializes dependent calls across turns") is documented on the service and on ToolRequest.handler. The service is bound in DI so that adopters can rebind it. Refactor Anthropic and Google onto the executor (no behavior change) and fix the OpenAI Response API and Ollama providers to execute tool calls concurrently while preserving their per-call side effects and ordering. Part of #17533 Signed-off-by: Christian W. Damus --- CHANGELOG.md | 2 + .../src/node/anthropic-backend-module.ts | 21 +++ .../src/node/anthropic-language-model.ts | 43 +++-- .../anthropic-language-models-manager-impl.ts | 35 ++-- packages/ai-core/src/common/index.ts | 1 + packages/ai-core/src/common/language-model.ts | 13 ++ .../src/common/tool-call-execution.spec.ts | 157 ++++++++++++++++++ .../ai-core/src/common/tool-call-execution.ts | 143 ++++++++++++++++ .../src/node/ai-core-backend-module.ts | 8 +- .../src/node/google-backend-module.ts | 16 ++ .../src/node/google-language-model.ts | 49 +++--- .../google-language-models-manager-impl.ts | 25 +-- .../src/node/ollama-backend-module.ts | 15 +- .../src/node/ollama-language-model.ts | 52 ++++-- .../ollama-language-models-manager-impl.ts | 19 ++- packages/ai-ollama/src/package.spec.ts | 43 ++++- .../node/openai-response-api-utils.spec.ts | 56 ++++++- .../src/node/openai-response-api-utils.ts | 80 ++++----- 18 files changed, 638 insertions(+), 140 deletions(-) create mode 100644 packages/ai-core/src/common/tool-call-execution.spec.ts create mode 100644 packages/ai-core/src/common/tool-call-execution.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index bac83479e5543..0d8416c7f89f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - [ai-core] discovered skills from `.agents/skills` directories alongside `.prompts/skills` (workspace and home directory) [#17553](https://github.com/eclipse-theia/theia/pull/17553) - [terminal] fixed Cmd+V / Ctrl+V paste in the integrated terminal and restored the effect of the `terminal.enablePaste` and `terminal.enableCopy` preferences [#17603](https://github.com/eclipse-theia/theia/pull/17603) +- [ai-anthropic, ai-google, ai-ollama, ai-openai, ai-copilot] added rebindable `LanguageModelFactory` bindings for instantiating provider language models [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-core] executed a model turn's tool calls concurrently, including parallel agent delegations, instead of sequentially, via a new injectable `ToolCallExecutor` [#17623](https://github.com/eclipse-theia/theia/pull/17623) [Breaking Changes:](#breaking_changes_1.73.0) diff --git a/packages/ai-anthropic/src/node/anthropic-backend-module.ts b/packages/ai-anthropic/src/node/anthropic-backend-module.ts index 5d8bd79ea13af..1a12423f5c6f4 100644 --- a/packages/ai-anthropic/src/node/anthropic-backend-module.ts +++ b/packages/ai-anthropic/src/node/anthropic-backend-module.ts @@ -15,9 +15,11 @@ // ***************************************************************************** import { ContainerModule } from '@theia/core/shared/inversify'; +import { ToolCallExecutor } from '@theia/ai-core'; import { ANTHROPIC_LANGUAGE_MODELS_MANAGER_PATH, AnthropicLanguageModelsManager } from '../common/anthropic-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { AnthropicLanguageModelsManagerImpl } from './anthropic-language-models-manager-impl'; +import { AnthropicLanguageModelFactory, AnthropicModel, AnthropicModelParams } from './anthropic-language-model'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; import { AnthropicPreferencesSchema } from '../common/anthropic-preferences'; @@ -25,6 +27,25 @@ import { AnthropicPreferencesSchema } from '../common/anthropic-preferences'; const anthropicConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(AnthropicLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(AnthropicLanguageModelsManager).toService(AnthropicLanguageModelsManagerImpl); + bind(AnthropicLanguageModelFactory).toFactory( + ({ container }) => params => new AnthropicModel( + params.id, + params.model, + params.status, + params.enableStreaming, + params.useCaching, + params.apiKey, + params.url, + params.maxTokens, + params.maxRetries, + params.proxy, + params.reasoningSupport, + params.reasoningApi, + params.supportsXHighEffort, + params.maxInputTokens, + container.get(ToolCallExecutor) + ) + ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(ANTHROPIC_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(AnthropicLanguageModelsManager)) ).inSingletonScope(); diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.ts b/packages/ai-anthropic/src/node/anthropic-language-model.ts index 8f79e0aeebb4c..3067a4068f376 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.ts @@ -15,7 +15,6 @@ // ***************************************************************************** import { - createToolCallError, ImageContent, ImageMimeType, LanguageModel, @@ -29,7 +28,7 @@ import { ReasoningApi, ReasoningSupport, ToolCallResult, - ToolInvocationContext, + ToolCallExecutor, UserRequest } from '@theia/ai-core'; import { CancellationToken, isArray } from '@theia/core'; @@ -218,6 +217,27 @@ function formatToolCallResult(result: ToolCallResult): ToolResultBlockParam['con return result; } +/** Parameters for constructing an {@link AnthropicModel}. */ +export interface AnthropicModelParams { + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + useCaching: boolean; + apiKey: () => string | undefined; + url: string | undefined; + maxTokens?: number; + maxRetries?: number; + proxy?: string; + reasoningSupport?: ReasoningSupport; + reasoningApi?: ReasoningApi; + supportsXHighEffort?: boolean; + maxInputTokens?: number; +} + +export const AnthropicLanguageModelFactory = Symbol('AnthropicLanguageModelFactory'); +export type AnthropicLanguageModelFactory = (params: AnthropicModelParams) => AnthropicModel; + /** * Implements the Anthropic language model integration for Theia. Reasoning-level * translation lives in {@link anthropicReasoningFor}. @@ -238,7 +258,8 @@ export class AnthropicModel implements LanguageModel { public reasoningSupport?: ReasoningSupport, public reasoningApi?: ReasoningApi, public supportsXHighEffort?: boolean, - public maxInputTokens?: number + public maxInputTokens?: number, + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() ) { } protected getSettings(request: LanguageModelRequest): Readonly> { @@ -379,16 +400,12 @@ export class AnthropicModel implements LanguageModel { } } if (toolCalls.length > 0) { - const toolResult = await Promise.all(toolCalls.map(async tc => { - const tool = request.tools?.find(t => t.name === tc.name); - const argsObject = tc.args.length === 0 ? '{}' : tc.args; - const handlerResult = tool - ? await tool.handler(argsObject, ToolInvocationContext.create(tc.id)) - : createToolCallError(`Tool '${tc.name}' not found in the available tools for this request.`, 'tool-not-available'); - - return { name: tc.name, result: handlerResult, id: tc.id, arguments: argsObject }; - - })); + // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. + const toolResult = await that.toolCallExecutor.executeToolCalls( + toolCalls.map(tc => ({ id: tc.id, name: tc.name, arguments: tc.args.length === 0 ? '{}' : tc.args })), + request.tools, + { cancellationToken } + ); const calls = toolResult.map(tr => ({ finished: true, id: tr.id, result: tr.result, function: { name: tr.name, arguments: tr.arguments } })); yield { tool_calls: calls }; diff --git a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts index 65ced61284390..4e686c5e43e12 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-models-manager-impl.ts @@ -19,7 +19,7 @@ import { createProxyFetch, getProxyUrl } from '@theia/ai-core/lib/node'; import { inject, injectable } from '@theia/core/shared/inversify'; import { Anthropic } from '@anthropic-ai/sdk'; import type { ModelInfo } from '@anthropic-ai/sdk/resources/models'; -import { AnthropicModel, DEFAULT_MAX_TOKENS } from './anthropic-language-model'; +import { AnthropicLanguageModelFactory, AnthropicModel, DEFAULT_MAX_TOKENS } from './anthropic-language-model'; import { AnthropicLanguageModelsManager, AnthropicModelDescription } from '../common'; const ANTHROPIC_REASONING_SUPPORT: ReasoningSupport = { @@ -47,6 +47,9 @@ export class AnthropicLanguageModelsManagerImpl implements AnthropicLanguageMode @inject(LanguageModelRegistry) protected readonly languageModelRegistry: LanguageModelRegistry; + @inject(AnthropicLanguageModelFactory) + protected readonly anthropicLanguageModelFactory: AnthropicLanguageModelFactory; + get apiKey(): string | undefined { return this._apiKey ?? process.env.ANTHROPIC_API_KEY; } @@ -94,22 +97,22 @@ export class AnthropicLanguageModelsManagerImpl implements AnthropicLanguageMode }); } else { this.languageModelRegistry.addLanguageModels([ - new AnthropicModel( - modelDescription.id, - modelDescription.model, + this.anthropicLanguageModelFactory({ + id: modelDescription.id, + model: modelDescription.model, status, - modelDescription.enableStreaming, - modelDescription.useCaching, - apiKeyProvider, - modelDescription.url, - metadata.maxTokens, - modelDescription.maxRetries, - proxyUrl, - metadata.reasoningSupport, - metadata.reasoningApi, - metadata.supportsXHighEffort, - metadata.maxInputTokens - ) + enableStreaming: modelDescription.enableStreaming, + useCaching: modelDescription.useCaching, + apiKey: apiKeyProvider, + url: modelDescription.url, + maxTokens: metadata.maxTokens, + maxRetries: modelDescription.maxRetries, + proxy: proxyUrl, + reasoningSupport: metadata.reasoningSupport, + reasoningApi: metadata.reasoningApi, + supportsXHighEffort: metadata.supportsXHighEffort, + maxInputTokens: metadata.maxInputTokens + }) ]); } } diff --git a/packages/ai-core/src/common/index.ts b/packages/ai-core/src/common/index.ts index 8ab33f7008793..726663e106196 100644 --- a/packages/ai-core/src/common/index.ts +++ b/packages/ai-core/src/common/index.ts @@ -22,6 +22,7 @@ export * from './language-model-delegate'; export * from './language-model-util'; export * from './language-model'; export * from './language-model-alias'; +export * from './tool-call-execution'; export * from './prompt-service'; export * from './prompt-service-util'; export * from './proxy-util'; diff --git a/packages/ai-core/src/common/language-model.ts b/packages/ai-core/src/common/language-model.ts index 329cac865183b..0681908c5698d 100644 --- a/packages/ai-core/src/common/language-model.ts +++ b/packages/ai-core/src/common/language-model.ts @@ -142,6 +142,19 @@ export interface ToolRequest Promise; providerName?: string; diff --git a/packages/ai-core/src/common/tool-call-execution.spec.ts b/packages/ai-core/src/common/tool-call-execution.spec.ts new file mode 100644 index 0000000000000..d72f22e552095 --- /dev/null +++ b/packages/ai-core/src/common/tool-call-execution.spec.ts @@ -0,0 +1,157 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { expect } from 'chai'; +import { CancellationToken, CancellationTokenSource } from '@theia/core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; +import { + ToolRequest, + ToolInvocationContext, + hasToolCallError, + isToolNotAvailableError, + isToolCallContent +} from './language-model'; +import { ToolCallExecutor, ToolCallExecutionResult } from './tool-call-execution'; + +/** Builds a minimal {@link ToolRequest} whose handler delegates to `handler`. */ +function tool(name: string, handler: ToolRequest['handler']): ToolRequest { + return { id: name, name, parameters: { type: 'object', properties: {} }, handler }; +} + +describe('ToolCallExecutor', () => { + let executor: ToolCallExecutor; + + beforeEach(() => { + executor = new ToolCallExecutor(); + }); + + it('executes the tool calls of a turn concurrently (not sequentially)', async () => { + // `a` only completes once `b` has started. A sequential implementation that runs + // `a` before `b` would deadlock here, so this is the core regression test for #17533. + const bStarted = new Deferred(); + const tools = [ + tool('a', async () => { + await bStarted.promise; + return 'a-done'; + }), + tool('b', async () => { + bStarted.resolve(); + return 'b-done'; + }) + ]; + + const results = await executor.executeToolCalls( + [{ id: '1', name: 'a', arguments: '{}' }, { id: '2', name: 'b', arguments: '{}' }], + tools + ); + + expect(results.map(r => r.result)).to.deep.equal(['a-done', 'b-done']); + }); + + it('preserves input order even when calls complete out of order', async () => { + const aResolves = new Deferred(); + const tools = [ + tool('slow', async () => { + await aResolves.promise; + return 'slow-done'; + }), + tool('fast', async () => { + // Let `fast` finish first, then release `slow`. + aResolves.resolve(); + return 'fast-done'; + }) + ]; + + const results = await executor.executeToolCalls( + [{ id: 'a', name: 'slow', arguments: '{}' }, { id: 'b', name: 'fast', arguments: '{}' }], + tools + ); + + expect(results.map(r => r.id)).to.deep.equal(['a', 'b']); + expect(results.map(r => r.result)).to.deep.equal(['slow-done', 'fast-done']); + }); + + it('reports a tool-not-available error when no tool matches the name', async () => { + const results = await executor.executeToolCalls( + [{ id: '1', name: 'missing', arguments: '{}' }], + [tool('present', async () => 'ok')] + ); + + expect(results).to.have.lengthOf(1); + expect(results[0].notFound).to.equal(true); + expect(results[0].error).to.equal(undefined); + expect(isToolCallContent(results[0].result)).to.equal(true); + expect(isToolCallContent(results[0].result) && results[0].result.content.some(isToolNotAvailableError)).to.equal(true); + }); + + it('isolates a throwing handler: it does not reject, exposes the error, and siblings still resolve', async () => { + const boom = new Error('handler boom'); + const results = await executor.executeToolCalls( + [{ id: '1', name: 'throws', arguments: '{}' }, { id: '2', name: 'ok', arguments: '{}' }], + [ + tool('throws', async () => { throw boom; }), + tool('ok', async () => 'ok-result') + ] + ); + + const thrown = results.find(r => r.id === '1')!; + const succeeded = results.find(r => r.id === '2')!; + expect(thrown.error).to.equal(boom); + expect(hasToolCallError(thrown.result)).to.equal(true); + expect(thrown.notFound).to.equal(false); + expect(succeeded.result).to.equal('ok-result'); + }); + + it('invokes onResult exactly once per tool call', async () => { + const seen: string[] = []; + await executor.executeToolCalls( + [{ id: '1', name: 'a', arguments: '{}' }, { id: '2', name: 'b', arguments: '{}' }], + [tool('a', async () => 'a'), tool('b', async () => 'b')], + { onResult: (r: ToolCallExecutionResult) => seen.push(r.id) } + ); + + expect(seen.slice().sort()).to.deep.equal(['1', '2']); + }); + + it('forwards the cancellation token into each ToolInvocationContext', async () => { + const source = new CancellationTokenSource(); + let received: CancellationToken | undefined; + await executor.executeToolCalls( + [{ id: '1', name: 'a', arguments: '{}' }], + [tool('a', async (_args, ctx) => { received = ToolInvocationContext.getCancellationToken(ctx); return 'a'; })], + { cancellationToken: source.token } + ); + + expect(received).to.equal(source.token); + }); + + it('does not let a throwing onResult callback reject the execution or affect the results', async () => { + const results = await executor.executeToolCalls( + [{ id: '1', name: 'a', arguments: '{}' }, { id: '2', name: 'b', arguments: '{}' }], + [tool('a', async () => 'a-done'), tool('b', async () => 'b-done')], + { onResult: () => { throw new Error('reporting boom'); } } + ); + + expect(results.map(r => r.result)).to.deep.equal(['a-done', 'b-done']); + }); + + it('returns an empty array and never calls onResult for empty input', async () => { + let called = false; + const results = await executor.executeToolCalls([], [tool('a', async () => 'a')], { onResult: () => { called = true; } }); + expect(results).to.deep.equal([]); + expect(called).to.equal(false); + }); +}); diff --git a/packages/ai-core/src/common/tool-call-execution.ts b/packages/ai-core/src/common/tool-call-execution.ts new file mode 100644 index 0000000000000..25c4557fc7b6b --- /dev/null +++ b/packages/ai-core/src/common/tool-call-execution.ts @@ -0,0 +1,143 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { CancellationToken } from '@theia/core'; +import { injectable } from '@theia/core/shared/inversify'; +import { + ToolRequest, + ToolCallResult, + ToolInvocationContext, + createToolCallError +} from './language-model'; + +/** + * A single tool call collected from one model response/turn, in a provider-neutral shape. + * Each language model normalizes its own representation into this shape before execution. + */ +export interface PreparedToolCall { + /** The tool call ID assigned by the model; forwarded into the {@link ToolInvocationContext}. */ + readonly id: string; + /** The name used to match against {@link ToolRequest.name}. */ + readonly name: string; + /** The raw JSON argument string passed verbatim to {@link ToolRequest.handler}. */ + readonly arguments: string; +} + +/** + * The normalized outcome of executing one tool call. Returned in the same order as the + * input array, regardless of the order in which the calls actually completed. + */ +export interface ToolCallExecutionResult { + readonly id: string; + readonly name: string; + readonly arguments: string; + /** + * The handler result, or a {@link createToolCallError} value if the tool was not found + * or the handler threw. A thrown handler yields an error result here, never a rejection. + */ + readonly result: ToolCallResult; + /** The original error if the handler threw; `undefined` for success and for tool-not-found. */ + readonly error?: Error; + /** `true` when no {@link ToolRequest} matched {@link PreparedToolCall.name}. */ + readonly notFound: boolean; +} + +export interface ToolCallExecutionOptions { + /** + * Optional per-call hook invoked exactly once per tool call as soon as that call settles + * (success, throw, or not-found). Use it for per-call side effects such as streaming a UI + * event or appending a provider message. + * + * **Note:** because calls run concurrently, this hook fires in completion order, not input + * order. Side effects that must be ordered should instead be performed by the caller over + * the returned (input-ordered) array of execution results. + */ + readonly onResult?: (result: ToolCallExecutionResult) => void; + /** Optional cancellation token forwarded into each {@link ToolInvocationContext}. */ + readonly cancellationToken?: CancellationToken; +} + +/** + * Executes the tool calls of a single language model response/turn. + * + * Concurrency contract: tool calls emitted within a single model response/turn are executed + * concurrently. All matched handlers are started before any of them is awaited. + * A tool handler must therefore not assume that it runs in isolation or in array + * order, and must be safe to overlap with sibling calls from the same turn. Models serialize + * dependent calls across turns (by withholding a dependent call until it has seen the prior + * result), so "issued together in one turn" is the model's signal that the calls are independent. + * + * Error handling is uniform across all language model providers: + * - If no {@link ToolRequest} matches a call's name, its result is a + * {@link createToolCallError} with kind `'tool-not-available'` and `notFound` is `true`. + * - If a handler throws/rejects, the rejection is caught, logged, and converted to a + * {@link createToolCallError}; the original error is exposed on `error`. + * - The overall tool execution promise never rejects but returns a result that may + * include errors from failed tool calls + */ +@injectable() +export class ToolCallExecutor { + + /** + * Executes all `toolCalls` concurrently and returns their outcomes in input order. + * + * @param toolCalls the tool calls collected for this turn (id + name + raw args) + * @param tools the tools available for this request (typically `request.tools`) + * @param options optional per-call hook and cancellation token + */ + async executeToolCalls( + toolCalls: readonly PreparedToolCall[], + tools: readonly ToolRequest[] | undefined, + options: ToolCallExecutionOptions = {} + ): Promise { + return Promise.all(toolCalls.map(toolCall => this.executeToolCall(toolCall, tools, options))); + } + + /** + * Executes a single tool call, applying the uniform error handling described on the class. + * Subclasses may override this to customize per-call behavior. + */ + protected async executeToolCall( + toolCall: PreparedToolCall, + tools: readonly ToolRequest[] | undefined, + options: ToolCallExecutionOptions + ): Promise { + const { id, name, arguments: args } = toolCall; + const tool = tools?.find(candidate => candidate.name === name); + let outcome: ToolCallExecutionResult; + if (!tool) { + outcome = { + id, name, arguments: args, notFound: true, + result: createToolCallError(`Tool '${name}' not found in the available tools for this request.`, 'tool-not-available') + }; + } else { + try { + const result = await tool.handler(args, ToolInvocationContext.create(id, options.cancellationToken)); + outcome = { id, name, arguments: args, result, notFound: false }; + } catch (e) { + const error = e instanceof Error ? e : new Error(String(e)); + console.error(`Error executing tool ${name}:`, e); + outcome = { id, name, arguments: args, notFound: false, error, result: createToolCallError(error.message || 'Tool execution failed') }; + } + } + try { + options.onResult?.(outcome); + } catch (error) { + console.error('Uncaught error in tool-call onResult call-back.', error); + } + return outcome; + } +} diff --git a/packages/ai-core/src/node/ai-core-backend-module.ts b/packages/ai-core/src/node/ai-core-backend-module.ts index bd85f5816fd30..9f056d168ec87 100644 --- a/packages/ai-core/src/node/ai-core-backend-module.ts +++ b/packages/ai-core/src/node/ai-core-backend-module.ts @@ -40,7 +40,8 @@ import { LanguageModelRegistryClient, TokenUsageService, TokenUsageServiceClient, - TOKEN_USAGE_SERVICE_PATH + TOKEN_USAGE_SERVICE_PATH, + ToolCallExecutor } from '../common'; import { BackendLanguageModelRegistryImpl } from './backend-language-model-registry'; import { TokenUsageServiceImpl } from './token-usage-service-impl'; @@ -112,5 +113,10 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: AgentSettingsPreferenceSchema }); bindAICorePreferences(bind); + // Bound on the root container because it is injected by extant root-level dependents (notably + // OpenAiResponseApiUtils). This is safe only because the service is stateless. A pending refactoring + // should relocate it, together with those dependents, into the connection-scoped container so that a + // substituted, potentially stateful executor cannot leak state across frontend connections. + bind(ToolCallExecutor).toSelf().inSingletonScope(); bind(ConnectionContainerModule).toConstantValue(aiCoreConnectionModule); }); diff --git a/packages/ai-google/src/node/google-backend-module.ts b/packages/ai-google/src/node/google-backend-module.ts index 65481338eeb91..dd35465fe2d5e 100644 --- a/packages/ai-google/src/node/google-backend-module.ts +++ b/packages/ai-google/src/node/google-backend-module.ts @@ -15,9 +15,11 @@ // ***************************************************************************** import { ContainerModule } from '@theia/core/shared/inversify'; +import { ToolCallExecutor } from '@theia/ai-core'; import { GOOGLE_LANGUAGE_MODELS_MANAGER_PATH, GoogleLanguageModelsManager } from '../common/google-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { GoogleLanguageModelsManagerImpl } from './google-language-models-manager-impl'; +import { GoogleLanguageModelFactory, GoogleModel, GoogleModelParams } from './google-language-model'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; import { GooglePreferencesSchema } from '../common/google-preferences'; @@ -25,6 +27,20 @@ import { GooglePreferencesSchema } from '../common/google-preferences'; const geminiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(GoogleLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(GoogleLanguageModelsManager).toService(GoogleLanguageModelsManagerImpl); + bind(GoogleLanguageModelFactory).toFactory( + ({ container }) => params => new GoogleModel( + params.id, + params.model, + params.status, + params.enableStreaming, + params.apiKey, + params.retrySettings, + params.reasoningSupport, + params.reasoningApi, + params.maxInputTokens, + container.get(ToolCallExecutor) + ) + ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(GOOGLE_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(GoogleLanguageModelsManager)) ).inSingletonScope(); diff --git a/packages/ai-google/src/node/google-language-model.ts b/packages/ai-google/src/node/google-language-model.ts index a0912d98b6f8d..a48b9eee12fb7 100644 --- a/packages/ai-google/src/node/google-language-model.ts +++ b/packages/ai-google/src/node/google-language-model.ts @@ -14,7 +14,6 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** import { - createToolCallError, ImageContent, LanguageModel, LanguageModelMessage, @@ -27,7 +26,7 @@ import { ReasoningApi, ReasoningSupport, ToolCallResult, - ToolInvocationContext, + ToolCallExecutor, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; @@ -137,6 +136,22 @@ function toGoogleRole(message: LanguageModelMessage): 'user' | 'model' { } } +/** Parameters for constructing a {@link GoogleModel}. */ +export interface GoogleModelParams { + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + apiKey: () => string | undefined; + retrySettings: () => GoogleLanguageModelRetrySettings; + reasoningSupport?: ReasoningSupport; + reasoningApi?: ReasoningApi; + maxInputTokens?: number; +} + +export const GoogleLanguageModelFactory = Symbol('GoogleLanguageModelFactory'); +export type GoogleLanguageModelFactory = (params: GoogleModelParams) => GoogleModel; + /** * Implements the Gemini language model integration for Theia. Reasoning-level * translation lives in {@link googleReasoningFor}. @@ -152,7 +167,8 @@ export class GoogleModel implements LanguageModel { public retrySettings: () => GoogleLanguageModelRetrySettings, public reasoningSupport?: ReasoningSupport, public reasoningApi?: ReasoningApi, - public maxInputTokens?: number + public maxInputTokens?: number, + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() ) { } protected getSettings(request: LanguageModelRequest): Readonly> { @@ -328,27 +344,12 @@ export class GoogleModel implements LanguageModel { // Process tool calls if any exist const toolCalls = Object.values(toolCallMap); if (toolCalls.length > 0) { - // Collect tool results - const toolResult = await Promise.all(toolCalls.map(async tc => { - const tool = request.tools?.find(t => t.name === tc.name); - let result; - if (!tool) { - result = createToolCallError(`Tool '${tc.name}' not found in the available tools for this request.`, 'tool-not-available'); - } else { - try { - result = await tool.handler(tc.args, ToolInvocationContext.create(tc.id)); - } catch (e) { - console.error(`Error executing tool ${tc.name}:`, e); - result = createToolCallError(e.message || 'Tool execution failed'); - } - } - return { - name: tc.name, - result: result, - id: tc.id, - arguments: tc.args, - }; - })); + // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. + const toolResult = await that.toolCallExecutor.executeToolCalls( + toolCalls.map(tc => ({ id: tc.id, name: tc.name, arguments: tc.args })), + request.tools, + { cancellationToken } + ); // Generate tool call responses const calls = toolResult.map(tr => ({ diff --git a/packages/ai-google/src/node/google-language-models-manager-impl.ts b/packages/ai-google/src/node/google-language-models-manager-impl.ts index 0037c1f638083..8bb6b032ba1b7 100644 --- a/packages/ai-google/src/node/google-language-models-manager-impl.ts +++ b/packages/ai-google/src/node/google-language-models-manager-impl.ts @@ -17,7 +17,7 @@ import { LanguageModelRegistry, LanguageModelStatus, ReasoningApi, ReasoningSupport } from '@theia/ai-core'; import { inject, injectable } from '@theia/core/shared/inversify'; import { GoogleGenAI, Model } from '@google/genai'; -import { GoogleModel } from './google-language-model'; +import { GoogleLanguageModelFactory, GoogleModel } from './google-language-model'; import { GoogleLanguageModelsManager, GoogleModelDescription } from '../common'; export interface GoogleLanguageModelRetrySettings { @@ -66,6 +66,9 @@ export class GoogleLanguageModelsManagerImpl implements GoogleLanguageModelsMana @inject(LanguageModelRegistry) protected readonly languageModelRegistry: LanguageModelRegistry; + @inject(GoogleLanguageModelFactory) + protected readonly googleLanguageModelFactory: GoogleLanguageModelFactory; + get apiKey(): string | undefined { return this._apiKey ?? process.env.GOOGLE_API_KEY ?? process.env.GEMINI_API_KEY; } @@ -114,17 +117,17 @@ export class GoogleLanguageModelsManagerImpl implements GoogleLanguageModelsMana }); } else { this.languageModelRegistry.addLanguageModels([ - new GoogleModel( - modelDescription.id, - modelDescription.model, + this.googleLanguageModelFactory({ + id: modelDescription.id, + model: modelDescription.model, status, - modelDescription.enableStreaming, - apiKeyProvider, - retrySettingsProvider, - metadata.reasoningSupport, - metadata.reasoningApi, - metadata.maxInputTokens - ) + enableStreaming: modelDescription.enableStreaming, + apiKey: apiKeyProvider, + retrySettings: retrySettingsProvider, + reasoningSupport: metadata.reasoningSupport, + reasoningApi: metadata.reasoningApi, + maxInputTokens: metadata.maxInputTokens + }) ]); } } diff --git a/packages/ai-ollama/src/node/ollama-backend-module.ts b/packages/ai-ollama/src/node/ollama-backend-module.ts index 918b3b2170be5..6f02e5db15edc 100644 --- a/packages/ai-ollama/src/node/ollama-backend-module.ts +++ b/packages/ai-ollama/src/node/ollama-backend-module.ts @@ -15,18 +15,29 @@ // ***************************************************************************** import { ContainerModule } from '@theia/core/shared/inversify'; +import { ToolCallExecutor } from '@theia/ai-core'; import { OLLAMA_LANGUAGE_MODELS_MANAGER_PATH, OllamaLanguageModelsManager } from '../common/ollama-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { OllamaLanguageModelsManagerImpl } from './ollama-language-models-manager-impl'; +import { OllamaLanguageModelFactory, OllamaModel, OllamaModelParams } from './ollama-language-model'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; import { OllamaPreferencesSchema } from '../common/ollama-preferences'; -export const OllamaModelFactory = Symbol('OllamaModelFactory'); - // We use a connection module to handle AI services separately for each frontend. const ollamaConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(OllamaLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(OllamaLanguageModelsManager).toService(OllamaLanguageModelsManagerImpl); + bind(OllamaLanguageModelFactory).toFactory( + ({ container }) => params => new OllamaModel( + params.id, + params.model, + params.status, + params.host, + params.proxy, + params.reasoningSupport, + container.get(ToolCallExecutor) + ) + ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(OLLAMA_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(OllamaLanguageModelsManager)) ).inSingletonScope(); diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index d8279400f6407..cf48dbcdf79ca 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -24,6 +24,8 @@ import { ReasoningSettings, ReasoningSupport, ToolCall, + ToolCallExecutor, + ToolCallResult, ToolRequest, ToolRequestParametersProperties, ImageContent, @@ -39,6 +41,19 @@ import { ollamaThinkParamFor } from './ollama-reasoning'; export const OllamaModelIdentifier = Symbol('OllamaModelIdentifier'); +/** Parameters for constructing an {@link OllamaModel}. */ +export interface OllamaModelParams { + id: string; + model: string; + status: LanguageModelStatus; + host: () => string | undefined; + proxy?: string; + reasoningSupport?: ReasoningSupport; +} + +export const OllamaLanguageModelFactory = Symbol('OllamaLanguageModelFactory'); +export type OllamaLanguageModelFactory = (params: OllamaModelParams) => OllamaModel; + export class OllamaModel implements LanguageModel { protected readonly DEFAULT_REQUEST_SETTINGS: Partial> = { @@ -61,7 +76,8 @@ export class OllamaModel implements LanguageModel { public status: LanguageModelStatus, protected host: () => string | undefined, public proxy?: string, - public reasoningSupport?: ReasoningSupport + public reasoningSupport?: ReasoningSupport, + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() ) { } async request(request: UserRequest, cancellationToken?: CancellationToken): Promise { @@ -363,18 +379,28 @@ export class OllamaModel implements LanguageModel { private async processToolCalls(toolCalls: ToolCall[], chatRequest: ExtendedChatRequest): Promise { const tools: ToolWithHandler[] = chatRequest.tools ?? []; + // Adapt Ollama's ToolWithHandler to the ToolRequest shape expected by the executor. + const toolRequests: ToolRequest[] = tools.map(tool => ({ + id: tool.function.name ?? '', + name: tool.function.name ?? '', + parameters: { type: 'object', properties: {} }, + handler: async argString => (await tool.handler(argString)) as ToolCallResult + })); + + // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. + const results = await this.toolCallExecutor.executeToolCalls( + toolCalls.map(call => ({ id: call.id ?? call.function!.name!, name: call.function!.name!, arguments: call.function!.arguments! })), + toolRequests + ); + + // Build the messages and response entries from the input-ordered results so that the + // next turn sees a deterministic ordering. const toolCallsForResponse: ToolCall[] = []; - - for (const call of toolCalls) { - const functionToCall = tools.find(tool => tool.function.name === call.function!.name); - let funcResult: string; - - if (functionToCall) { - const rawResult = await functionToCall.handler(call.function!.arguments!); - funcResult = typeof rawResult === 'string' ? rawResult : JSON.stringify(rawResult); - } else { - funcResult = 'error: Tool not found'; - } + toolCalls.forEach((call, index) => { + const outcome = results[index]; + const funcResult = outcome.notFound + ? 'error: Tool not found' + : typeof outcome.result === 'string' ? outcome.result : JSON.stringify(outcome.result); chatRequest.messages.push({ role: 'tool', @@ -387,7 +413,7 @@ export class OllamaModel implements LanguageModel { result: String(funcResult), finished: true }); - } + }); return toolCallsForResponse; } diff --git a/packages/ai-ollama/src/node/ollama-language-models-manager-impl.ts b/packages/ai-ollama/src/node/ollama-language-models-manager-impl.ts index 483260ea00d7b..a894bb46e3616 100644 --- a/packages/ai-ollama/src/node/ollama-language-models-manager-impl.ts +++ b/packages/ai-ollama/src/node/ollama-language-models-manager-impl.ts @@ -17,7 +17,7 @@ import { LanguageModelRegistry, LanguageModelStatus } from '@theia/ai-core'; import { getProxyUrl } from '@theia/ai-core/lib/node'; import { inject, injectable } from '@theia/core/shared/inversify'; -import { OllamaModel } from './ollama-language-model'; +import { OllamaLanguageModelFactory, OllamaModel } from './ollama-language-model'; import { OllamaLanguageModelsManager, OllamaModelDescription } from '../common'; @injectable() @@ -29,6 +29,9 @@ export class OllamaLanguageModelsManagerImpl implements OllamaLanguageModelsMana @inject(LanguageModelRegistry) protected readonly languageModelRegistry: LanguageModelRegistry; + @inject(OllamaLanguageModelFactory) + protected readonly ollamaLanguageModelFactory: OllamaLanguageModelFactory; + get host(): string | undefined { return this._host ?? process.env.OLLAMA_HOST; } @@ -62,14 +65,14 @@ export class OllamaLanguageModelsManagerImpl implements OllamaLanguageModelsMana } else { const status = this.calculateStatus(host); this.languageModelRegistry.addLanguageModels([ - new OllamaModel( - modelDescription.id, - modelDescription.model, + this.ollamaLanguageModelFactory({ + id: modelDescription.id, + model: modelDescription.model, status, - hostProvider, - proxyUrl, - modelDescription.reasoningSupport - ) + host: hostProvider, + proxy: proxyUrl, + reasoningSupport: modelDescription.reasoningSupport + }) ]); } } diff --git a/packages/ai-ollama/src/package.spec.ts b/packages/ai-ollama/src/package.spec.ts index 56b0581617b51..13b3039b7ebd2 100644 --- a/packages/ai-ollama/src/package.spec.ts +++ b/packages/ai-ollama/src/package.spec.ts @@ -14,7 +14,8 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ToolRequest } from '@theia/ai-core'; +import { ToolCall, ToolRequest } from '@theia/ai-core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; import { OllamaModel } from './node/ollama-language-model'; import { Tool } from 'ollama'; import { expect } from 'chai'; @@ -33,6 +34,40 @@ describe('ai-ollama package', () => { expect(ollamaTool.function.parameters?.properties).to.deep.equal(req.parameters.properties); expect(ollamaTool.function.parameters?.required).to.deep.equal(['question']); }); + + it('executes tool calls of a turn concurrently and preserves input order', async () => { + const model = new OllamaModelUnderTest(); + // `a` only completes once `b` has started: a sequential implementation would deadlock here. + const bStarted = new Deferred(); + const chatRequest = { + messages: [], + tools: [ + { function: { name: 'a' }, handler: async () => { await bStarted.promise; return 'a-result'; } }, + { function: { name: 'b' }, handler: async () => { bStarted.resolve(); return 'b-result'; } } + ] + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + const toolCalls: ToolCall[] = [ + { id: '1', function: { name: 'a', arguments: '{}' } }, + { id: '2', function: { name: 'b', arguments: '{}' } } + ]; + + const result = await model.runProcessToolCalls(toolCalls, chatRequest); + + expect(result.map(r => r.result)).to.deep.equal(['a-result', 'b-result']); + expect(chatRequest.messages.map((m: { content: string }) => m.content)).to.deep.equal([ + 'Tool call a returned: a-result', + 'Tool call b returned: b-result' + ]); + }); + + it('reports a missing tool with the legacy error string', async () => { + const model = new OllamaModelUnderTest(); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const chatRequest = { messages: [], tools: [] } as any; + const result = await model.runProcessToolCalls([{ id: '1', function: { name: 'missing', arguments: '{}' } }], chatRequest); + expect(result[0].result).to.equal('error: Tool not found'); + }); }); class OllamaModelUnderTest extends OllamaModel { @@ -43,6 +78,12 @@ class OllamaModelUnderTest extends OllamaModel { override toOllamaTool(tool: ToolRequest): Tool & { handler: (arg_string: string) => Promise } { return super.toOllamaTool(tool); } + + // Exposes the private processToolCalls for testing concurrent tool execution. + runProcessToolCalls(toolCalls: ToolCall[], chatRequest: unknown): Promise { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (this as any).processToolCalls(toolCalls, chatRequest); + } } function createToolRequest(): ToolRequest { return { diff --git a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts index 4327bac54c09c..ad2cc86346716 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts @@ -15,7 +15,8 @@ // ***************************************************************************** import { expect } from 'chai'; -import { isUsageResponsePart, LanguageModelStreamResponsePart, UserRequest } from '@theia/ai-core'; +import { isToolCallResponsePart, isUsageResponsePart, LanguageModelStreamResponsePart, UserRequest } from '@theia/ai-core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; import { OpenAiModelUtils } from './openai-language-model'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; @@ -25,6 +26,13 @@ async function* toStream(events: unknown[]): AsyncIterable { } } +function functionCallItem(id: string, name: string, args: string): unknown { + return { + type: 'response.output_item.added', + item: { id, call_id: id, type: 'function_call', name, arguments: args } + }; +} + describe('OpenAiResponseApiUtils', () => { it('emits per-iteration usage for Response API tool calls instead of accumulated usage', async () => { const utils = new OpenAiResponseApiUtils(); @@ -106,4 +114,50 @@ describe('OpenAiResponseApiUtils', () => { { input_tokens: 200, output_tokens: 20 } ]); }); + + it('executes the tool calls of a single turn concurrently', async () => { + const utils = new OpenAiResponseApiUtils(); + const streams = [ + [ + functionCallItem('call-a', 'a', '{}'), + functionCallItem('call-b', 'b', '{}'), + { type: 'response.completed', response: { usage: { input_tokens: 1, output_tokens: 1 } } } + ], + [ + { type: 'response.output_text.delta', delta: 'done' }, + { type: 'response.completed', response: { usage: { input_tokens: 2, output_tokens: 2 } } } + ] + ]; + const openai = { + responses: { + stream: () => toStream(streams.shift() ?? []) + } + }; + // `a` only resolves once `b` has started: a sequential implementation would deadlock here. + const bStarted = new Deferred(); + const request: UserRequest = { + sessionId: 'session-1', + requestId: 'request-1', + messages: [{ actor: 'user', type: 'text', text: 'hello' }], + tools: [ + { id: 'a', name: 'a', parameters: { type: 'object', properties: {} }, handler: async () => { await bStarted.promise; return 'a-result'; } }, + { id: 'b', name: 'b', parameters: { type: 'object', properties: {} }, handler: async () => { bStarted.resolve(); return 'b-result'; } } + ] + }; + + const response = await utils.handleRequest(openai as never, request, {}, 'gpt-5', new OpenAiModelUtils(), 'developer', { maxChatCompletions: 3 }, 'openai/gpt-5', true); + const parts: LanguageModelStreamResponsePart[] = []; + if ('stream' in response) { + for await (const part of response.stream) { + parts.push(part); + } + } + + const finishedResults = parts + .filter(isToolCallResponsePart) + .flatMap(part => part.tool_calls) + .filter(call => call.finished) + .map(call => call.result); + expect(finishedResults).to.have.members(['a-result', 'b-result']); + }); }); diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index f525009797e9e..f83b434e255b9 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -15,20 +15,19 @@ // ***************************************************************************** import { - createToolCallError, ImageContent, LanguageModelMessage, LanguageModelResponse, LanguageModelStreamResponsePart, TextMessage, - ToolInvocationContext, + ToolCallExecutor, ToolRequest, ToolRequestParameters, UserRequest } from '@theia/ai-core'; import { CancellationToken, unreachable } from '@theia/core'; import { Deferred } from '@theia/core/lib/common/promise-util'; -import { injectable } from '@theia/core/shared/inversify'; +import { inject, injectable } from '@theia/core/shared/inversify'; import { OpenAI } from 'openai'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; import type { @@ -62,6 +61,10 @@ interface ToolCall { @injectable() export class OpenAiResponseApiUtils { + // Injected when resolved via DI; the initializer keeps direct instantiation (e.g. in tests) working. + @inject(ToolCallExecutor) + readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor(); + /** * Handles Response API requests with proper tool calling cycles. * Works for both streaming and non-streaming cases. @@ -613,66 +616,43 @@ class ResponseApiToolCallIterator implements AsyncIterableIterator { - for (const [itemId, toolCall] of this.currentToolCalls) { - if (toolCall.executed) { - continue; - } - - const tool = this.request.tools?.find(t => t.name === toolCall.name); - if (tool) { - try { - const result = await tool.handler(toolCall.arguments, ToolInvocationContext.create(itemId)); - toolCall.result = result; + // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. + // The per-call `onResult` hook below preserves the original side effects (streaming the + // finished tool-call event and recording the result/error consumed by prepareNextIteration). + const pending = [...this.currentToolCalls].filter(([, toolCall]) => !toolCall.executed); + + await this.utils.toolCallExecutor.executeToolCalls( + pending.map(([itemId, toolCall]) => ({ id: itemId, name: toolCall.name, arguments: toolCall.arguments })), + this.request.tools, + { + cancellationToken: this.cancellationToken, + onResult: outcome => { + const toolCall = this.currentToolCalls.get(outcome.id)!; + if (outcome.notFound) { + toolCall.error = new Error(`Tool ${toolCall.name} not found`); + } else if (outcome.error) { + toolCall.error = outcome.error; + } else { + toolCall.result = outcome.result; + } - // Yield the tool call completion + // Yield the tool call completion (success result, or the error content) this.handleIncoming({ tool_calls: [{ - id: itemId, + id: outcome.id, finished: true, function: { name: toolCall.name, arguments: toolCall.arguments }, - result + result: outcome.result }] }); - } catch (error) { - console.error(`Error executing tool ${toolCall.name}:`, error); - toolCall.error = error instanceof Error ? error : new Error(String(error)); - // Yield the tool call error - this.handleIncoming({ - tool_calls: [{ - id: itemId, - finished: true, - function: { - name: toolCall.name, - arguments: toolCall.arguments - }, - result: createToolCallError(error instanceof Error ? error.message : String(error)) - }] - }); + toolCall.executed = true; } - } else { - console.warn(`Tool ${toolCall.name} not found in request tools`); - toolCall.error = new Error(`Tool ${toolCall.name} not found`); - - // Yield the tool call error - this.handleIncoming({ - tool_calls: [{ - id: itemId, - finished: true, - function: { - name: toolCall.name, - arguments: toolCall.arguments - }, - result: createToolCallError(`Tool '${toolCall.name}' not found in the available tools for this request.`, 'tool-not-available') - }] - }); } - - toolCall.executed = true; - } + ); } protected prepareNextIteration(): void { From b22f9cb51fc5ff422bad8643806a99c33d619c2f Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Thu, 4 Jun 2026 12:49:07 -0400 Subject: [PATCH 02/10] fix(ai-chat): dispose per-content change listeners on clearContent ChatResponseImpl::doAddContent attached a forwarding onDidChange listener to each newly added tool-call content but never disposed it. The stream parser clears and re-adds prior content (including a preceding tool call) on every streamed text token, so the same content accumulated one listener per token, producing "Possible Emitter memory leak detected" warnings once an agent streamed a long text response after a tool call. This surfaced via parallel sub-agent delegation, which drives the Explore agents through exactly that pattern. Register the forwarding listeners against a DisposableCollection that clearContent disposes, so that re-adding content no longer leaks listeners. Signed-off-by: Christian W. Damus --- packages/ai-chat/src/common/chat-model.ts | 9 ++++++-- .../src/common/chat-response-model.spec.ts | 22 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/ai-chat/src/common/chat-model.ts b/packages/ai-chat/src/common/chat-model.ts index a7fb17c2bb179..8b83657f9c073 100644 --- a/packages/ai-chat/src/common/chat-model.ts +++ b/packages/ai-chat/src/common/chat-model.ts @@ -2789,6 +2789,8 @@ class ChatResponseImpl implements ChatResponse { protected _content: ChatResponseContent[]; protected _responseRepresentation: string; protected _responseRepresentationForDisplay: string; + /** Forwarding listeners on per-content change events, disposed and rebuilt whenever the content is reset. */ + protected readonly toDisposeOnClearContent = new DisposableCollection(); constructor() { this._content = []; @@ -2799,6 +2801,7 @@ class ChatResponseImpl implements ChatResponse { } clearContent(): void { + this.toDisposeOnClearContent.dispose(); this._content = []; this._updateResponseRepresentation(); this._onDidChangeEmitter.fire(); @@ -2833,8 +2836,10 @@ class ChatResponseImpl implements ChatResponse { this._content.push(nextContent); // Forward content-level change events (e.g. partial-result updates from a // renderer) so auto-save can persist them. Without this, mutations that - // don't go through addContent/merge are invisible to listeners. - nextContent.onDidChange(() => this._onDidChangeEmitter.fire()); + // don't go through addContent/merge are invisible to listeners. Registered against + // toDisposeOnClearContent so the subscription is disposed when the content is cleared, + // otherwise re-adding the same content (as the stream parser does on every text token) leaks listeners. + nextContent.onDidChange(() => this._onDidChangeEmitter.fire(), undefined, this.toDisposeOnClearContent); } } else { const lastElement = this._content.length > 0 diff --git a/packages/ai-chat/src/common/chat-response-model.spec.ts b/packages/ai-chat/src/common/chat-response-model.spec.ts index 5799e4a17b98f..30ab24a4bb8a7 100644 --- a/packages/ai-chat/src/common/chat-response-model.spec.ts +++ b/packages/ai-chat/src/common/chat-response-model.spec.ts @@ -35,6 +35,28 @@ describe('MutableChatResponseModel', () => { // lost on reload. expect(fireCount).to.equal(1); }); + + it('does not leak content-change listeners when content is repeatedly cleared and re-added', () => { + const response = new MutableChatResponseModel('req-1'); + const toolCall = new ToolCallChatResponseContentImpl('tool-1', 'tool', '{}', false); + response.response.addContent(toolCall); + + // The stream parser clears and re-adds prior content (e.g. a preceding tool call) on + // every streamed text token. Each re-add must not accumulate another forwarding listener. + for (let i = 0; i < 50; i++) { + response.response.clearContent(); + response.response.addContents([toolCall]); + } + + let fireCount = 0; + response.onDidChange(() => { fireCount++; }); + + toolCall.updateResult('partial'); + + // A single content change must propagate exactly once regardless of how many times the + // content was re-added; otherwise each re-add leaks another listener on the content. + expect(fireCount).to.equal(1); + }); }); describe('setTokenUsage', () => { From 2084318202bb0f92c68809d98ffce4ab1b2ccc13 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Thu, 4 Jun 2026 09:21:54 -0400 Subject: [PATCH 03/10] feat(ai): run OpenAI Chat Completions & Copilot tool calls concurrently The OpenAI Chat Completions and Copilot providers used the OpenAI SDK's `chat.completions.runTools()` runner, whose built-in tool loop executes tool calls strictly sequentially with no parallel option. As a result, parallel agent delegations (which are tool calls) ran one after another on these providers. Replace the SDK runner with ChatCompletionStreamingAsyncIterator, which drives the raw chunk stream and a multi-turn tool loop. The tool calls of each turn are executed concurrently via the shared ToolCallExecutor; the iterator emits the same stream-response part shapes as before (open / arguments-delta / finished), threads the assistant tool_calls plus matching tool messages into the next turn, honors maxChatCompletions, and aborts the in-flight stream on cancellation. createTools now returns plain function tool definitions since the SDK runner no longer invokes the handlers. Fixes #17533 Signed-off-by: Christian W. Damus --- CHANGELOG.md | 3 + .../src/node/copilot-backend-module.ts | 17 + .../src/node/copilot-language-model.spec.ts | 22 +- .../src/node/copilot-language-model.ts | 73 +++-- .../copilot-language-models-manager-impl.ts | 25 +- .../src/node/ai-core-backend-module.ts | 7 +- .../src/node/openai-backend-module.ts | 30 +- .../openai-chat-completion-stream.spec.ts | 299 ++++++++++++++++++ .../src/node/openai-chat-completion-stream.ts | 279 ++++++++++++++++ .../src/node/openai-language-model.spec.ts | 19 ++ .../src/node/openai-language-model.ts | 79 +++-- .../openai-language-models-manager-impl.ts | 44 ++- 12 files changed, 796 insertions(+), 101 deletions(-) create mode 100644 packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts create mode 100644 packages/ai-openai/src/node/openai-chat-completion-stream.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d8416c7f89f9..23c406ebd015b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,9 @@ - [ai-core] `DefaultSkillService.getDefaultSkillsDirectoryPath()` has been renamed to `getDefaultSkillsDirectoryPaths()` and now returns `string[]` instead of `string` to include both the product configuration `skills` directory and the user's `~/.agents/skills` directory [#17553](https://github.com/eclipse-theia/theia/pull/17553) - [ai-core] `combineSkillDirectories` signature changed: `workspaceSkillsDir` and `defaultSkillsDir` parameters are now `string[]` (previously `string | undefined`), and the return type is now `SkillDirectoryEntry[]` (an array of `{ path, tier }` entries) instead of `string[]` [#17553](https://github.com/eclipse-theia/theia/pull/17553) - [terminal] `TerminalWidget` gained a new abstract method `paste(text: string)`; downstream subclasses must implement it (consistent with `getSelection()` / `hasSelection()` added in [#17290](https://github.com/eclipse-theia/theia/pull/17290)) [#17603](https://github.com/eclipse-theia/theia/pull/17603) +- [ai-openai] `OpenAiResponseApiUtils` is now bound in the connection-scoped backend container instead of the root container, so it can no longer be injected into root-scoped services [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai] `OpenAiLanguageModelsManagerImpl` no longer injects `OpenAiModelUtils` or `OpenAiResponseApiUtils` (the `openAiModelUtils` and `responseApiUtils` protected fields were removed); provider models are now constructed via the injected `OpenAiLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai, ai-copilot] `OpenAiModel.createTools()` and `CopilotLanguageModel.createTools()` now return `ChatCompletionTool[]` instead of `RunnableToolFunctionWithoutParse[]`, because the OpenAI SDK `runTools` runner is no longer used [#17623](https://github.com/eclipse-theia/theia/pull/17623) ## 1.72.0 - 5/28/2026 diff --git a/packages/ai-copilot/src/node/copilot-backend-module.ts b/packages/ai-copilot/src/node/copilot-backend-module.ts index b65017621fa8c..03a18a3066da8 100644 --- a/packages/ai-copilot/src/node/copilot-backend-module.ts +++ b/packages/ai-copilot/src/node/copilot-backend-module.ts @@ -16,6 +16,7 @@ import { ContainerModule } from '@theia/core/shared/inversify'; import { ConnectionHandler, RpcConnectionHandler } from '@theia/core'; +import { ToolCallExecutor } from '@theia/ai-core'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; import { CopilotLanguageModelsManager, @@ -26,6 +27,7 @@ import { } from '../common'; import { CopilotOAuthConfig, DEFAULT_COPILOT_OAUTH_CONFIG } from '../common/copilot-oauth-config'; import { CopilotLanguageModelsManagerImpl } from './copilot-language-models-manager-impl'; +import { CopilotLanguageModel, CopilotLanguageModelFactory, CopilotLanguageModelParams } from './copilot-language-model'; import { CopilotAuthServiceImpl } from './copilot-auth-service-impl'; const copilotConnectionModule = ConnectionContainerModule.create(({ bind }) => { @@ -35,6 +37,21 @@ const copilotConnectionModule = ConnectionContainerModule.create(({ bind }) => { bind(CopilotLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(CopilotLanguageModelsManager).toService(CopilotLanguageModelsManagerImpl); + bind(CopilotLanguageModelFactory).toFactory( + ({ container }) => params => new CopilotLanguageModel( + params.id, + params.model, + params.status, + params.enableStreaming, + params.supportsStructuredOutput, + params.maxRetries, + params.accessTokenProvider, + params.enterpriseUrlProvider, + params.userAgentProvider, + container.get(ToolCallExecutor) + ) + ); + bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler( COPILOT_AUTH_SERVICE_PATH, diff --git a/packages/ai-copilot/src/node/copilot-language-model.spec.ts b/packages/ai-copilot/src/node/copilot-language-model.spec.ts index 6514574358460..40abdfc336044 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.spec.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.spec.ts @@ -15,7 +15,7 @@ // ***************************************************************************** import { expect } from 'chai'; -import { LanguageModelMessage } from '@theia/ai-core'; +import { LanguageModelMessage, LanguageModelRequest } from '@theia/ai-core'; import { ChatCompletionMessageParam } from 'openai/resources'; import { CopilotLanguageModel } from './copilot-language-model'; @@ -37,6 +37,10 @@ class TestableCopilotLanguageModel extends CopilotLanguageModel { public callProcessMessages(messages: LanguageModelMessage[]): ChatCompletionMessageParam[] { return this.processMessages(messages); } + + public callCreateTools(request: LanguageModelRequest): unknown { + return this.createTools(request); + } } describe('CopilotLanguageModel - processMessages', () => { @@ -104,3 +108,19 @@ describe('CopilotLanguageModel - processMessages', () => { expect(result[2]).to.deep.equal({ role: 'assistant', content: 'final answer' }); }); }); + +describe('CopilotLanguageModel - createTools', () => { + it('produces plain function tool definitions without an embedded handler function', () => { + const model = new TestableCopilotLanguageModel(); + const tools = model.callCreateTools({ + messages: [], + tools: [{ id: 't', name: 't', parameters: { type: 'object', properties: {} }, handler: async () => 'x' }] + }) as Array<{ type: string; function: Record }>; + + expect(tools).to.have.lengthOf(1); + expect(tools[0].type).to.equal('function'); + expect(tools[0].function.name).to.equal('t'); + // The SDK runTools() runner is no longer used, so no executable function is embedded. + expect('function' in tools[0].function).to.equal(false); + }); +}); diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index 00f30433fcbff..2059b6153654f 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -23,16 +23,32 @@ import { LanguageModelResponse, LanguageModelStatus, LanguageModelTextResponse, + ToolCallExecutor, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; import OpenAI from 'openai'; -import { RunnableToolFunctionWithoutParse } from 'openai/lib/RunnableFunction'; -import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam } from 'openai/resources'; +import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; import { StreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-streaming-iterator'; +import { ChatCompletionStreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-chat-completion-stream'; import { COPILOT_PROVIDER_ID, getCopilotApiBaseUrl } from '../common'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; -import type { ChatCompletionStream } from 'openai/lib/ChatCompletionStream'; + +/** Parameters for constructing a {@link CopilotLanguageModel}. */ +export interface CopilotLanguageModelParams { + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + supportsStructuredOutput: boolean; + maxRetries: number; + accessTokenProvider: () => Promise; + enterpriseUrlProvider: () => string | undefined; + userAgentProvider: () => string; +} + +export const CopilotLanguageModelFactory = Symbol('CopilotLanguageModelFactory'); +export type CopilotLanguageModelFactory = (params: CopilotLanguageModelParams) => CopilotLanguageModel; /** * Language model implementation for GitHub Copilot. @@ -54,6 +70,7 @@ export class CopilotLanguageModel implements LanguageModel { protected readonly accessTokenProvider: () => Promise, protected readonly enterpriseUrlProvider: () => string | undefined, protected readonly userAgentProvider: () => string, + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() ) { } protected getSettings(request: LanguageModelRequest): Record { @@ -81,32 +98,33 @@ export class CopilotLanguageModel implements LanguageModel { settings['stream_options'] = { include_usage: true }; } - let runner: ChatCompletionStream; const tools = this.createTools(request); if (tools) { - runner = openai.chat.completions.runTools({ - model: this.model, - messages: this.processMessages(request.messages), - stream: true, - tools: tools, - tool_choice: 'auto', - ...settings - }, { - ...this.runnerOptions, - maxRetries: this.maxRetries - }); - } else { - runner = openai.chat.completions.stream({ - model: this.model, - messages: this.processMessages(request.messages), - stream: true, - ...settings - }); + // Drive the tool loop ourselves so that the tool calls of a single turn run concurrently. + return { + stream: new ChatCompletionStreamingAsyncIterator({ + openai, + model: this.model, + request, + messages: this.processMessages(request.messages), + settings, + tools, + maxChatCompletions: this.runnerOptions.maxChatCompletions ?? 100, + maxRetries: this.maxRetries, + toolCallExecutor: this.toolCallExecutor, + cancellationToken + }) + }; } - // eslint-disable-next-line @typescript-eslint/no-explicit-any - return { stream: new StreamingAsyncIterator(runner as any, cancellationToken) }; + const runner = openai.chat.completions.stream({ + model: this.model, + messages: this.processMessages(request.messages), + stream: true, + ...settings + }); + return { stream: new StreamingAsyncIterator(runner, cancellationToken) }; } protected async handleNonStreamingRequest(openai: OpenAI, request: UserRequest): Promise { @@ -152,16 +170,15 @@ export class CopilotLanguageModel implements LanguageModel { }; } - protected createTools(request: LanguageModelRequest): RunnableToolFunctionWithoutParse[] | undefined { + protected createTools(request: LanguageModelRequest): ChatCompletionTool[] | undefined { return request.tools?.map(tool => ({ type: 'function', function: { name: tool.name, description: tool.description, - parameters: tool.parameters, - function: (args_string: string) => tool.handler(args_string) + parameters: tool.parameters } - } as RunnableToolFunctionWithoutParse)); + } as unknown as ChatCompletionTool)); } protected async initializeCopilotClient(): Promise { diff --git a/packages/ai-copilot/src/node/copilot-language-models-manager-impl.ts b/packages/ai-copilot/src/node/copilot-language-models-manager-impl.ts index 5a21e3354a0bc..c5ec1192adf74 100644 --- a/packages/ai-copilot/src/node/copilot-language-models-manager-impl.ts +++ b/packages/ai-copilot/src/node/copilot-language-models-manager-impl.ts @@ -19,7 +19,7 @@ import { Disposable, DisposableCollection } from '@theia/core'; import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import { CopilotLanguageModelsManager, CopilotModelDescription, COPILOT_PROVIDER_ID, getCopilotApiBaseUrl } from '../common'; import { CopilotOAuthConfig } from '../common/copilot-oauth-config'; -import { CopilotLanguageModel } from './copilot-language-model'; +import { CopilotLanguageModel, CopilotLanguageModelFactory } from './copilot-language-model'; import { CopilotAuthServiceImpl } from './copilot-auth-service-impl'; /** @@ -32,6 +32,9 @@ export class CopilotLanguageModelsManagerImpl implements CopilotLanguageModelsMa @inject(LanguageModelRegistry) protected readonly languageModelRegistry: LanguageModelRegistry; + @inject(CopilotLanguageModelFactory) + protected readonly copilotLanguageModelFactory: CopilotLanguageModelFactory; + @inject(CopilotAuthServiceImpl) protected readonly authService: CopilotAuthServiceImpl; @@ -84,17 +87,17 @@ export class CopilotLanguageModelsManagerImpl implements CopilotLanguageModelsMa }); } else { this.languageModelRegistry.addLanguageModels([ - new CopilotLanguageModel( - modelDescription.id, - modelDescription.model, + this.copilotLanguageModelFactory({ + id: modelDescription.id, + model: modelDescription.model, status, - modelDescription.enableStreaming, - modelDescription.supportsStructuredOutput, - modelDescription.maxRetries, - () => this.authService.getAccessToken(), - () => this.enterpriseUrl, - () => this.oauthConfig.userAgent - ) + enableStreaming: modelDescription.enableStreaming, + supportsStructuredOutput: modelDescription.supportsStructuredOutput, + maxRetries: modelDescription.maxRetries, + accessTokenProvider: () => this.authService.getAccessToken(), + enterpriseUrlProvider: () => this.enterpriseUrl, + userAgentProvider: () => this.oauthConfig.userAgent + }) ]); } } diff --git a/packages/ai-core/src/node/ai-core-backend-module.ts b/packages/ai-core/src/node/ai-core-backend-module.ts index 9f056d168ec87..5d0ecc25db8c7 100644 --- a/packages/ai-core/src/node/ai-core-backend-module.ts +++ b/packages/ai-core/src/node/ai-core-backend-module.ts @@ -54,6 +54,8 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac bind(BackendLanguageModelRegistryImpl).toSelf().inSingletonScope(); bind(LanguageModelRegistry).toService(BackendLanguageModelRegistryImpl); + bind(ToolCallExecutor).toSelf().inSingletonScope(); + bind(TokenUsageService).to(TokenUsageServiceImpl).inSingletonScope(); bind(ConnectionHandler) @@ -113,10 +115,5 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: AgentSettingsPreferenceSchema }); bindAICorePreferences(bind); - // Bound on the root container because it is injected by extant root-level dependents (notably - // OpenAiResponseApiUtils). This is safe only because the service is stateless. A pending refactoring - // should relocate it, together with those dependents, into the connection-scoped container so that a - // substituted, potentially stateful executor cannot leak state across frontend connections. - bind(ToolCallExecutor).toSelf().inSingletonScope(); bind(ConnectionContainerModule).toConstantValue(aiCoreConnectionModule); }); diff --git a/packages/ai-openai/src/node/openai-backend-module.ts b/packages/ai-openai/src/node/openai-backend-module.ts index 5e7cedc189f9b..5dfbb17733abb 100644 --- a/packages/ai-openai/src/node/openai-backend-module.ts +++ b/packages/ai-openai/src/node/openai-backend-module.ts @@ -15,20 +15,43 @@ // ***************************************************************************** import { ContainerModule } from '@theia/core/shared/inversify'; +import { ToolCallExecutor } from '@theia/ai-core'; import { OPENAI_LANGUAGE_MODELS_MANAGER_PATH, OpenAiLanguageModelsManager } from '../common/openai-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { OpenAiLanguageModelsManagerImpl } from './openai-language-models-manager-impl'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; -import { OpenAiModelUtils } from './openai-language-model'; +import { OpenAiLanguageModelFactory, OpenAiModel, OpenAiModelParams, OpenAiModelUtils } from './openai-language-model'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; import { OpenAiPreferencesSchema } from '../common/openai-preferences'; -export const OpenAiModelFactory = Symbol('OpenAiModelFactory'); - // We use a connection module to handle AI services separately for each frontend. const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(OpenAiLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(OpenAiLanguageModelsManager).toService(OpenAiLanguageModelsManagerImpl); + // Connection-scoped because it injects the connection-scoped ToolCallExecutor. + bind(OpenAiResponseApiUtils).toSelf().inSingletonScope(); + bind(OpenAiLanguageModelFactory).toFactory( + ({ container }) => params => new OpenAiModel( + params.id, + params.model, + params.status, + params.enableStreaming, + params.apiKey, + params.apiVersion, + params.supportsStructuredOutput, + params.url, + params.deployment, + container.get(OpenAiModelUtils), + container.get(OpenAiResponseApiUtils), + params.developerMessageSettings, + params.maxRetries, + params.useResponseApi, + params.proxy, + params.reasoningSupport, + params.maxInputTokens, + container.get(ToolCallExecutor) + ) + ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(OPENAI_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(OpenAiLanguageModelsManager)) ).inSingletonScope(); @@ -37,6 +60,5 @@ const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBac export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: OpenAiPreferencesSchema }); bind(OpenAiModelUtils).toSelf().inSingletonScope(); - bind(OpenAiResponseApiUtils).toSelf().inSingletonScope(); bind(ConnectionContainerModule).toConstantValue(openAiConnectionModule); }); diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts new file mode 100644 index 0000000000000..d55d6d72be44c --- /dev/null +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -0,0 +1,299 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { expect } from 'chai'; +import * as sinon from 'sinon'; +import { CancellationTokenSource } from '@theia/core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; +import { + createToolCallError, + isTextResponsePart, + isToolCallResponsePart, + isUsageResponsePart, + LanguageModelStreamResponsePart, + PreparedToolCall, + ToolCallExecutionOptions, + ToolCallExecutionResult, + ToolCallExecutor, + ToolRequest, + UserRequest +} from '@theia/ai-core'; +import { ChatCompletionStreamingAsyncIterator, ChatCompletionToolLoopOptions } from './openai-chat-completion-stream'; + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +/** A fake of the OpenAI chat-completion stream: async-iterable over chunks and an abortable controller. */ +class FakeStream { + readonly controller = { abort: sinon.spy() } as unknown as AbortController; + constructor(protected readonly chunks: any[], protected readonly gate?: Promise) { } + async *[Symbol.asyncIterator](): AsyncIterator { + if (this.gate) { + await this.gate; + } + for (const chunk of this.chunks) { + yield chunk; + } + } +} + +function textChunk(content: string): any { + return { choices: [{ delta: { content } }] }; +} + +function toolChunk(index: number, id: string, name: string, args: string): any { + return { choices: [{ delta: { tool_calls: [{ index, id, function: { name, arguments: args } }] } }] }; +} + +function usageChunk(inputTokens: number, outputTokens: number): any { + return { choices: [{ delta: {} }], usage: { prompt_tokens: inputTokens, completion_tokens: outputTokens } }; +} + +interface FakeOpenAi { + readonly calls: any[]; + readonly chat: { completions: { create: (body: any, options?: any) => Promise } }; +} + +/** Returns the queued streams in order; records the request bodies passed to `create`. */ +function fakeOpenAi(streams: FakeStream[]): FakeOpenAi { + const calls: any[] = []; + return { + calls, + chat: { + completions: { + create: async (body: any) => { + calls.push(body); + return streams.shift()!; + } + } + } + }; +} + +function toolRequest(name: string, handler: ToolRequest['handler']): ToolRequest { + return { id: name, name, parameters: { type: 'object', properties: {} }, handler }; +} + +function makeIterator(openai: FakeOpenAi, overrides: Partial = {}): ChatCompletionStreamingAsyncIterator { + const defaultRequest: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }] + }; + return new ChatCompletionStreamingAsyncIterator({ + openai: openai as any, + model: 'gpt-test', + request: overrides.request ?? defaultRequest, + messages: overrides.messages ?? [{ role: 'user', content: 'hi' }], + settings: overrides.settings ?? {}, + tools: overrides.tools ?? [{ type: 'function', function: { name: 'a' } } as any], + maxChatCompletions: overrides.maxChatCompletions ?? 100, + maxRetries: overrides.maxRetries ?? 0, + toolCallExecutor: overrides.toolCallExecutor ?? new ToolCallExecutor(), + cancellationToken: overrides.cancellationToken + }); +} + +async function drain(iterator: AsyncIterableIterator): Promise { + const parts: LanguageModelStreamResponsePart[] = []; + for await (const part of iterator) { + parts.push(part); + } + return parts; +} + +const flush = (): Promise => new Promise(resolve => setImmediate(resolve)); + +/** Captures the batches of tool calls passed to the executor, to assert single-turn batching. */ +class RecordingExecutor extends ToolCallExecutor { + readonly batches: PreparedToolCall[][] = []; + override executeToolCalls( + toolCalls: readonly PreparedToolCall[], + tools: readonly ToolRequest[] | undefined, + options?: ToolCallExecutionOptions + ): Promise { + this.batches.push([...toolCalls]); + return super.executeToolCalls(toolCalls, tools, options); + } +} + +describe('ChatCompletionStreamingAsyncIterator', () => { + + it('streams a plain text turn with no tool calls', async () => { + const openai = fakeOpenAi([new FakeStream([textChunk('Hello '), textChunk('world')])]); + const parts = await drain(makeIterator(openai)); + + expect(parts.filter(isTextResponsePart).map(p => p.content)).to.deep.equal(['Hello ', 'world']); + expect(openai.calls).to.have.lengthOf(1); + }); + + it('emits token usage parts', async () => { + const openai = fakeOpenAi([new FakeStream([textChunk('hi'), usageChunk(10, 4)])]); + const parts = await drain(makeIterator(openai)); + + expect(parts.filter(isUsageResponsePart)).to.deep.equal([{ input_tokens: 10, output_tokens: 4 }]); + }); + + it('executes the tool calls of a single turn concurrently and in one batch', async () => { + // `a` only completes once `b` has started: a sequential loop would deadlock here. + const bStarted = new Deferred(); + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [ + toolRequest('a', async () => { await bStarted.promise; return 'a-result'; }), + toolRequest('b', async () => { bStarted.resolve(); return 'b-result'; }) + ] + }; + const executor = new RecordingExecutor(); + const openai = fakeOpenAi([ + new FakeStream([toolChunk(0, 'call-a', 'a', '{}'), toolChunk(1, 'call-b', 'b', '{}')]), + new FakeStream([textChunk('done')]) + ]); + + const parts = await drain(makeIterator(openai, { request, toolCallExecutor: executor })); + + // Both tool calls were handed to the executor together (single turn => single batch). + expect(executor.batches).to.have.lengthOf(1); + expect(executor.batches[0].map(c => c.name)).to.deep.equal(['a', 'b']); + + const finished = parts.filter(isToolCallResponsePart).flatMap(p => p.tool_calls).filter(c => c.finished); + expect(finished.map(c => c.result)).to.have.members(['a-result', 'b-result']); + expect(parts.filter(isTextResponsePart).map(p => p.content)).to.deep.equal(['done']); + expect(openai.calls).to.have.lengthOf(2); + }); + + it('emits open / arguments-delta / finished parts for a tool call', async () => { + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => 'a-result')] + }; + const openai = fakeOpenAi([ + new FakeStream([toolChunk(0, 'call-a', 'a', '{"x":1}')]), + new FakeStream([textChunk('done')]) + ]); + + const parts = await drain(makeIterator(openai, { request })); + const toolParts = parts.filter(isToolCallResponsePart).flatMap(p => p.tool_calls); + + const open = toolParts.find(c => c.finished === false); + const delta = toolParts.find(c => c.argumentsDelta); + const finished = toolParts.find(c => c.finished); + expect(open?.function?.name).to.equal('a'); + expect(delta?.function?.arguments).to.equal('{"x":1}'); + expect(finished?.result).to.equal('a-result'); + }); + + it('threads the assistant tool_calls and a matching tool message into the next turn', async () => { + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => 'a-result')] + }; + const openai = fakeOpenAi([ + new FakeStream([toolChunk(0, 'call-a', 'a', '{}')]), + new FakeStream([textChunk('done')]) + ]); + + await drain(makeIterator(openai, { request })); + + const secondTurnMessages = openai.calls[1].messages; + const assistant = secondTurnMessages[secondTurnMessages.length - 2]; + const toolMessage = secondTurnMessages[secondTurnMessages.length - 1]; + expect(assistant.role).to.equal('assistant'); + expect(assistant.tool_calls[0].id).to.equal('call-a'); + expect(assistant.tool_calls[0].function.name).to.equal('a'); + expect(toolMessage.role).to.equal('tool'); + expect(toolMessage.tool_call_id).to.equal('call-a'); + expect(toolMessage.content).to.equal('a-result'); + }); + + it('surfaces a tool error and still appends a tool message so the model can continue', async () => { + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => { throw new Error('boom'); })] + }; + const openai = fakeOpenAi([ + new FakeStream([toolChunk(0, 'call-a', 'a', '{}')]), + new FakeStream([textChunk('recovered')]) + ]); + + const parts = await drain(makeIterator(openai, { request })); + + const finished = parts.filter(isToolCallResponsePart).flatMap(p => p.tool_calls).find(c => c.finished); + expect(JSON.stringify(finished?.result)).to.equal(JSON.stringify(createToolCallError('boom'))); + // The tool message must still be present for the follow-up turn. + const secondTurnMessages = openai.calls[1].messages; + expect(secondTurnMessages[secondTurnMessages.length - 1].role).to.equal('tool'); + expect(parts.filter(isTextResponsePart).map(p => p.content)).to.deep.equal(['recovered']); + }); + + it('stops after maxChatCompletions turns even if the model keeps requesting tools', async () => { + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => 'a-result')] + }; + // Every turn returns another tool call, so only the maxChatCompletions guard ends the loop. + const openai: FakeOpenAi = { + calls: [], + chat: { + completions: { + create: async (body: any) => { + openai.calls.push(body); + return new FakeStream([toolChunk(0, `call-${openai.calls.length}`, 'a', '{}')]); + } + } + } + }; + + await drain(makeIterator(openai, { request, maxChatCompletions: 3 })); + + expect(openai.calls).to.have.lengthOf(3); + }); + + it('does not start a request when already cancelled', async () => { + const source = new CancellationTokenSource(); + source.cancel(); + const openai = fakeOpenAi([new FakeStream([textChunk('unused')])]); + + const parts = await drain(makeIterator(openai, { cancellationToken: source.token })); + + expect(openai.calls).to.have.lengthOf(0); + expect(parts).to.deep.equal([]); + }); + + it('aborts the in-flight stream when cancelled mid-turn', async () => { + const gate = new Deferred(); + const stream = new FakeStream([textChunk('partial')], gate.promise); + const openai = fakeOpenAi([stream]); + const source = new CancellationTokenSource(); + + const drained = drain(makeIterator(openai, { cancellationToken: source.token })).catch(error => error); + await flush(); // allow create() to resolve and the stream to be awaited + source.cancel(); + + expect((stream.controller.abort as sinon.SinonSpy).called).to.equal(true); + gate.resolve(); + await drained; + }); +}); diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.ts new file mode 100644 index 0000000000000..529c5a651aac2 --- /dev/null +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.ts @@ -0,0 +1,279 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { LanguageModelStreamResponsePart, ToolCallExecutor, ToolCallResult, UserRequest } from '@theia/ai-core'; +import { CancellationError, CancellationToken, Disposable, DisposableCollection } from '@theia/core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; +import { OpenAI } from 'openai'; +import { + ChatCompletionAssistantMessageParam, + ChatCompletionChunk, + ChatCompletionCreateParamsStreaming, + ChatCompletionMessageParam, + ChatCompletionTool +} from 'openai/resources'; + +type IterResult = IteratorResult; + +/** A chat completion stream as returned by `chat.completions.create({ stream: true })`. */ +type ChatCompletionChunkStream = AsyncIterable & { controller: AbortController }; + +/** Accumulator for a single tool call streamed across multiple chunk deltas (keyed by `index`). */ +interface CollectedToolCall { + index: number; + id: string; + name: string; + arguments: string; + /** Whether the `finished: false` "open" part has already been emitted for this call. */ + opened: boolean; +} + +export interface ChatCompletionToolLoopOptions { + readonly openai: OpenAI; + readonly model: string; + /** The originating request; its `tools` carry the actual handlers to invoke. */ + readonly request: UserRequest; + /** The already-processed messages that seed the conversation. */ + readonly messages: ChatCompletionMessageParam[]; + /** Additional request settings (may include `stream_options`, temperature, etc.). */ + readonly settings: Record; + readonly tools: ChatCompletionTool[]; + /** Maximum number of chat completions (turns); each tool round counts as one. */ + readonly maxChatCompletions: number; + readonly maxRetries: number; + readonly toolCallExecutor: ToolCallExecutor; + readonly cancellationToken?: CancellationToken; +} + +/** + * Drives a multi-turn OpenAI Chat Completions conversation with tool calling, executing the tool + * calls of each turn **concurrently** via {@link ToolCallExecutor}. + * + * This replaces the OpenAI SDK's `chat.completions.runTools(...)` runner, whose built-in tool loop + * executes tool calls strictly sequentially. By driving the raw chunk stream + * (`chat.completions.create({ stream: true })`) ourselves, multiple tool calls emitted in a single + * model turn (e.g. parallel agent delegations) run in parallel. + */ +export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterator, Disposable { + protected readonly requestQueue = new Array>(); + protected readonly messageCache = new Array(); + protected done = false; + protected terminalError: Error | undefined = undefined; + protected readonly toDispose = new DisposableCollection(); + + protected readonly messages: ChatCompletionMessageParam[]; + protected iteration = 0; + protected currentStream?: ChatCompletionChunkStream; + + constructor(protected readonly options: ChatCompletionToolLoopOptions) { + this.messages = [...options.messages]; + if (options.cancellationToken) { + this.toDispose.push(options.cancellationToken.onCancellationRequested(() => this.currentStream?.controller.abort())); + } + this.startIteration(); + } + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise { + if (this.messageCache.length && this.requestQueue.length) { + throw new Error('Assertion error: cache and queue should not both be populated.'); + } + // Deliver all the messages we got, even if we've since terminated. + if (this.messageCache.length) { + return Promise.resolve({ done: false, value: this.messageCache.shift()! }); + } else if (this.terminalError) { + return Promise.reject(this.terminalError); + } else if (this.done) { + return Promise.resolve({ done: true, value: undefined }); + } else { + const toQueue = new Deferred(); + this.requestQueue.push(toQueue); + return toQueue.promise; + } + } + + protected get cancellationRequested(): boolean { + return !!this.options.cancellationToken?.isCancellationRequested; + } + + protected async startIteration(): Promise { + try { + while (this.iteration < this.options.maxChatCompletions && !this.cancellationRequested) { + const { assistantText, toolCalls } = await this.processStream(); + if (toolCalls.length === 0) { + // No tool calls: the conversation is complete. + this.dispose(); + return; + } + await this.executeAndAppendToolCalls(assistantText, toolCalls); + this.iteration++; + } + // Reached the maximum number of completions (or was cancelled). + this.dispose(); + } catch (error) { + if (this.cancellationRequested) { + this.terminalError = new CancellationError(); + } else { + console.error('Error in OpenAI chat completion stream:', error); + this.terminalError = error instanceof Error ? error : new Error(String(error)); + } + this.dispose(); + } + } + + /** Streams a single completion, emitting text/tool-call parts and collecting the turn's tool calls. */ + protected async processStream(): Promise<{ assistantText: string; toolCalls: CollectedToolCall[] }> { + const collected = new Map(); + let assistantText = ''; + + const body = { + model: this.options.model, + messages: this.messages, + stream: true, + tools: this.options.tools, + tool_choice: 'auto', + ...this.options.settings + } as ChatCompletionCreateParamsStreaming; + const stream = await this.options.openai.chat.completions.create(body, { maxRetries: this.options.maxRetries }) as unknown as ChatCompletionChunkStream; + this.currentStream = stream; + + for await (const chunk of stream) { + if (this.cancellationRequested) { + break; + } + if (chunk.usage) { + const inputTokens = chunk.usage.prompt_tokens || 0; + const outputTokens = chunk.usage.completion_tokens || 0; + if (inputTokens > 0 || outputTokens > 0) { + this.handleIncoming({ input_tokens: inputTokens, output_tokens: outputTokens }); + } + } + const delta = chunk.choices[0]?.delta; + if (!delta) { + continue; + } + // Some providers stream reasoning tokens on the delta; surface them as thoughts. + const reasoning = (delta as { reasoning?: string; reasoning_content?: string }).reasoning + ?? (delta as { reasoning_content?: string }).reasoning_content; + if (reasoning) { + this.handleIncoming({ thought: reasoning, signature: '' }); + } + if (delta.content) { + assistantText += delta.content; + this.handleIncoming({ content: delta.content }); + } + for (const toolCallDelta of delta.tool_calls ?? []) { + this.collectToolCallDelta(collected, toolCallDelta); + } + } + + return { assistantText, toolCalls: [...collected.values()].sort((a, b) => a.index - b.index) }; + } + + protected collectToolCallDelta(collected: Map, toolCallDelta: ChatCompletionChunk.Choice.Delta.ToolCall): void { + let slot = collected.get(toolCallDelta.index); + if (!slot) { + slot = { index: toolCallDelta.index, id: toolCallDelta.id ?? '', name: '', arguments: '', opened: false }; + collected.set(toolCallDelta.index, slot); + } + if (toolCallDelta.id && !slot.id) { + slot.id = toolCallDelta.id; + } + if (toolCallDelta.function?.name) { + slot.name += toolCallDelta.function.name; + } + // Open the tool call (emit a `finished: false` part) as soon as we know its ID. + if (!slot.opened && slot.id) { + slot.opened = true; + this.handleIncoming({ tool_calls: [{ id: slot.id, finished: false, function: { name: slot.name, arguments: '' } }] }); + } + if (toolCallDelta.function?.arguments) { + slot.arguments += toolCallDelta.function.arguments; + if (slot.opened) { + this.handleIncoming({ tool_calls: [{ id: slot.id, argumentsDelta: true, function: { arguments: toolCallDelta.function.arguments } }] }); + } + } + } + + protected async executeAndAppendToolCalls(assistantText: string, toolCalls: CollectedToolCall[]): Promise { + // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. + const results = await this.options.toolCallExecutor.executeToolCalls( + toolCalls.map(toolCall => ({ id: toolCall.id, name: toolCall.name, arguments: toolCall.arguments || '{}' })), + this.options.request.tools, + { cancellationToken: this.options.cancellationToken } + ); + + // Emit the finished tool calls (in stable input order). + for (const result of results) { + this.handleIncoming({ + tool_calls: [{ id: result.id, finished: true, function: { name: result.name, arguments: result.arguments }, result: result.result }] + }); + } + + // Append the assistant turn and the tool results so the next completion sees them. + // Every assistant tool_call must be paired with a tool message that has the same ID. + const assistantMessage: ChatCompletionAssistantMessageParam = { + role: 'assistant', + tool_calls: toolCalls.map(toolCall => ({ + id: toolCall.id, + type: 'function', + function: { name: toolCall.name, arguments: toolCall.arguments || '{}' } + })) + }; + if (assistantText) { + assistantMessage.content = assistantText; + } + this.messages.push(assistantMessage); + for (const result of results) { + this.messages.push({ role: 'tool', tool_call_id: result.id, content: this.formatToolResult(result.result) }); + } + } + + protected formatToolResult(result: ToolCallResult): string { + if (result === undefined) { + return ''; + } + return typeof result === 'string' ? result : JSON.stringify(result); + } + + protected handleIncoming(message: LanguageModelStreamResponsePart): void { + if (this.messageCache.length && this.requestQueue.length) { + throw new Error('Assertion error: cache and queue should not both be populated.'); + } + if (this.requestQueue.length) { + this.requestQueue.shift()!.resolve({ done: false, value: message }); + } else { + this.messageCache.push(message); + } + } + + dispose(): void { + this.done = true; + this.toDispose.dispose(); + // No more messages will arrive; resolve or reject any outstanding requests. + if (this.terminalError) { + this.requestQueue.forEach(request => request.reject(this.terminalError)); + } else { + this.requestQueue.forEach(request => request.resolve({ done: true, value: undefined })); + } + // Leave the message cache intact: if it is populated the request queue was empty, and we + // still want to deliver those messages when asked. + this.requestQueue.length = 0; + } +} diff --git a/packages/ai-openai/src/node/openai-language-model.spec.ts b/packages/ai-openai/src/node/openai-language-model.spec.ts index 7a7c303afde74..dedca6267ab75 100644 --- a/packages/ai-openai/src/node/openai-language-model.spec.ts +++ b/packages/ai-openai/src/node/openai-language-model.spec.ts @@ -33,6 +33,9 @@ class TestableOpenAiModel extends OpenAiModel { public callGetSettings(request: LanguageModelRequest, forResponseApi: boolean = false): Record { return this.getSettings(request, forResponseApi); } + public callCreateTools(request: LanguageModelRequest): unknown { + return this.createTools(request); + } } function createModel(modelId: string, reasoningSupport?: ReasoningSupport): TestableOpenAiModel { @@ -101,4 +104,20 @@ describe('OpenAiModel reasoning translation', () => { expect(result.reasoning_effort).to.equal(undefined); }); }); + + describe('createTools', () => { + it('produces plain function tool definitions without an embedded handler function', () => { + const model = createModel('gpt-4o', undefined); + const tools = model.callCreateTools({ + messages: [], + tools: [{ id: 't', name: 't', parameters: { type: 'object', properties: {} }, handler: async () => 'x' }] + }) as Array<{ type: string; function: Record }>; + + expect(tools).to.have.lengthOf(1); + expect(tools[0].type).to.equal('function'); + expect(tools[0].function.name).to.equal('t'); + // The SDK runTools() runner is no longer used, so no executable function is embedded. + expect('function' in tools[0].function).to.equal(false); + }); + }); }); diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index a27f507597e9c..b215c18bc2819 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -24,15 +24,15 @@ import { UserRequest, ImageContent, LanguageModelStatus, - ReasoningSupport + ReasoningSupport, + ToolCallExecutor } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; import { injectable } from '@theia/core/shared/inversify'; import { OpenAI, AzureOpenAI } from 'openai'; -import { ChatCompletionStream } from 'openai/lib/ChatCompletionStream'; -import { RunnableToolFunctionWithoutParse } from 'openai/lib/RunnableFunction'; -import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam } from 'openai/resources'; +import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; import { StreamingAsyncIterator } from './openai-streaming-iterator'; +import { ChatCompletionStreamingAsyncIterator } from './openai-chat-completion-stream'; import { OPENAI_PROVIDER_ID } from '../common'; import type { FinalRequestOptions } from 'openai/internal/request-options'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; @@ -67,6 +67,28 @@ export const OpenAiModelIdentifier = Symbol('OpenAiModelIdentifier'); export type DeveloperMessageSettings = 'user' | 'system' | 'developer' | 'mergeWithFollowingUserMessage' | 'skip'; +/** Parameters for constructing an {@link OpenAiModel}. */ +export interface OpenAiModelParams { + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + apiKey: () => string | undefined; + apiVersion: () => string | undefined; + supportsStructuredOutput: boolean; + url: string | undefined; + deployment: string | undefined; + developerMessageSettings?: DeveloperMessageSettings; + maxRetries?: number; + useResponseApi?: boolean; + proxy?: string; + reasoningSupport?: ReasoningSupport; + maxInputTokens?: number; +} + +export const OpenAiLanguageModelFactory = Symbol('OpenAiLanguageModelFactory'); +export type OpenAiLanguageModelFactory = (params: OpenAiModelParams) => OpenAiModel; + export class OpenAiModel implements LanguageModel { /** @@ -107,7 +129,8 @@ export class OpenAiModel implements LanguageModel { public useResponseApi: boolean = false, public proxy?: string, public reasoningSupport?: ReasoningSupport, - public maxInputTokens?: number + public maxInputTokens?: number, + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() ) { } /** Reasoning-level translation lives in {@link openAiReasoningFor}. */ @@ -144,29 +167,32 @@ export class OpenAiModel implements LanguageModel { if (cancellationToken?.isCancellationRequested) { return { text: '' }; } - let runner: ChatCompletionStream; const tools = this.createTools(request); if (tools) { - runner = openai.chat.completions.runTools({ - model: this.model, - messages: this.processMessages(request.messages), - stream: true, - tools: tools, - tool_choice: 'auto', - ...settings - }, { - ...this.runnerOptions, maxRetries: this.maxRetries - }); - } else { - runner = openai.chat.completions.stream({ - model: this.model, - messages: this.processMessages(request.messages), - stream: true, - ...settings - }); + // Drive the tool loop ourselves so that the tool calls of a single turn run concurrently. + return { + stream: new ChatCompletionStreamingAsyncIterator({ + openai, + model: this.model, + request, + messages: this.processMessages(request.messages), + settings, + tools, + maxChatCompletions: this.runnerOptions.maxChatCompletions ?? 100, + maxRetries: this.maxRetries, + toolCallExecutor: this.toolCallExecutor, + cancellationToken + }) + }; } + const runner = openai.chat.completions.stream({ + model: this.model, + messages: this.processMessages(request.messages), + stream: true, + ...settings + }); return { stream: new StreamingAsyncIterator(runner, cancellationToken) }; } @@ -217,16 +243,15 @@ export class OpenAiModel implements LanguageModel { }; } - protected createTools(request: LanguageModelRequest): RunnableToolFunctionWithoutParse[] | undefined { + protected createTools(request: LanguageModelRequest): ChatCompletionTool[] | undefined { return request.tools?.map(tool => ({ type: 'function', function: { name: tool.name, description: tool.description, - parameters: tool.parameters, - function: (args_string: string) => tool.handler(args_string) + parameters: tool.parameters } - } as RunnableToolFunctionWithoutParse)); + } as unknown as ChatCompletionTool)); } protected initializeOpenAi(): OpenAI { diff --git a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts index 5a8ade66e17b9..7ad3d1301e5c9 100644 --- a/packages/ai-openai/src/node/openai-language-models-manager-impl.ts +++ b/packages/ai-openai/src/node/openai-language-models-manager-impl.ts @@ -17,8 +17,7 @@ import { LanguageModelRegistry, LanguageModelStatus, ReasoningSupport } from '@theia/ai-core'; import { getProxyUrl } from '@theia/ai-core/lib/node'; import { inject, injectable } from '@theia/core/shared/inversify'; -import { DeveloperMessageSettings, OpenAiModel, OpenAiModelUtils } from './openai-language-model'; -import { OpenAiResponseApiUtils } from './openai-response-api-utils'; +import { DeveloperMessageSettings, OpenAiLanguageModelFactory, OpenAiModel } from './openai-language-model'; import { getOpenAiModelDefaults } from './openai-model-defaults'; import { OpenAiLanguageModelsManager, OpenAiModelDescription } from '../common'; @@ -33,11 +32,8 @@ interface ResolvedModelMetadata { @injectable() export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsManager { - @inject(OpenAiModelUtils) - protected readonly openAiModelUtils: OpenAiModelUtils; - - @inject(OpenAiResponseApiUtils) - protected readonly responseApiUtils: OpenAiResponseApiUtils; + @inject(OpenAiLanguageModelFactory) + protected readonly openAiLanguageModelFactory: OpenAiLanguageModelFactory; protected _apiKey: string | undefined; protected _apiVersion: string | undefined; @@ -115,25 +111,23 @@ export class OpenAiLanguageModelsManagerImpl implements OpenAiLanguageModelsMana }); } else { this.languageModelRegistry.addLanguageModels([ - new OpenAiModel( - modelDescription.id, - modelDescription.model, + this.openAiLanguageModelFactory({ + id: modelDescription.id, + model: modelDescription.model, status, - metadata.enableStreaming, - apiKeyProvider, - apiVersionProvider, - metadata.supportsStructuredOutput, - modelDescription.url, - modelDescription.deployment, - this.openAiModelUtils, - this.responseApiUtils, - metadata.developerMessageSettings, - modelDescription.maxRetries, - modelDescription.useResponseApi ?? false, - proxyUrl, - metadata.reasoningSupport, - metadata.maxInputTokens - ) + enableStreaming: metadata.enableStreaming, + apiKey: apiKeyProvider, + apiVersion: apiVersionProvider, + supportsStructuredOutput: metadata.supportsStructuredOutput, + url: modelDescription.url, + deployment: modelDescription.deployment, + developerMessageSettings: metadata.developerMessageSettings, + maxRetries: modelDescription.maxRetries, + useResponseApi: modelDescription.useResponseApi ?? false, + proxy: proxyUrl, + reasoningSupport: metadata.reasoningSupport, + maxInputTokens: metadata.maxInputTokens + }) ]); } } From a1d58e4f4fef44a1ed91bd3528a231ce3c527bb9 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 09:43:02 -0400 Subject: [PATCH 04/10] chore(ai): drop pointless comments Remove comments in the code that don't explain anything that needed explanation. Signed-off-by: Christian W. Damus --- .../src/node/anthropic-language-model.ts | 2 -- .../ai-copilot/src/node/copilot-language-model.ts | 2 -- packages/ai-core/src/common/language-model.ts | 13 ------------- .../ai-google/src/node/google-language-model.ts | 2 -- .../ai-ollama/src/node/ollama-language-model.ts | 3 --- .../src/node/openai-chat-completion-stream.ts | 1 - .../ai-openai/src/node/openai-language-model.ts | 2 -- .../ai-openai/src/node/openai-response-api-utils.ts | 3 --- 8 files changed, 28 deletions(-) diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.ts b/packages/ai-anthropic/src/node/anthropic-language-model.ts index 3067a4068f376..86eb86663ddcc 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.ts @@ -217,7 +217,6 @@ function formatToolCallResult(result: ToolCallResult): ToolResultBlockParam['con return result; } -/** Parameters for constructing an {@link AnthropicModel}. */ export interface AnthropicModelParams { id: string; model: string; @@ -400,7 +399,6 @@ export class AnthropicModel implements LanguageModel { } } if (toolCalls.length > 0) { - // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. const toolResult = await that.toolCallExecutor.executeToolCalls( toolCalls.map(tc => ({ id: tc.id, name: tc.name, arguments: tc.args.length === 0 ? '{}' : tc.args })), request.tools, diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index 2059b6153654f..839c141d6480d 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -34,7 +34,6 @@ import { ChatCompletionStreamingAsyncIterator } from '@theia/ai-openai/lib/node/ import { COPILOT_PROVIDER_ID, getCopilotApiBaseUrl } from '../common'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; -/** Parameters for constructing a {@link CopilotLanguageModel}. */ export interface CopilotLanguageModelParams { id: string; model: string; @@ -101,7 +100,6 @@ export class CopilotLanguageModel implements LanguageModel { const tools = this.createTools(request); if (tools) { - // Drive the tool loop ourselves so that the tool calls of a single turn run concurrently. return { stream: new ChatCompletionStreamingAsyncIterator({ openai, diff --git a/packages/ai-core/src/common/language-model.ts b/packages/ai-core/src/common/language-model.ts index 0681908c5698d..329cac865183b 100644 --- a/packages/ai-core/src/common/language-model.ts +++ b/packages/ai-core/src/common/language-model.ts @@ -142,19 +142,6 @@ export interface ToolRequest Promise; providerName?: string; diff --git a/packages/ai-google/src/node/google-language-model.ts b/packages/ai-google/src/node/google-language-model.ts index a48b9eee12fb7..0f490365a6a5d 100644 --- a/packages/ai-google/src/node/google-language-model.ts +++ b/packages/ai-google/src/node/google-language-model.ts @@ -136,7 +136,6 @@ function toGoogleRole(message: LanguageModelMessage): 'user' | 'model' { } } -/** Parameters for constructing a {@link GoogleModel}. */ export interface GoogleModelParams { id: string; model: string; @@ -344,7 +343,6 @@ export class GoogleModel implements LanguageModel { // Process tool calls if any exist const toolCalls = Object.values(toolCallMap); if (toolCalls.length > 0) { - // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. const toolResult = await that.toolCallExecutor.executeToolCalls( toolCalls.map(tc => ({ id: tc.id, name: tc.name, arguments: tc.args })), request.tools, diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index cf48dbcdf79ca..737961f14dfe1 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -41,7 +41,6 @@ import { ollamaThinkParamFor } from './ollama-reasoning'; export const OllamaModelIdentifier = Symbol('OllamaModelIdentifier'); -/** Parameters for constructing an {@link OllamaModel}. */ export interface OllamaModelParams { id: string; model: string; @@ -379,7 +378,6 @@ export class OllamaModel implements LanguageModel { private async processToolCalls(toolCalls: ToolCall[], chatRequest: ExtendedChatRequest): Promise { const tools: ToolWithHandler[] = chatRequest.tools ?? []; - // Adapt Ollama's ToolWithHandler to the ToolRequest shape expected by the executor. const toolRequests: ToolRequest[] = tools.map(tool => ({ id: tool.function.name ?? '', name: tool.function.name ?? '', @@ -387,7 +385,6 @@ export class OllamaModel implements LanguageModel { handler: async argString => (await tool.handler(argString)) as ToolCallResult })); - // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. const results = await this.toolCallExecutor.executeToolCalls( toolCalls.map(call => ({ id: call.id ?? call.function!.name!, name: call.function!.name!, arguments: call.function!.arguments! })), toolRequests diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.ts index 529c5a651aac2..47dffe15df074 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.ts @@ -212,7 +212,6 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat } protected async executeAndAppendToolCalls(assistantText: string, toolCalls: CollectedToolCall[]): Promise { - // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. const results = await this.options.toolCallExecutor.executeToolCalls( toolCalls.map(toolCall => ({ id: toolCall.id, name: toolCall.name, arguments: toolCall.arguments || '{}' })), this.options.request.tools, diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index b215c18bc2819..fba29bd66d7dc 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -67,7 +67,6 @@ export const OpenAiModelIdentifier = Symbol('OpenAiModelIdentifier'); export type DeveloperMessageSettings = 'user' | 'system' | 'developer' | 'mergeWithFollowingUserMessage' | 'skip'; -/** Parameters for constructing an {@link OpenAiModel}. */ export interface OpenAiModelParams { id: string; model: string; @@ -170,7 +169,6 @@ export class OpenAiModel implements LanguageModel { const tools = this.createTools(request); if (tools) { - // Drive the tool loop ourselves so that the tool calls of a single turn run concurrently. return { stream: new ChatCompletionStreamingAsyncIterator({ openai, diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index f83b434e255b9..976713b8e06f8 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -616,9 +616,6 @@ class ResponseApiToolCallIterator implements AsyncIterableIterator { - // Tool calls of a single turn are executed concurrently; see ToolCallExecutor. - // The per-call `onResult` hook below preserves the original side effects (streaming the - // finished tool-call event and recording the result/error consumed by prepareNextIteration). const pending = [...this.currentToolCalls].filter(([, toolCall]) => !toolCall.executed); await this.utils.toolCallExecutor.executeToolCalls( From 318b3adc077b3234a5146d7ae3ff33ef8dff69aa Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 09:44:00 -0400 Subject: [PATCH 05/10] refactor(ai-core): rename low-level tool-call interfaces Rename PreparedToolCall to ToolInvocation and ToolCallExecutionResult to ToolCallOutcome, as suggested in review. Signed-off-by: Christian W. Damus --- .../src/common/tool-call-execution.spec.ts | 4 ++-- .../ai-core/src/common/tool-call-execution.ts | 18 +++++++++--------- .../node/openai-chat-completion-stream.spec.ts | 10 +++++----- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/packages/ai-core/src/common/tool-call-execution.spec.ts b/packages/ai-core/src/common/tool-call-execution.spec.ts index d72f22e552095..523bfd9175228 100644 --- a/packages/ai-core/src/common/tool-call-execution.spec.ts +++ b/packages/ai-core/src/common/tool-call-execution.spec.ts @@ -24,7 +24,7 @@ import { isToolNotAvailableError, isToolCallContent } from './language-model'; -import { ToolCallExecutor, ToolCallExecutionResult } from './tool-call-execution'; +import { ToolCallExecutor, ToolCallOutcome } from './tool-call-execution'; /** Builds a minimal {@link ToolRequest} whose handler delegates to `handler`. */ function tool(name: string, handler: ToolRequest['handler']): ToolRequest { @@ -120,7 +120,7 @@ describe('ToolCallExecutor', () => { await executor.executeToolCalls( [{ id: '1', name: 'a', arguments: '{}' }, { id: '2', name: 'b', arguments: '{}' }], [tool('a', async () => 'a'), tool('b', async () => 'b')], - { onResult: (r: ToolCallExecutionResult) => seen.push(r.id) } + { onResult: (r: ToolCallOutcome) => seen.push(r.id) } ); expect(seen.slice().sort()).to.deep.equal(['1', '2']); diff --git a/packages/ai-core/src/common/tool-call-execution.ts b/packages/ai-core/src/common/tool-call-execution.ts index 25c4557fc7b6b..dcc7a7aa6d2f1 100644 --- a/packages/ai-core/src/common/tool-call-execution.ts +++ b/packages/ai-core/src/common/tool-call-execution.ts @@ -27,7 +27,7 @@ import { * A single tool call collected from one model response/turn, in a provider-neutral shape. * Each language model normalizes its own representation into this shape before execution. */ -export interface PreparedToolCall { +export interface ToolInvocation { /** The tool call ID assigned by the model; forwarded into the {@link ToolInvocationContext}. */ readonly id: string; /** The name used to match against {@link ToolRequest.name}. */ @@ -40,7 +40,7 @@ export interface PreparedToolCall { * The normalized outcome of executing one tool call. Returned in the same order as the * input array, regardless of the order in which the calls actually completed. */ -export interface ToolCallExecutionResult { +export interface ToolCallOutcome { readonly id: string; readonly name: string; readonly arguments: string; @@ -51,7 +51,7 @@ export interface ToolCallExecutionResult { readonly result: ToolCallResult; /** The original error if the handler threw; `undefined` for success and for tool-not-found. */ readonly error?: Error; - /** `true` when no {@link ToolRequest} matched {@link PreparedToolCall.name}. */ + /** `true` when no {@link ToolRequest} matched {@link ToolInvocation.name}. */ readonly notFound: boolean; } @@ -65,7 +65,7 @@ export interface ToolCallExecutionOptions { * order. Side effects that must be ordered should instead be performed by the caller over * the returned (input-ordered) array of execution results. */ - readonly onResult?: (result: ToolCallExecutionResult) => void; + readonly onResult?: (result: ToolCallOutcome) => void; /** Optional cancellation token forwarded into each {@link ToolInvocationContext}. */ readonly cancellationToken?: CancellationToken; } @@ -99,10 +99,10 @@ export class ToolCallExecutor { * @param options optional per-call hook and cancellation token */ async executeToolCalls( - toolCalls: readonly PreparedToolCall[], + toolCalls: readonly ToolInvocation[], tools: readonly ToolRequest[] | undefined, options: ToolCallExecutionOptions = {} - ): Promise { + ): Promise { return Promise.all(toolCalls.map(toolCall => this.executeToolCall(toolCall, tools, options))); } @@ -111,13 +111,13 @@ export class ToolCallExecutor { * Subclasses may override this to customize per-call behavior. */ protected async executeToolCall( - toolCall: PreparedToolCall, + toolCall: ToolInvocation, tools: readonly ToolRequest[] | undefined, options: ToolCallExecutionOptions - ): Promise { + ): Promise { const { id, name, arguments: args } = toolCall; const tool = tools?.find(candidate => candidate.name === name); - let outcome: ToolCallExecutionResult; + let outcome: ToolCallOutcome; if (!tool) { outcome = { id, name, arguments: args, notFound: true, diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts index d55d6d72be44c..fcb490317b6d2 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -24,9 +24,9 @@ import { isToolCallResponsePart, isUsageResponsePart, LanguageModelStreamResponsePart, - PreparedToolCall, + ToolInvocation, ToolCallExecutionOptions, - ToolCallExecutionResult, + ToolCallOutcome, ToolCallExecutor, ToolRequest, UserRequest @@ -118,12 +118,12 @@ const flush = (): Promise => new Promise(resolve => setImmediate(resolve)) /** Captures the batches of tool calls passed to the executor, to assert single-turn batching. */ class RecordingExecutor extends ToolCallExecutor { - readonly batches: PreparedToolCall[][] = []; + readonly batches: ToolInvocation[][] = []; override executeToolCalls( - toolCalls: readonly PreparedToolCall[], + toolCalls: readonly ToolInvocation[], tools: readonly ToolRequest[] | undefined, options?: ToolCallExecutionOptions - ): Promise { + ): Promise { this.batches.push([...toolCalls]); return super.executeToolCalls(toolCalls, tools, options); } From 70753c57ff75a793b5dbeba0b3d593f57033f624 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 09:49:58 -0400 Subject: [PATCH 06/10] refactor(ai-core): refactor ToolCallExecutor as an interface Split ToolCallExecutor into a symbol & interface for substitutability. Signed-off-by: Christian W. Damus --- .../src/node/anthropic-language-model.ts | 3 ++- .../src/node/copilot-language-model.ts | 3 ++- .../src/common/tool-call-execution.spec.ts | 4 ++-- .../ai-core/src/common/tool-call-execution.ts | 20 ++++++++++++------- .../src/node/ai-core-backend-module.ts | 6 ++++-- .../src/node/google-language-model.ts | 3 ++- .../src/node/ollama-language-model.ts | 3 ++- .../openai-chat-completion-stream.spec.ts | 6 +++--- .../src/node/openai-language-model.ts | 5 +++-- .../node/openai-response-api-utils.spec.ts | 4 +++- .../src/node/openai-response-api-utils.ts | 3 +-- 11 files changed, 37 insertions(+), 23 deletions(-) diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.ts b/packages/ai-anthropic/src/node/anthropic-language-model.ts index 86eb86663ddcc..b5bbc878966ed 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.ts @@ -29,6 +29,7 @@ import { ReasoningSupport, ToolCallResult, ToolCallExecutor, + ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken, isArray } from '@theia/core'; @@ -258,7 +259,7 @@ export class AnthropicModel implements LanguageModel { public reasoningApi?: ReasoningApi, public supportsXHighEffort?: boolean, public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() ) { } protected getSettings(request: LanguageModelRequest): Readonly> { diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index 839c141d6480d..e19fd5fb35b4f 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -24,6 +24,7 @@ import { LanguageModelStatus, LanguageModelTextResponse, ToolCallExecutor, + ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; @@ -69,7 +70,7 @@ export class CopilotLanguageModel implements LanguageModel { protected readonly accessTokenProvider: () => Promise, protected readonly enterpriseUrlProvider: () => string | undefined, protected readonly userAgentProvider: () => string, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() ) { } protected getSettings(request: LanguageModelRequest): Record { diff --git a/packages/ai-core/src/common/tool-call-execution.spec.ts b/packages/ai-core/src/common/tool-call-execution.spec.ts index 523bfd9175228..de10f7db651e6 100644 --- a/packages/ai-core/src/common/tool-call-execution.spec.ts +++ b/packages/ai-core/src/common/tool-call-execution.spec.ts @@ -24,7 +24,7 @@ import { isToolNotAvailableError, isToolCallContent } from './language-model'; -import { ToolCallExecutor, ToolCallOutcome } from './tool-call-execution'; +import { ToolCallExecutor, ToolCallExecutorImpl, ToolCallOutcome } from './tool-call-execution'; /** Builds a minimal {@link ToolRequest} whose handler delegates to `handler`. */ function tool(name: string, handler: ToolRequest['handler']): ToolRequest { @@ -35,7 +35,7 @@ describe('ToolCallExecutor', () => { let executor: ToolCallExecutor; beforeEach(() => { - executor = new ToolCallExecutor(); + executor = new ToolCallExecutorImpl(); }); it('executes the tool calls of a turn concurrently (not sequentially)', async () => { diff --git a/packages/ai-core/src/common/tool-call-execution.ts b/packages/ai-core/src/common/tool-call-execution.ts index dcc7a7aa6d2f1..c0386d12bf9d8 100644 --- a/packages/ai-core/src/common/tool-call-execution.ts +++ b/packages/ai-core/src/common/tool-call-execution.ts @@ -88,9 +88,7 @@ export interface ToolCallExecutionOptions { * - The overall tool execution promise never rejects but returns a result that may * include errors from failed tool calls */ -@injectable() -export class ToolCallExecutor { - +export interface ToolCallExecutor { /** * Executes all `toolCalls` concurrently and returns their outcomes in input order. * @@ -98,6 +96,18 @@ export class ToolCallExecutor { * @param tools the tools available for this request (typically `request.tools`) * @param options optional per-call hook and cancellation token */ + executeToolCalls( + toolCalls: readonly ToolInvocation[], + tools: readonly ToolRequest[] | undefined, + options?: ToolCallExecutionOptions + ): Promise; +} + +export const ToolCallExecutor = Symbol('ToolCallExecutor'); + +@injectable() +export class ToolCallExecutorImpl implements ToolCallExecutor { + async executeToolCalls( toolCalls: readonly ToolInvocation[], tools: readonly ToolRequest[] | undefined, @@ -106,10 +116,6 @@ export class ToolCallExecutor { return Promise.all(toolCalls.map(toolCall => this.executeToolCall(toolCall, tools, options))); } - /** - * Executes a single tool call, applying the uniform error handling described on the class. - * Subclasses may override this to customize per-call behavior. - */ protected async executeToolCall( toolCall: ToolInvocation, tools: readonly ToolRequest[] | undefined, diff --git a/packages/ai-core/src/node/ai-core-backend-module.ts b/packages/ai-core/src/node/ai-core-backend-module.ts index 5d0ecc25db8c7..f4b89360e771f 100644 --- a/packages/ai-core/src/node/ai-core-backend-module.ts +++ b/packages/ai-core/src/node/ai-core-backend-module.ts @@ -41,7 +41,8 @@ import { TokenUsageService, TokenUsageServiceClient, TOKEN_USAGE_SERVICE_PATH, - ToolCallExecutor + ToolCallExecutor, + ToolCallExecutorImpl } from '../common'; import { BackendLanguageModelRegistryImpl } from './backend-language-model-registry'; import { TokenUsageServiceImpl } from './token-usage-service-impl'; @@ -54,7 +55,8 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac bind(BackendLanguageModelRegistryImpl).toSelf().inSingletonScope(); bind(LanguageModelRegistry).toService(BackendLanguageModelRegistryImpl); - bind(ToolCallExecutor).toSelf().inSingletonScope(); + bind(ToolCallExecutorImpl).toSelf().inSingletonScope(); + bind(ToolCallExecutor).toService(ToolCallExecutorImpl); bind(TokenUsageService).to(TokenUsageServiceImpl).inSingletonScope(); diff --git a/packages/ai-google/src/node/google-language-model.ts b/packages/ai-google/src/node/google-language-model.ts index 0f490365a6a5d..21db86d20e695 100644 --- a/packages/ai-google/src/node/google-language-model.ts +++ b/packages/ai-google/src/node/google-language-model.ts @@ -27,6 +27,7 @@ import { ReasoningSupport, ToolCallResult, ToolCallExecutor, + ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; @@ -167,7 +168,7 @@ export class GoogleModel implements LanguageModel { public reasoningSupport?: ReasoningSupport, public reasoningApi?: ReasoningApi, public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() ) { } protected getSettings(request: LanguageModelRequest): Readonly> { diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index 737961f14dfe1..426e20e98bc8e 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -25,6 +25,7 @@ import { ReasoningSupport, ToolCall, ToolCallExecutor, + ToolCallExecutorImpl, ToolCallResult, ToolRequest, ToolRequestParametersProperties, @@ -76,7 +77,7 @@ export class OllamaModel implements LanguageModel { protected host: () => string | undefined, public proxy?: string, public reasoningSupport?: ReasoningSupport, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() ) { } async request(request: UserRequest, cancellationToken?: CancellationToken): Promise { diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts index fcb490317b6d2..002bc375d6697 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -27,7 +27,7 @@ import { ToolInvocation, ToolCallExecutionOptions, ToolCallOutcome, - ToolCallExecutor, + ToolCallExecutorImpl, ToolRequest, UserRequest } from '@theia/ai-core'; @@ -101,7 +101,7 @@ function makeIterator(openai: FakeOpenAi, overrides: Partial => new Promise(resolve => setImmediate(resolve)); /** Captures the batches of tool calls passed to the executor, to assert single-turn batching. */ -class RecordingExecutor extends ToolCallExecutor { +class RecordingExecutor extends ToolCallExecutorImpl { readonly batches: ToolInvocation[][] = []; override executeToolCalls( toolCalls: readonly ToolInvocation[], diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index fba29bd66d7dc..35f31dccb7477 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -25,7 +25,8 @@ import { ImageContent, LanguageModelStatus, ReasoningSupport, - ToolCallExecutor + ToolCallExecutor, + ToolCallExecutorImpl } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; import { injectable } from '@theia/core/shared/inversify'; @@ -129,7 +130,7 @@ export class OpenAiModel implements LanguageModel { public proxy?: string, public reasoningSupport?: ReasoningSupport, public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor() + protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() ) { } /** Reasoning-level translation lives in {@link openAiReasoningFor}. */ diff --git a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts index ad2cc86346716..a3b86163b81b7 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts @@ -15,7 +15,7 @@ // ***************************************************************************** import { expect } from 'chai'; -import { isToolCallResponsePart, isUsageResponsePart, LanguageModelStreamResponsePart, UserRequest } from '@theia/ai-core'; +import { isToolCallResponsePart, isUsageResponsePart, LanguageModelStreamResponsePart, ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { OpenAiModelUtils } from './openai-language-model'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; @@ -36,6 +36,7 @@ function functionCallItem(id: string, name: string, args: string): unknown { describe('OpenAiResponseApiUtils', () => { it('emits per-iteration usage for Response API tool calls instead of accumulated usage', async () => { const utils = new OpenAiResponseApiUtils(); + utils.toolCallExecutor = new ToolCallExecutorImpl(); const streams = [ [ { @@ -117,6 +118,7 @@ describe('OpenAiResponseApiUtils', () => { it('executes the tool calls of a single turn concurrently', async () => { const utils = new OpenAiResponseApiUtils(); + utils.toolCallExecutor = new ToolCallExecutorImpl(); const streams = [ [ functionCallItem('call-a', 'a', '{}'), diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index 976713b8e06f8..363f80def5e5d 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -61,9 +61,8 @@ interface ToolCall { @injectable() export class OpenAiResponseApiUtils { - // Injected when resolved via DI; the initializer keeps direct instantiation (e.g. in tests) working. @inject(ToolCallExecutor) - readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutor(); + toolCallExecutor: ToolCallExecutor; /** * Handles Response API requests with proper tool calling cycles. From f9a2064bd5f3d845bbaefa30343ce03e873ad809 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 09:50:50 -0400 Subject: [PATCH 07/10] refactor(ai-core): bind ToolCallExecutor in the root container The executor holds no per-connection state, so bind it once in the root container rather than per connection. Signed-off-by: Christian W. Damus --- CHANGELOG.md | 1 - packages/ai-core/src/node/ai-core-backend-module.ts | 5 ++--- packages/ai-openai/src/node/openai-backend-module.ts | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82adacf4cc104..3929dd9cd6199 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,6 @@ - added container parameter to DefaultDebugSessionFactory and PluginDebugSessionFactory constructors - renamed DebugSessionFactory.get to DebugSessionFactory.createSession and removed the manager parameter - [terminal] `TerminalWidget` gained a new abstract method `paste(text: string)`; downstream subclasses must implement it (consistent with `getSelection()` / `hasSelection()` added in [#17290](https://github.com/eclipse-theia/theia/pull/17290)) [#17603](https://github.com/eclipse-theia/theia/pull/17603) -- [ai-openai] `OpenAiResponseApiUtils` is now bound in the connection-scoped backend container instead of the root container, so it can no longer be injected into root-scoped services [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-openai] `OpenAiLanguageModelsManagerImpl` no longer injects `OpenAiModelUtils` or `OpenAiResponseApiUtils` (the `openAiModelUtils` and `responseApiUtils` protected fields were removed); provider models are now constructed via the injected `OpenAiLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-openai, ai-copilot] `OpenAiModel.createTools()` and `CopilotLanguageModel.createTools()` now return `ChatCompletionTool[]` instead of `RunnableToolFunctionWithoutParse[]`, because the OpenAI SDK `runTools` runner is no longer used [#17623](https://github.com/eclipse-theia/theia/pull/17623) diff --git a/packages/ai-core/src/node/ai-core-backend-module.ts b/packages/ai-core/src/node/ai-core-backend-module.ts index f4b89360e771f..d5d8a401ffc05 100644 --- a/packages/ai-core/src/node/ai-core-backend-module.ts +++ b/packages/ai-core/src/node/ai-core-backend-module.ts @@ -55,9 +55,6 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac bind(BackendLanguageModelRegistryImpl).toSelf().inSingletonScope(); bind(LanguageModelRegistry).toService(BackendLanguageModelRegistryImpl); - bind(ToolCallExecutorImpl).toSelf().inSingletonScope(); - bind(ToolCallExecutor).toService(ToolCallExecutorImpl); - bind(TokenUsageService).to(TokenUsageServiceImpl).inSingletonScope(); bind(ConnectionHandler) @@ -117,5 +114,7 @@ const aiCoreConnectionModule = ConnectionContainerModule.create(({ bind, bindBac export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: AgentSettingsPreferenceSchema }); bindAICorePreferences(bind); + bind(ToolCallExecutorImpl).toSelf().inSingletonScope(); + bind(ToolCallExecutor).toService(ToolCallExecutorImpl); bind(ConnectionContainerModule).toConstantValue(aiCoreConnectionModule); }); diff --git a/packages/ai-openai/src/node/openai-backend-module.ts b/packages/ai-openai/src/node/openai-backend-module.ts index 5dfbb17733abb..72e792e3cdb64 100644 --- a/packages/ai-openai/src/node/openai-backend-module.ts +++ b/packages/ai-openai/src/node/openai-backend-module.ts @@ -28,8 +28,6 @@ import { OpenAiPreferencesSchema } from '../common/openai-preferences'; const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(OpenAiLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(OpenAiLanguageModelsManager).toService(OpenAiLanguageModelsManagerImpl); - // Connection-scoped because it injects the connection-scoped ToolCallExecutor. - bind(OpenAiResponseApiUtils).toSelf().inSingletonScope(); bind(OpenAiLanguageModelFactory).toFactory( ({ container }) => params => new OpenAiModel( params.id, @@ -60,5 +58,6 @@ const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBac export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: OpenAiPreferencesSchema }); bind(OpenAiModelUtils).toSelf().inSingletonScope(); + bind(OpenAiResponseApiUtils).toSelf().inSingletonScope(); bind(ConnectionContainerModule).toConstantValue(openAiConnectionModule); }); From bdd40f20475378d4032b17da428d3b11f27feafa Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 09:52:09 -0400 Subject: [PATCH 08/10] refactor(ai-openai): drop the maxChatCompletions turn cap The cap existed only to bound the OpenAI SDK runTools runner, which the hand-rolled streaming tool loop replaced; the other providers run uncapped. Signed-off-by: Christian W. Damus --- CHANGELOG.md | 1 + .../src/node/copilot-language-model.ts | 6 ----- .../openai-chat-completion-stream.spec.ts | 26 ------------------- .../src/node/openai-chat-completion-stream.ts | 8 ++---- .../src/node/openai-language-model.ts | 1 - 5 files changed, 3 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3929dd9cd6199..7d3557763ca0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ - [terminal] `TerminalWidget` gained a new abstract method `paste(text: string)`; downstream subclasses must implement it (consistent with `getSelection()` / `hasSelection()` added in [#17290](https://github.com/eclipse-theia/theia/pull/17290)) [#17603](https://github.com/eclipse-theia/theia/pull/17603) - [ai-openai] `OpenAiLanguageModelsManagerImpl` no longer injects `OpenAiModelUtils` or `OpenAiResponseApiUtils` (the `openAiModelUtils` and `responseApiUtils` protected fields were removed); provider models are now constructed via the injected `OpenAiLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-openai, ai-copilot] `OpenAiModel.createTools()` and `CopilotLanguageModel.createTools()` now return `ChatCompletionTool[]` instead of `RunnableToolFunctionWithoutParse[]`, because the OpenAI SDK `runTools` runner is no longer used [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-copilot] removed the `protected runnerOptions` field from `CopilotLanguageModel`; its only purpose was the `maxChatCompletions` turn cap, which is gone now that the OpenAI SDK `runTools` runner is unused, so the tool loop runs until the model stops requesting tools. Subclasses that read or overrode `runnerOptions` must adapt. `OpenAiModel.runnerOptions` is retained, but its `maxChatCompletions` now bounds only the Response API path, not the Chat Completions tool loop [#17623](https://github.com/eclipse-theia/theia/pull/17623) ## 1.72.0 - 5/28/2026 diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index e19fd5fb35b4f..4afb356b5fd09 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -33,7 +33,6 @@ import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam, ChatCo import { StreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-streaming-iterator'; import { ChatCompletionStreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-chat-completion-stream'; import { COPILOT_PROVIDER_ID, getCopilotApiBaseUrl } from '../common'; -import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; export interface CopilotLanguageModelParams { id: string; @@ -56,10 +55,6 @@ export type CopilotLanguageModelFactory = (params: CopilotLanguageModelParams) = */ export class CopilotLanguageModel implements LanguageModel { - protected runnerOptions: RunnerOptions = { - maxChatCompletions: 100 - }; - constructor( public readonly id: string, public model: string, @@ -109,7 +104,6 @@ export class CopilotLanguageModel implements LanguageModel { messages: this.processMessages(request.messages), settings, tools, - maxChatCompletions: this.runnerOptions.maxChatCompletions ?? 100, maxRetries: this.maxRetries, toolCallExecutor: this.toolCallExecutor, cancellationToken diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts index 002bc375d6697..5cb18acf048f0 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -99,7 +99,6 @@ function makeIterator(openai: FakeOpenAi, overrides: Partial { expect(parts.filter(isTextResponsePart).map(p => p.content)).to.deep.equal(['recovered']); }); - it('stops after maxChatCompletions turns even if the model keeps requesting tools', async () => { - const request: UserRequest = { - sessionId: 'session', - requestId: 'request', - messages: [{ actor: 'user', type: 'text', text: 'hi' }], - tools: [toolRequest('a', async () => 'a-result')] - }; - // Every turn returns another tool call, so only the maxChatCompletions guard ends the loop. - const openai: FakeOpenAi = { - calls: [], - chat: { - completions: { - create: async (body: any) => { - openai.calls.push(body); - return new FakeStream([toolChunk(0, `call-${openai.calls.length}`, 'a', '{}')]); - } - } - } - }; - - await drain(makeIterator(openai, { request, maxChatCompletions: 3 })); - - expect(openai.calls).to.have.lengthOf(3); - }); - it('does not start a request when already cancelled', async () => { const source = new CancellationTokenSource(); source.cancel(); diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.ts index 47dffe15df074..829b669c3f78c 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.ts @@ -51,8 +51,6 @@ export interface ChatCompletionToolLoopOptions { /** Additional request settings (may include `stream_options`, temperature, etc.). */ readonly settings: Record; readonly tools: ChatCompletionTool[]; - /** Maximum number of chat completions (turns); each tool round counts as one. */ - readonly maxChatCompletions: number; readonly maxRetries: number; readonly toolCallExecutor: ToolCallExecutor; readonly cancellationToken?: CancellationToken; @@ -75,7 +73,6 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat protected readonly toDispose = new DisposableCollection(); protected readonly messages: ChatCompletionMessageParam[]; - protected iteration = 0; protected currentStream?: ChatCompletionChunkStream; constructor(protected readonly options: ChatCompletionToolLoopOptions) { @@ -114,7 +111,7 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat protected async startIteration(): Promise { try { - while (this.iteration < this.options.maxChatCompletions && !this.cancellationRequested) { + while (!this.cancellationRequested) { const { assistantText, toolCalls } = await this.processStream(); if (toolCalls.length === 0) { // No tool calls: the conversation is complete. @@ -122,9 +119,8 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat return; } await this.executeAndAppendToolCalls(assistantText, toolCalls); - this.iteration++; } - // Reached the maximum number of completions (or was cancelled). + // Cancelled before the model stopped requesting tools. this.dispose(); } catch (error) { if (this.cancellationRequested) { diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index 35f31dccb7477..3db0315abc9bd 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -178,7 +178,6 @@ export class OpenAiModel implements LanguageModel { messages: this.processMessages(request.messages), settings, tools, - maxChatCompletions: this.runnerOptions.maxChatCompletions ?? 100, maxRetries: this.maxRetries, toolCallExecutor: this.toolCallExecutor, cancellationToken From 02afdc282431654e7ef2cf9374a314eedcfa23bb Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 16 Jun 2026 10:47:19 -0400 Subject: [PATCH 09/10] refactor(ai): bind provider models as injectable services Make the provider language models @injectable and bind them as transient self-services. Their runtime configuration is injected as a ModelParams object and their service dependencies are injected separately; the LanguageModelFactory now resolves the model from a child container that carries the params. Public mutable fields are seeded from the params in @postConstruct so that the managers' patchLanguageModel/status mutations keep working. Extract OpenAiModelUtils into its own file (openai-model-utils.ts) so the model can inject it without a same-file forward-reference, per the one-class-per-file guideline. Use named loggers for new logging. This lets adopters rebind any of these services to substitute custom implementations. Signed-off-by: Christian W. Damus --- CHANGELOG.md | 4 + .../src/node/anthropic-backend-module.ts | 27 +-- .../src/node/anthropic-language-model.spec.ts | 162 +++++++------ .../src/node/anthropic-language-model.ts | 61 +++-- .../src/node/copilot-backend-module.ts | 22 +- .../src/node/copilot-language-model.spec.ts | 52 ++-- .../src/node/copilot-language-model.ts | 53 +++-- .../src/common/tool-call-execution.spec.ts | 9 +- .../ai-core/src/common/tool-call-execution.ts | 11 +- .../src/node/google-backend-module.ts | 22 +- .../src/node/google-language-model.spec.ts | 31 ++- .../src/node/google-language-model.ts | 46 +++- .../src/node/ollama-backend-module.ts | 19 +- .../src/node/ollama-language-model.spec.ts | 30 ++- .../src/node/ollama-language-model.ts | 42 ++-- packages/ai-ollama/src/package.spec.ts | 35 ++- .../src/node/openai-backend-module.ts | 47 ++-- .../openai-chat-completion-stream.spec.ts | 26 +- .../src/node/openai-chat-completion-stream.ts | 90 +++---- .../src/node/openai-language-model.spec.ts | 42 +++- .../src/node/openai-language-model.ts | 224 +++++------------- .../src/node/openai-model-utils.spec.ts | 2 +- .../ai-openai/src/node/openai-model-utils.ts | 145 ++++++++++++ .../node/openai-response-api-utils.spec.ts | 16 +- .../src/node/openai-response-api-utils.ts | 3 +- .../src/node/openai-streaming-iterator.ts | 67 +----- .../src/node/streaming-response-iterator.ts | 83 +++++++ 27 files changed, 800 insertions(+), 571 deletions(-) create mode 100644 packages/ai-openai/src/node/openai-model-utils.ts create mode 100644 packages/ai-openai/src/node/streaming-response-iterator.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d3557763ca0e..7ad6c3161e873 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - [terminal] fixed Cmd+V / Ctrl+V paste in the integrated terminal and restored the effect of the `terminal.enablePaste` and `terminal.enableCopy` preferences [#17603](https://github.com/eclipse-theia/theia/pull/17603) - [ai-anthropic, ai-google, ai-ollama, ai-openai, ai-copilot] added rebindable `LanguageModelFactory` bindings for instantiating provider language models [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-core] executed a model turn's tool calls concurrently, including parallel agent delegations, instead of sequentially, via a new injectable `ToolCallExecutor` [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-anthropic, ai-google, ai-ollama, ai-openai, ai-copilot] the provider language model classes are now `@injectable`, transient-scoped services constructed via their `LanguageModelFactory`, so adopters can rebind them to substitute custom implementations [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai] added a rebindable `ChatCompletionStreamingAsyncIteratorFactory`; the chat-completion tool-call streaming iterator (used by the OpenAI and Copilot models) is now an injectable service that can be substituted [#17623](https://github.com/eclipse-theia/theia/pull/17623) [Breaking Changes:](#breaking_changes_1.73.0) @@ -25,6 +27,8 @@ - [ai-openai] `OpenAiLanguageModelsManagerImpl` no longer injects `OpenAiModelUtils` or `OpenAiResponseApiUtils` (the `openAiModelUtils` and `responseApiUtils` protected fields were removed); provider models are now constructed via the injected `OpenAiLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-openai, ai-copilot] `OpenAiModel.createTools()` and `CopilotLanguageModel.createTools()` now return `ChatCompletionTool[]` instead of `RunnableToolFunctionWithoutParse[]`, because the OpenAI SDK `runTools` runner is no longer used [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-copilot] removed the `protected runnerOptions` field from `CopilotLanguageModel`; its only purpose was the `maxChatCompletions` turn cap, which is gone now that the OpenAI SDK `runTools` runner is unused, so the tool loop runs until the model stops requesting tools. Subclasses that read or overrode `runnerOptions` must adapt. `OpenAiModel.runnerOptions` is retained, but its `maxChatCompletions` now bounds only the Response API path, not the Chat Completions tool loop [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-anthropic, ai-google, ai-ollama, ai-openai, ai-copilot] the provider language model classes (`AnthropicModel`, `GoogleModel`, `OllamaModel`, `OpenAiModel`, `CopilotLanguageModel`) no longer expose public constructors; they are `@injectable` and receive their configuration through an injected `ModelParams` object (a symbol) plus injected service dependencies. Instantiate them via the corresponding `LanguageModelFactory` (or the DI container) instead of `new`, and drop constructor overrides in subclasses [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai] `OpenAiModelUtils` moved from `@theia/ai-openai/lib/node/openai-language-model` to `@theia/ai-openai/lib/node/openai-model-utils` [#17623](https://github.com/eclipse-theia/theia/pull/17623) ## 1.72.0 - 5/28/2026 diff --git a/packages/ai-anthropic/src/node/anthropic-backend-module.ts b/packages/ai-anthropic/src/node/anthropic-backend-module.ts index 1a12423f5c6f4..e297dca540f7d 100644 --- a/packages/ai-anthropic/src/node/anthropic-backend-module.ts +++ b/packages/ai-anthropic/src/node/anthropic-backend-module.ts @@ -14,8 +14,7 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ContainerModule } from '@theia/core/shared/inversify'; -import { ToolCallExecutor } from '@theia/ai-core'; +import { Container, ContainerModule } from '@theia/core/shared/inversify'; import { ANTHROPIC_LANGUAGE_MODELS_MANAGER_PATH, AnthropicLanguageModelsManager } from '../common/anthropic-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { AnthropicLanguageModelsManagerImpl } from './anthropic-language-models-manager-impl'; @@ -27,24 +26,14 @@ import { AnthropicPreferencesSchema } from '../common/anthropic-preferences'; const anthropicConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(AnthropicLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(AnthropicLanguageModelsManager).toService(AnthropicLanguageModelsManagerImpl); + bind(AnthropicModel).toSelf().inTransientScope(); bind(AnthropicLanguageModelFactory).toFactory( - ({ container }) => params => new AnthropicModel( - params.id, - params.model, - params.status, - params.enableStreaming, - params.useCaching, - params.apiKey, - params.url, - params.maxTokens, - params.maxRetries, - params.proxy, - params.reasoningSupport, - params.reasoningApi, - params.supportsXHighEffort, - params.maxInputTokens, - container.get(ToolCallExecutor) - ) + ({ container }) => params => { + const child = new Container(); + child.parent = container; + child.bind(AnthropicModelParams).toConstantValue(params); + return child.get(AnthropicModel); + } ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(ANTHROPIC_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(AnthropicLanguageModelsManager)) diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts b/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts index 8a63b4f47bd31..2e71f821628a6 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.spec.ts @@ -15,8 +15,13 @@ // ***************************************************************************** import { expect } from 'chai'; -import { AnthropicModel, DEFAULT_MAX_TOKENS, addCacheControlToLastMessage, mergeConsecutiveSameRoleMessages } from './anthropic-language-model'; -import { isUsageResponsePart, LanguageModelRequest, LanguageModelStreamResponsePart, ReasoningApi, ReasoningSupport, UserRequest } from '@theia/ai-core'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { AnthropicModel, AnthropicModelParams, DEFAULT_MAX_TOKENS, addCacheControlToLastMessage, mergeConsecutiveSameRoleMessages } from './anthropic-language-model'; +import { + isUsageResponsePart, LanguageModelRequest, LanguageModelStreamResponsePart, ReasoningApi, ReasoningSupport, ToolCallExecutor, ToolCallExecutorImpl, UserRequest +} from '@theia/ai-core'; import type { Anthropic } from '@anthropic-ai/sdk'; import type { MessageParam } from '@anthropic-ai/sdk/resources'; @@ -26,82 +31,89 @@ const REASONING_SUPPORT: ReasoningSupport = { }; /** Test helper that exposes the otherwise protected getSettings() method. */ +@injectable() class TestableAnthropicModel extends AnthropicModel { public callGetSettings(request: LanguageModelRequest): Readonly> { return this.getSettings(request); } } +function buildModel(modelType: new (...args: never[]) => T, params: AnthropicModelParams): T { + const parent = new Container(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + parent.bind(modelType).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(AnthropicModelParams).toConstantValue(params); + return child.get(modelType); +} + function createReasoningModel( modelId: string, reasoningApi: ReasoningApi, supportsXHighEffort: boolean = false ): TestableAnthropicModel { - return new TestableAnthropicModel( - 'test-id', modelId, { status: 'ready' }, true, false, - () => 'test-key', undefined, DEFAULT_MAX_TOKENS, - 3, undefined, REASONING_SUPPORT, reasoningApi, supportsXHighEffort - ); + return buildModel(TestableAnthropicModel, { + id: 'test-id', model: modelId, status: { status: 'ready' }, enableStreaming: true, useCaching: false, + apiKey: () => 'test-key', url: undefined, maxTokens: DEFAULT_MAX_TOKENS, + maxRetries: 3, reasoningSupport: REASONING_SUPPORT, reasoningApi, supportsXHighEffort + }); } function createNonReasoningModel(modelId: string): TestableAnthropicModel { - return new TestableAnthropicModel( - 'test-id', modelId, { status: 'ready' }, true, false, - () => 'test-key', undefined, DEFAULT_MAX_TOKENS - ); + return buildModel(TestableAnthropicModel, { + id: 'test-id', model: modelId, status: { status: 'ready' }, enableStreaming: true, useCaching: false, + apiKey: () => 'test-key', url: undefined, maxTokens: DEFAULT_MAX_TOKENS + }); } describe('AnthropicModel', () => { - describe('constructor', () => { + describe('parameters', () => { it('should set default maxRetries to 3 when not provided', () => { - const model = new AnthropicModel( - 'test-id', - 'claude-3-opus-20240229', - { - status: 'ready' - }, - true, - true, - () => 'test-api-key', - undefined, - DEFAULT_MAX_TOKENS - ); + const model = buildModel(AnthropicModel, { + id: 'test-id', + model: 'claude-3-opus-20240229', + status: { status: 'ready' }, + enableStreaming: true, + useCaching: true, + apiKey: () => 'test-api-key', + url: undefined, + maxTokens: DEFAULT_MAX_TOKENS + }); expect(model.maxRetries).to.equal(3); }); it('should set custom maxRetries when provided', () => { const customMaxRetries = 5; - const model = new AnthropicModel( - 'test-id', - 'claude-3-opus-20240229', - { - status: 'ready' - }, - true, - true, - () => 'test-api-key', - undefined, - DEFAULT_MAX_TOKENS, - customMaxRetries - ); + const model = buildModel(AnthropicModel, { + id: 'test-id', + model: 'claude-3-opus-20240229', + status: { status: 'ready' }, + enableStreaming: true, + useCaching: true, + apiKey: () => 'test-api-key', + url: undefined, + maxTokens: DEFAULT_MAX_TOKENS, + maxRetries: customMaxRetries + }); expect(model.maxRetries).to.equal(customMaxRetries); }); - it('should preserve all other constructor parameters', () => { - const model = new AnthropicModel( - 'test-id', - 'claude-3-opus-20240229', - { - status: 'ready' - }, - true, - true, - () => 'test-api-key', - undefined, - DEFAULT_MAX_TOKENS, - 5 - ); + it('should preserve all other parameters', () => { + const model = buildModel(AnthropicModel, { + id: 'test-id', + model: 'claude-3-opus-20240229', + status: { status: 'ready' }, + enableStreaming: true, + useCaching: true, + apiKey: () => 'test-api-key', + url: undefined, + maxTokens: DEFAULT_MAX_TOKENS, + maxRetries: 5 + }); expect(model.id).to.equal('test-id'); expect(model.model).to.equal('claude-3-opus-20240229'); @@ -111,19 +123,17 @@ describe('AnthropicModel', () => { }); it('should set custom url when provided', () => { - const model = new AnthropicModel( - 'test-id', - 'claude-3-opus-20240229', - { - status: 'ready' - }, - true, - true, - () => 'test-api-key', - 'custom-url', - DEFAULT_MAX_TOKENS, - 5 - ); + const model = buildModel(AnthropicModel, { + id: 'test-id', + model: 'claude-3-opus-20240229', + status: { status: 'ready' }, + enableStreaming: true, + useCaching: true, + apiKey: () => 'test-api-key', + url: 'custom-url', + maxTokens: DEFAULT_MAX_TOKENS, + maxRetries: 5 + }); expect(model.url).to.equal('custom-url'); }); @@ -329,15 +339,17 @@ describe('AnthropicModel', () => { function createModel(anthropicEventsByCall: object[][]): AnthropicModel { let callIndex = 0; - return new class extends AnthropicModel { + @injectable() + class MockAnthropicModel extends AnthropicModel { protected override initializeAnthropic(): Anthropic { const events = anthropicEventsByCall[Math.min(callIndex++, anthropicEventsByCall.length - 1)]; return buildMockAnthropic(events); } - }( - 'test-id', 'claude-opus-4-5', { status: 'ready' }, - true, false, () => 'test-key', undefined - ); + } + return buildModel(MockAnthropicModel, { + id: 'test-id', model: 'claude-opus-4-5', status: { status: 'ready' }, + enableStreaming: true, useCaching: false, apiKey: () => 'test-key', url: undefined + }); } async function collectStreamParts(model: AnthropicModel, text: string): Promise { @@ -480,14 +492,16 @@ describe('AnthropicModel', () => { { type: 'message_stop' }, ]; - const model = new class extends AnthropicModel { + @injectable() + class AbortingAnthropicModel extends AnthropicModel { protected override initializeAnthropic(): Anthropic { return buildAbortingAnthropic(events, 4); } - }( - 'test-id', 'claude-opus-4-5', { status: 'ready' }, - true, false, () => 'test-key', undefined - ); + } + const model = buildModel(AbortingAnthropicModel, { + id: 'test-id', model: 'claude-opus-4-5', status: { status: 'ready' }, + enableStreaming: true, useCaching: false, apiKey: () => 'test-key', url: undefined + }); const request: UserRequest = { messages: [{ actor: 'user', type: 'text', text: 'hi' }], diff --git a/packages/ai-anthropic/src/node/anthropic-language-model.ts b/packages/ai-anthropic/src/node/anthropic-language-model.ts index b5bbc878966ed..59b27804a9b98 100644 --- a/packages/ai-anthropic/src/node/anthropic-language-model.ts +++ b/packages/ai-anthropic/src/node/anthropic-language-model.ts @@ -29,10 +29,10 @@ import { ReasoningSupport, ToolCallResult, ToolCallExecutor, - ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken, isArray } from '@theia/core'; +import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import { Anthropic } from '@anthropic-ai/sdk'; import type { Base64ImageSource, ImageBlockParam, Message, MessageParam, TextBlockParam, ToolResultBlockParam } from '@anthropic-ai/sdk/resources'; import { createProxyFetch } from '@theia/ai-core/lib/node'; @@ -235,6 +235,8 @@ export interface AnthropicModelParams { maxInputTokens?: number; } +export const AnthropicModelParams = Symbol('AnthropicModelParams'); + export const AnthropicLanguageModelFactory = Symbol('AnthropicLanguageModelFactory'); export type AnthropicLanguageModelFactory = (params: AnthropicModelParams) => AnthropicModel; @@ -242,25 +244,48 @@ export type AnthropicLanguageModelFactory = (params: AnthropicModelParams) => An * Implements the Anthropic language model integration for Theia. Reasoning-level * translation lives in {@link anthropicReasoningFor}. */ +@injectable() export class AnthropicModel implements LanguageModel { - constructor( - public readonly id: string, - public model: string, - public status: LanguageModelStatus, - public enableStreaming: boolean, - public useCaching: boolean, - public apiKey: () => string | undefined, - public url: string | undefined, - public maxTokens: number = DEFAULT_MAX_TOKENS, - public maxRetries: number = 3, - public proxy?: string, - public reasoningSupport?: ReasoningSupport, - public reasoningApi?: ReasoningApi, - public supportsXHighEffort?: boolean, - public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() - ) { } + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + useCaching: boolean; + apiKey: () => string | undefined; + url: string | undefined; + maxTokens: number; + maxRetries: number; + proxy?: string; + reasoningSupport?: ReasoningSupport; + reasoningApi?: ReasoningApi; + supportsXHighEffort?: boolean; + maxInputTokens?: number; + + @inject(AnthropicModelParams) + protected readonly params: AnthropicModelParams; + + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + + @postConstruct() + protected init(): void { + const params = this.params; + this.id = params.id; + this.model = params.model; + this.status = params.status; + this.enableStreaming = params.enableStreaming; + this.useCaching = params.useCaching; + this.apiKey = params.apiKey; + this.url = params.url; + this.maxTokens = params.maxTokens ?? DEFAULT_MAX_TOKENS; + this.maxRetries = params.maxRetries ?? 3; + this.proxy = params.proxy; + this.reasoningSupport = params.reasoningSupport; + this.reasoningApi = params.reasoningApi; + this.supportsXHighEffort = params.supportsXHighEffort; + this.maxInputTokens = params.maxInputTokens; + } protected getSettings(request: LanguageModelRequest): Readonly> { return { diff --git a/packages/ai-copilot/src/node/copilot-backend-module.ts b/packages/ai-copilot/src/node/copilot-backend-module.ts index 03a18a3066da8..4f768f4f255ec 100644 --- a/packages/ai-copilot/src/node/copilot-backend-module.ts +++ b/packages/ai-copilot/src/node/copilot-backend-module.ts @@ -14,9 +14,8 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ContainerModule } from '@theia/core/shared/inversify'; +import { Container, ContainerModule } from '@theia/core/shared/inversify'; import { ConnectionHandler, RpcConnectionHandler } from '@theia/core'; -import { ToolCallExecutor } from '@theia/ai-core'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; import { CopilotLanguageModelsManager, @@ -37,19 +36,14 @@ const copilotConnectionModule = ConnectionContainerModule.create(({ bind }) => { bind(CopilotLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(CopilotLanguageModelsManager).toService(CopilotLanguageModelsManagerImpl); + bind(CopilotLanguageModel).toSelf().inTransientScope(); bind(CopilotLanguageModelFactory).toFactory( - ({ container }) => params => new CopilotLanguageModel( - params.id, - params.model, - params.status, - params.enableStreaming, - params.supportsStructuredOutput, - params.maxRetries, - params.accessTokenProvider, - params.enterpriseUrlProvider, - params.userAgentProvider, - container.get(ToolCallExecutor) - ) + ({ container }) => params => { + const child = new Container(); + child.parent = container; + child.bind(CopilotLanguageModelParams).toConstantValue(params); + return child.get(CopilotLanguageModel); + } ); bind(ConnectionHandler).toDynamicValue(ctx => diff --git a/packages/ai-copilot/src/node/copilot-language-model.spec.ts b/packages/ai-copilot/src/node/copilot-language-model.spec.ts index 40abdfc336044..a77ca505c73c5 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.spec.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.spec.ts @@ -15,25 +15,16 @@ // ***************************************************************************** import { expect } from 'chai'; -import { LanguageModelMessage, LanguageModelRequest } from '@theia/ai-core'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { LanguageModelMessage, LanguageModelRequest, ToolCallExecutor, ToolCallExecutorImpl } from '@theia/ai-core'; +import { ChatCompletionStreamingAsyncIteratorFactory } from '@theia/ai-openai/lib/node/openai-chat-completion-stream'; import { ChatCompletionMessageParam } from 'openai/resources'; -import { CopilotLanguageModel } from './copilot-language-model'; +import { CopilotLanguageModel, CopilotLanguageModelParams } from './copilot-language-model'; +@injectable() class TestableCopilotLanguageModel extends CopilotLanguageModel { - constructor() { - super( - 'test-id', - 'test-model', - { status: 'ready' }, - true, - false, - 3, - async () => 'test-token', - () => undefined, - () => 'test-user-agent' - ); - } - public callProcessMessages(messages: LanguageModelMessage[]): ChatCompletionMessageParam[] { return this.processMessages(messages); } @@ -43,8 +34,33 @@ class TestableCopilotLanguageModel extends CopilotLanguageModel { } } +function createModel(): TestableCopilotLanguageModel { + const parent = new Container(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + // These tests never issue a streaming request, so the iterator factory is never invoked. + const iteratorFactory: ChatCompletionStreamingAsyncIteratorFactory = () => { throw new Error('iterator not used in these tests'); }; + parent.bind(ChatCompletionStreamingAsyncIteratorFactory).toConstantValue(iteratorFactory); + parent.bind(TestableCopilotLanguageModel).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(CopilotLanguageModelParams).toConstantValue({ + id: 'test-id', + model: 'test-model', + status: { status: 'ready' }, + enableStreaming: true, + supportsStructuredOutput: false, + maxRetries: 3, + accessTokenProvider: async () => 'test-token', + enterpriseUrlProvider: () => undefined, + userAgentProvider: () => 'test-user-agent' + }); + return child.get(TestableCopilotLanguageModel); +} + describe('CopilotLanguageModel - processMessages', () => { - const model = new TestableCopilotLanguageModel(); + const model = createModel(); it('should merge an assistant text message followed by an assistant tool_use into a single message', () => { const messages: LanguageModelMessage[] = [ @@ -111,7 +127,7 @@ describe('CopilotLanguageModel - processMessages', () => { describe('CopilotLanguageModel - createTools', () => { it('produces plain function tool definitions without an embedded handler function', () => { - const model = new TestableCopilotLanguageModel(); + const model = createModel(); const tools = model.callCreateTools({ messages: [], tools: [{ id: 't', name: 't', parameters: { type: 'object', properties: {} }, handler: async () => 'x' }] diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index 4afb356b5fd09..af2b0240c1035 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -24,14 +24,14 @@ import { LanguageModelStatus, LanguageModelTextResponse, ToolCallExecutor, - ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; +import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import OpenAI from 'openai'; import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; import { StreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-streaming-iterator'; -import { ChatCompletionStreamingAsyncIterator } from '@theia/ai-openai/lib/node/openai-chat-completion-stream'; +import { ChatCompletionStreamingAsyncIteratorFactory } from '@theia/ai-openai/lib/node/openai-chat-completion-stream'; import { COPILOT_PROVIDER_ID, getCopilotApiBaseUrl } from '../common'; export interface CopilotLanguageModelParams { @@ -46,6 +46,8 @@ export interface CopilotLanguageModelParams { userAgentProvider: () => string; } +export const CopilotLanguageModelParams = Symbol('CopilotLanguageModelParams'); + export const CopilotLanguageModelFactory = Symbol('CopilotLanguageModelFactory'); export type CopilotLanguageModelFactory = (params: CopilotLanguageModelParams) => CopilotLanguageModel; @@ -53,20 +55,41 @@ export type CopilotLanguageModelFactory = (params: CopilotLanguageModelParams) = * Language model implementation for GitHub Copilot. * Uses the OpenAI SDK to communicate with the Copilot API. */ +@injectable() export class CopilotLanguageModel implements LanguageModel { - constructor( - public readonly id: string, - public model: string, - public status: LanguageModelStatus, - public enableStreaming: boolean, - public supportsStructuredOutput: boolean, - public maxRetries: number, - protected readonly accessTokenProvider: () => Promise, - protected readonly enterpriseUrlProvider: () => string | undefined, - protected readonly userAgentProvider: () => string, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() - ) { } + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + supportsStructuredOutput: boolean; + maxRetries: number; + protected accessTokenProvider: () => Promise; + protected enterpriseUrlProvider: () => string | undefined; + protected userAgentProvider: () => string; + + @inject(CopilotLanguageModelParams) + protected readonly params: CopilotLanguageModelParams; + + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + + @inject(ChatCompletionStreamingAsyncIteratorFactory) + protected readonly chatCompletionStreamFactory: ChatCompletionStreamingAsyncIteratorFactory; + + @postConstruct() + protected init(): void { + const params = this.params; + this.id = params.id; + this.model = params.model; + this.status = params.status; + this.enableStreaming = params.enableStreaming; + this.supportsStructuredOutput = params.supportsStructuredOutput; + this.maxRetries = params.maxRetries; + this.accessTokenProvider = params.accessTokenProvider; + this.enterpriseUrlProvider = params.enterpriseUrlProvider; + this.userAgentProvider = params.userAgentProvider; + } protected getSettings(request: LanguageModelRequest): Record { return request.settings ?? {}; @@ -97,7 +120,7 @@ export class CopilotLanguageModel implements LanguageModel { if (tools) { return { - stream: new ChatCompletionStreamingAsyncIterator({ + stream: this.chatCompletionStreamFactory({ openai, model: this.model, request, diff --git a/packages/ai-core/src/common/tool-call-execution.spec.ts b/packages/ai-core/src/common/tool-call-execution.spec.ts index de10f7db651e6..9bf504d63ef0e 100644 --- a/packages/ai-core/src/common/tool-call-execution.spec.ts +++ b/packages/ai-core/src/common/tool-call-execution.spec.ts @@ -15,7 +15,9 @@ // ***************************************************************************** import { expect } from 'chai'; -import { CancellationToken, CancellationTokenSource } from '@theia/core'; +import { CancellationToken, CancellationTokenSource, ILogger } from '@theia/core'; +import { Container } from '@theia/core/shared/inversify'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { ToolRequest, @@ -35,7 +37,10 @@ describe('ToolCallExecutor', () => { let executor: ToolCallExecutor; beforeEach(() => { - executor = new ToolCallExecutorImpl(); + const container = new Container(); + container.bind(ILogger).to(MockLogger); + container.bind(ToolCallExecutorImpl).toSelf(); + executor = container.get(ToolCallExecutorImpl); }); it('executes the tool calls of a turn concurrently (not sequentially)', async () => { diff --git a/packages/ai-core/src/common/tool-call-execution.ts b/packages/ai-core/src/common/tool-call-execution.ts index c0386d12bf9d8..aaee5338a22b1 100644 --- a/packages/ai-core/src/common/tool-call-execution.ts +++ b/packages/ai-core/src/common/tool-call-execution.ts @@ -14,8 +14,8 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { CancellationToken } from '@theia/core'; -import { injectable } from '@theia/core/shared/inversify'; +import { CancellationToken, ILogger } from '@theia/core'; +import { inject, injectable, named } from '@theia/core/shared/inversify'; import { ToolRequest, ToolCallResult, @@ -108,6 +108,9 @@ export const ToolCallExecutor = Symbol('ToolCallExecutor'); @injectable() export class ToolCallExecutorImpl implements ToolCallExecutor { + @inject(ILogger) @named('ai-core:ToolCallExecutorImpl') + protected readonly logger: ILogger; + async executeToolCalls( toolCalls: readonly ToolInvocation[], tools: readonly ToolRequest[] | undefined, @@ -135,14 +138,14 @@ export class ToolCallExecutorImpl implements ToolCallExecutor { outcome = { id, name, arguments: args, result, notFound: false }; } catch (e) { const error = e instanceof Error ? e : new Error(String(e)); - console.error(`Error executing tool ${name}:`, e); + this.logger.error(`Error executing tool ${name}:`, e); outcome = { id, name, arguments: args, notFound: false, error, result: createToolCallError(error.message || 'Tool execution failed') }; } } try { options.onResult?.(outcome); } catch (error) { - console.error('Uncaught error in tool-call onResult call-back.', error); + this.logger.error('Uncaught error in tool-call onResult call-back.', error); } return outcome; } diff --git a/packages/ai-google/src/node/google-backend-module.ts b/packages/ai-google/src/node/google-backend-module.ts index dd35465fe2d5e..46e428fe091af 100644 --- a/packages/ai-google/src/node/google-backend-module.ts +++ b/packages/ai-google/src/node/google-backend-module.ts @@ -14,8 +14,7 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ContainerModule } from '@theia/core/shared/inversify'; -import { ToolCallExecutor } from '@theia/ai-core'; +import { Container, ContainerModule } from '@theia/core/shared/inversify'; import { GOOGLE_LANGUAGE_MODELS_MANAGER_PATH, GoogleLanguageModelsManager } from '../common/google-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { GoogleLanguageModelsManagerImpl } from './google-language-models-manager-impl'; @@ -27,19 +26,14 @@ import { GooglePreferencesSchema } from '../common/google-preferences'; const geminiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(GoogleLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(GoogleLanguageModelsManager).toService(GoogleLanguageModelsManagerImpl); + bind(GoogleModel).toSelf().inTransientScope(); bind(GoogleLanguageModelFactory).toFactory( - ({ container }) => params => new GoogleModel( - params.id, - params.model, - params.status, - params.enableStreaming, - params.apiKey, - params.retrySettings, - params.reasoningSupport, - params.reasoningApi, - params.maxInputTokens, - container.get(ToolCallExecutor) - ) + ({ container }) => params => { + const child = new Container(); + child.parent = container; + child.bind(GoogleModelParams).toConstantValue(params); + return child.get(GoogleModel); + } ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(GOOGLE_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(GoogleLanguageModelsManager)) diff --git a/packages/ai-google/src/node/google-language-model.spec.ts b/packages/ai-google/src/node/google-language-model.spec.ts index 26bf8719f53ad..4913681168263 100644 --- a/packages/ai-google/src/node/google-language-model.spec.ts +++ b/packages/ai-google/src/node/google-language-model.spec.ts @@ -15,14 +15,18 @@ // ***************************************************************************** import { expect } from 'chai'; -import { LanguageModelRequest, ReasoningApi, ReasoningSupport } from '@theia/ai-core'; -import { GoogleModel } from './google-language-model'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { LanguageModelRequest, ReasoningApi, ReasoningSupport, ToolCallExecutor, ToolCallExecutorImpl } from '@theia/ai-core'; +import { GoogleModel, GoogleModelParams } from './google-language-model'; const GEMINI_REASONING_SUPPORT: ReasoningSupport = { supportedLevels: ['off', 'minimal', 'low', 'medium', 'high', 'auto'], defaultLevel: 'auto' }; +@injectable() class TestableGoogleModel extends GoogleModel { public callGetSettings(request: LanguageModelRequest): Readonly> { return this.getSettings(request); @@ -30,13 +34,24 @@ class TestableGoogleModel extends GoogleModel { } function createModel(modelId: string, reasoningApi?: ReasoningApi): TestableGoogleModel { - return new TestableGoogleModel( - 'test-id', modelId, { status: 'ready' }, true, - () => 'test-key', - () => ({ maxRetriesOnErrors: 0, retryDelayOnRateLimitError: -1, retryDelayOnOtherErrors: -1 }), - reasoningApi ? GEMINI_REASONING_SUPPORT : undefined, + const parent = new Container(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + parent.bind(TestableGoogleModel).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(GoogleModelParams).toConstantValue({ + id: 'test-id', + model: modelId, + status: { status: 'ready' }, + enableStreaming: true, + apiKey: () => 'test-key', + retrySettings: () => ({ maxRetriesOnErrors: 0, retryDelayOnRateLimitError: -1, retryDelayOnOtherErrors: -1 }), + reasoningSupport: reasoningApi ? GEMINI_REASONING_SUPPORT : undefined, reasoningApi - ); + }); + return child.get(TestableGoogleModel); } describe('GoogleModel reasoning translation', () => { diff --git a/packages/ai-google/src/node/google-language-model.ts b/packages/ai-google/src/node/google-language-model.ts index 21db86d20e695..a48ca4c67c8fc 100644 --- a/packages/ai-google/src/node/google-language-model.ts +++ b/packages/ai-google/src/node/google-language-model.ts @@ -27,10 +27,10 @@ import { ReasoningSupport, ToolCallResult, ToolCallExecutor, - ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; +import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import { GoogleGenAI, FunctionCallingConfigMode, FunctionDeclaration, Content, Schema, Part, Modality, FunctionResponse, ToolConfig } from '@google/genai'; import { wait } from '@theia/core/lib/common/promise-util'; import { GoogleLanguageModelRetrySettings } from './google-language-models-manager-impl'; @@ -149,6 +149,8 @@ export interface GoogleModelParams { maxInputTokens?: number; } +export const GoogleModelParams = Symbol('GoogleModelParams'); + export const GoogleLanguageModelFactory = Symbol('GoogleLanguageModelFactory'); export type GoogleLanguageModelFactory = (params: GoogleModelParams) => GoogleModel; @@ -156,20 +158,38 @@ export type GoogleLanguageModelFactory = (params: GoogleModelParams) => GoogleMo * Implements the Gemini language model integration for Theia. Reasoning-level * translation lives in {@link googleReasoningFor}. */ +@injectable() export class GoogleModel implements LanguageModel { - constructor( - public readonly id: string, - public model: string, - public status: LanguageModelStatus, - public enableStreaming: boolean, - public apiKey: () => string | undefined, - public retrySettings: () => GoogleLanguageModelRetrySettings, - public reasoningSupport?: ReasoningSupport, - public reasoningApi?: ReasoningApi, - public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() - ) { } + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + apiKey: () => string | undefined; + retrySettings: () => GoogleLanguageModelRetrySettings; + reasoningSupport?: ReasoningSupport; + reasoningApi?: ReasoningApi; + maxInputTokens?: number; + + @inject(GoogleModelParams) + protected readonly params: GoogleModelParams; + + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + + @postConstruct() + protected init(): void { + const params = this.params; + this.id = params.id; + this.model = params.model; + this.status = params.status; + this.enableStreaming = params.enableStreaming; + this.apiKey = params.apiKey; + this.retrySettings = params.retrySettings; + this.reasoningSupport = params.reasoningSupport; + this.reasoningApi = params.reasoningApi; + this.maxInputTokens = params.maxInputTokens; + } protected getSettings(request: LanguageModelRequest): Readonly> { return { diff --git a/packages/ai-ollama/src/node/ollama-backend-module.ts b/packages/ai-ollama/src/node/ollama-backend-module.ts index 6f02e5db15edc..7bf20d3142638 100644 --- a/packages/ai-ollama/src/node/ollama-backend-module.ts +++ b/packages/ai-ollama/src/node/ollama-backend-module.ts @@ -14,8 +14,7 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ContainerModule } from '@theia/core/shared/inversify'; -import { ToolCallExecutor } from '@theia/ai-core'; +import { Container, ContainerModule } from '@theia/core/shared/inversify'; import { OLLAMA_LANGUAGE_MODELS_MANAGER_PATH, OllamaLanguageModelsManager } from '../common/ollama-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { OllamaLanguageModelsManagerImpl } from './ollama-language-models-manager-impl'; @@ -27,16 +26,14 @@ import { OllamaPreferencesSchema } from '../common/ollama-preferences'; const ollamaConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(OllamaLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(OllamaLanguageModelsManager).toService(OllamaLanguageModelsManagerImpl); + bind(OllamaModel).toSelf().inTransientScope(); bind(OllamaLanguageModelFactory).toFactory( - ({ container }) => params => new OllamaModel( - params.id, - params.model, - params.status, - params.host, - params.proxy, - params.reasoningSupport, - container.get(ToolCallExecutor) - ) + ({ container }) => params => { + const child = new Container(); + child.parent = container; + child.bind(OllamaModelParams).toConstantValue(params); + return child.get(OllamaModel); + } ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(OLLAMA_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(OllamaLanguageModelsManager)) diff --git a/packages/ai-ollama/src/node/ollama-language-model.spec.ts b/packages/ai-ollama/src/node/ollama-language-model.spec.ts index cf4175d2f36b9..e57b5a540eaee 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.spec.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.spec.ts @@ -15,21 +15,39 @@ // ***************************************************************************** import { expect } from 'chai'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { ToolCallExecutor, ToolCallExecutorImpl } from '@theia/ai-core'; import { Message } from 'ollama'; -import { OllamaModel } from './ollama-language-model'; +import { OllamaModel, OllamaModelParams } from './ollama-language-model'; +@injectable() class TestableOllamaModel extends OllamaModel { - constructor() { - super('test-id', 'test-model', { status: 'ready' }, () => 'http://localhost:11434'); - } - public callMergeConsecutiveAssistantMessages(messages: Message[]): Message[] { return this.mergeConsecutiveAssistantMessages(messages); } } +function createModel(): TestableOllamaModel { + const parent = new Container(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + parent.bind(TestableOllamaModel).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(OllamaModelParams).toConstantValue({ + id: 'test-id', + model: 'test-model', + status: { status: 'ready' }, + host: () => 'http://localhost:11434' + }); + return child.get(TestableOllamaModel); +} + describe('OllamaModel - mergeConsecutiveAssistantMessages', () => { - const model = new TestableOllamaModel(); + const model = createModel(); it('should merge an assistant text message followed by an assistant tool_use into a single message', () => { const messages: Message[] = [ diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index 426e20e98bc8e..57beb67445c5b 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -25,7 +25,6 @@ import { ReasoningSupport, ToolCall, ToolCallExecutor, - ToolCallExecutorImpl, ToolCallResult, ToolRequest, ToolRequestParametersProperties, @@ -36,6 +35,7 @@ import { UserRequest } from '@theia/ai-core'; import { CancellationToken } from '@theia/core'; +import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import { ChatRequest, Message, Ollama, Options, Tool, ToolCall as OllamaToolCall } from 'ollama'; import { createProxyFetch } from '@theia/ai-core/lib/node'; import { ollamaThinkParamFor } from './ollama-reasoning'; @@ -51,9 +51,12 @@ export interface OllamaModelParams { reasoningSupport?: ReasoningSupport; } +export const OllamaModelParams = Symbol('OllamaModelParams'); + export const OllamaLanguageModelFactory = Symbol('OllamaLanguageModelFactory'); export type OllamaLanguageModelFactory = (params: OllamaModelParams) => OllamaModel; +@injectable() export class OllamaModel implements LanguageModel { protected readonly DEFAULT_REQUEST_SETTINGS: Partial> = { @@ -65,20 +68,29 @@ export class OllamaModel implements LanguageModel { readonly providerId = 'ollama'; readonly vendor: string = 'Ollama'; - /** - * @param id the unique id for this language model. It will be used to identify the model in the UI. - * @param model the unique model name as used in the Ollama environment. - * @param hostProvider a function to provide the host URL for the Ollama server. - */ - constructor( - public readonly id: string, - protected readonly model: string, - public status: LanguageModelStatus, - protected host: () => string | undefined, - public proxy?: string, - public reasoningSupport?: ReasoningSupport, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() - ) { } + id: string; + protected model: string; + status: LanguageModelStatus; + protected host: () => string | undefined; + proxy?: string; + reasoningSupport?: ReasoningSupport; + + @inject(OllamaModelParams) + protected readonly params: OllamaModelParams; + + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + + @postConstruct() + protected init(): void { + const params = this.params; + this.id = params.id; + this.model = params.model; + this.status = params.status; + this.host = params.host; + this.proxy = params.proxy; + this.reasoningSupport = params.reasoningSupport; + } async request(request: UserRequest, cancellationToken?: CancellationToken): Promise { const settings = this.getSettings(request); diff --git a/packages/ai-ollama/src/package.spec.ts b/packages/ai-ollama/src/package.spec.ts index 13b3039b7ebd2..0878768114f06 100644 --- a/packages/ai-ollama/src/package.spec.ts +++ b/packages/ai-ollama/src/package.spec.ts @@ -14,9 +14,12 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ToolCall, ToolRequest } from '@theia/ai-core'; +import { ToolCall, ToolCallExecutor, ToolCallExecutorImpl, ToolRequest } from '@theia/ai-core'; import { Deferred } from '@theia/core/lib/common/promise-util'; -import { OllamaModel } from './node/ollama-language-model'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { OllamaModel, OllamaModelParams } from './node/ollama-language-model'; import { Tool } from 'ollama'; import { expect } from 'chai'; import * as sinon from 'sinon'; @@ -25,7 +28,7 @@ describe('ai-ollama package', () => { it('Transform to Ollama tools', () => { const req: ToolRequest = createToolRequest(); - const model = new OllamaModelUnderTest(); + const model = createModel(); const ollamaTool = model.toOllamaTool(req); expect(ollamaTool.function.name).equals('example-tool'); @@ -36,7 +39,7 @@ describe('ai-ollama package', () => { }); it('executes tool calls of a turn concurrently and preserves input order', async () => { - const model = new OllamaModelUnderTest(); + const model = createModel(); // `a` only completes once `b` has started: a sequential implementation would deadlock here. const bStarted = new Deferred(); const chatRequest = { @@ -62,7 +65,7 @@ describe('ai-ollama package', () => { }); it('reports a missing tool with the legacy error string', async () => { - const model = new OllamaModelUnderTest(); + const model = createModel(); // eslint-disable-next-line @typescript-eslint/no-explicit-any const chatRequest = { messages: [], tools: [] } as any; const result = await model.runProcessToolCalls([{ id: '1', function: { name: 'missing', arguments: '{}' } }], chatRequest); @@ -70,11 +73,25 @@ describe('ai-ollama package', () => { }); }); -class OllamaModelUnderTest extends OllamaModel { - constructor() { - super('id', 'model', { status: 'ready' }, () => ''); - } +function createModel(): OllamaModelUnderTest { + const parent = new Container(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + parent.bind(OllamaModelUnderTest).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(OllamaModelParams).toConstantValue({ + id: 'id', + model: 'model', + status: { status: 'ready' }, + host: () => '' + }); + return child.get(OllamaModelUnderTest); +} +@injectable() +class OllamaModelUnderTest extends OllamaModel { override toOllamaTool(tool: ToolRequest): Tool & { handler: (arg_string: string) => Promise } { return super.toOllamaTool(tool); } diff --git a/packages/ai-openai/src/node/openai-backend-module.ts b/packages/ai-openai/src/node/openai-backend-module.ts index 72e792e3cdb64..acdda6532bbeb 100644 --- a/packages/ai-openai/src/node/openai-backend-module.ts +++ b/packages/ai-openai/src/node/openai-backend-module.ts @@ -14,13 +14,18 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ContainerModule } from '@theia/core/shared/inversify'; -import { ToolCallExecutor } from '@theia/ai-core'; +import { Container, ContainerModule } from '@theia/core/shared/inversify'; import { OPENAI_LANGUAGE_MODELS_MANAGER_PATH, OpenAiLanguageModelsManager } from '../common/openai-language-models-manager'; import { ConnectionHandler, PreferenceContribution, RpcConnectionHandler } from '@theia/core'; import { OpenAiLanguageModelsManagerImpl } from './openai-language-models-manager-impl'; import { ConnectionContainerModule } from '@theia/core/lib/node/messaging/connection-container-module'; -import { OpenAiLanguageModelFactory, OpenAiModel, OpenAiModelParams, OpenAiModelUtils } from './openai-language-model'; +import { OpenAiLanguageModelFactory, OpenAiModel, OpenAiModelParams } from './openai-language-model'; +import { OpenAiModelUtils } from './openai-model-utils'; +import { + ChatCompletionStreamingAsyncIterator, + ChatCompletionStreamingAsyncIteratorFactory, + ChatCompletionToolLoopOptions +} from './openai-chat-completion-stream'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; import { OpenAiPreferencesSchema } from '../common/openai-preferences'; @@ -28,27 +33,14 @@ import { OpenAiPreferencesSchema } from '../common/openai-preferences'; const openAiConnectionModule = ConnectionContainerModule.create(({ bind, bindBackendService, bindFrontendService }) => { bind(OpenAiLanguageModelsManagerImpl).toSelf().inSingletonScope(); bind(OpenAiLanguageModelsManager).toService(OpenAiLanguageModelsManagerImpl); + bind(OpenAiModel).toSelf().inTransientScope(); bind(OpenAiLanguageModelFactory).toFactory( - ({ container }) => params => new OpenAiModel( - params.id, - params.model, - params.status, - params.enableStreaming, - params.apiKey, - params.apiVersion, - params.supportsStructuredOutput, - params.url, - params.deployment, - container.get(OpenAiModelUtils), - container.get(OpenAiResponseApiUtils), - params.developerMessageSettings, - params.maxRetries, - params.useResponseApi, - params.proxy, - params.reasoningSupport, - params.maxInputTokens, - container.get(ToolCallExecutor) - ) + ({ container }) => params => { + const child = new Container(); + child.parent = container; + child.bind(OpenAiModelParams).toConstantValue(params); + return child.get(OpenAiModel); + } ); bind(ConnectionHandler).toDynamicValue(ctx => new RpcConnectionHandler(OPENAI_LANGUAGE_MODELS_MANAGER_PATH, () => ctx.container.get(OpenAiLanguageModelsManager)) @@ -59,5 +51,14 @@ export default new ContainerModule(bind => { bind(PreferenceContribution).toConstantValue({ schema: OpenAiPreferencesSchema }); bind(OpenAiModelUtils).toSelf().inSingletonScope(); bind(OpenAiResponseApiUtils).toSelf().inSingletonScope(); + bind(ChatCompletionStreamingAsyncIterator).toSelf().inTransientScope(); + bind(ChatCompletionStreamingAsyncIteratorFactory).toFactory( + ({ container }) => options => { + const child = new Container(); + child.parent = container; + child.bind(ChatCompletionToolLoopOptions).toConstantValue(options); + return child.get(ChatCompletionStreamingAsyncIterator); + } + ); bind(ConnectionContainerModule).toConstantValue(openAiConnectionModule); }); diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts index 5cb18acf048f0..1504bceb17317 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -16,7 +16,9 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; -import { CancellationTokenSource } from '@theia/core'; +import { CancellationTokenSource, ILogger } from '@theia/core'; +import { Container, injectable, interfaces } from '@theia/core/shared/inversify'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { createToolCallError, @@ -86,13 +88,21 @@ function toolRequest(name: string, handler: ToolRequest['handler']): ToolRequest return { id: name, name, parameters: { type: 'object', properties: {} }, handler }; } +/** Resolves an executor (or executor subclass) through DI so its named logger is injected. */ +function withLogger(constructor: interfaces.Newable): T { + const container = new Container(); + container.bind(ILogger).to(MockLogger); + container.bind(constructor).toSelf(); + return container.get(constructor); +} + function makeIterator(openai: FakeOpenAi, overrides: Partial = {}): ChatCompletionStreamingAsyncIterator { const defaultRequest: UserRequest = { sessionId: 'session', requestId: 'request', messages: [{ actor: 'user', type: 'text', text: 'hi' }] }; - return new ChatCompletionStreamingAsyncIterator({ + const options: ChatCompletionToolLoopOptions = { openai: openai as any, model: 'gpt-test', request: overrides.request ?? defaultRequest, @@ -100,9 +110,14 @@ function makeIterator(openai: FakeOpenAi, overrides: Partial): Promise { @@ -116,6 +131,7 @@ async function drain(iterator: AsyncIterableIterator => new Promise(resolve => setImmediate(resolve)); /** Captures the batches of tool calls passed to the executor, to assert single-turn batching. */ +@injectable() class RecordingExecutor extends ToolCallExecutorImpl { readonly batches: ToolInvocation[][] = []; override executeToolCalls( @@ -157,7 +173,7 @@ describe('ChatCompletionStreamingAsyncIterator', () => { toolRequest('b', async () => { bStarted.resolve(); return 'b-result'; }) ] }; - const executor = new RecordingExecutor(); + const executor = withLogger(RecordingExecutor); const openai = fakeOpenAi([ new FakeStream([toolChunk(0, 'call-a', 'a', '{}'), toolChunk(1, 'call-b', 'b', '{}')]), new FakeStream([textChunk('done')]) diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.ts index 829b669c3f78c..95a1c8ba78951 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.ts @@ -14,10 +14,11 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { LanguageModelStreamResponsePart, ToolCallExecutor, ToolCallResult, UserRequest } from '@theia/ai-core'; -import { CancellationError, CancellationToken, Disposable, DisposableCollection } from '@theia/core'; -import { Deferred } from '@theia/core/lib/common/promise-util'; +import { ToolCallExecutor, ToolCallResult, UserRequest } from '@theia/ai-core'; +import { CancellationError, CancellationToken, ILogger } from '@theia/core'; +import { inject, injectable, named, postConstruct } from '@theia/core/shared/inversify'; import { OpenAI } from 'openai'; +import { AbstractStreamingResponseIterator } from './streaming-response-iterator'; import { ChatCompletionAssistantMessageParam, ChatCompletionChunk, @@ -26,8 +27,6 @@ import { ChatCompletionTool } from 'openai/resources'; -type IterResult = IteratorResult; - /** A chat completion stream as returned by `chat.completions.create({ stream: true })`. */ type ChatCompletionChunkStream = AsyncIterable & { controller: AbortController }; @@ -41,6 +40,8 @@ interface CollectedToolCall { opened: boolean; } +export const ChatCompletionToolLoopOptions = Symbol('ChatCompletionToolLoopOptions'); + export interface ChatCompletionToolLoopOptions { readonly openai: OpenAI; readonly model: string; @@ -65,44 +66,25 @@ export interface ChatCompletionToolLoopOptions { * (`chat.completions.create({ stream: true })`) ourselves, multiple tool calls emitted in a single * model turn (e.g. parallel agent delegations) run in parallel. */ -export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterator, Disposable { - protected readonly requestQueue = new Array>(); - protected readonly messageCache = new Array(); - protected done = false; - protected terminalError: Error | undefined = undefined; - protected readonly toDispose = new DisposableCollection(); +@injectable() +export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingResponseIterator { - protected readonly messages: ChatCompletionMessageParam[]; + protected messages: ChatCompletionMessageParam[]; protected currentStream?: ChatCompletionChunkStream; - constructor(protected readonly options: ChatCompletionToolLoopOptions) { - this.messages = [...options.messages]; - if (options.cancellationToken) { - this.toDispose.push(options.cancellationToken.onCancellationRequested(() => this.currentStream?.controller.abort())); - } - this.startIteration(); - } + @inject(ChatCompletionToolLoopOptions) + protected readonly options: ChatCompletionToolLoopOptions; - [Symbol.asyncIterator](): AsyncIterableIterator { - return this; - } + @inject(ILogger) @named('ai-openai:ChatCompletionStreamingAsyncIterator') + protected readonly logger: ILogger; - next(): Promise { - if (this.messageCache.length && this.requestQueue.length) { - throw new Error('Assertion error: cache and queue should not both be populated.'); - } - // Deliver all the messages we got, even if we've since terminated. - if (this.messageCache.length) { - return Promise.resolve({ done: false, value: this.messageCache.shift()! }); - } else if (this.terminalError) { - return Promise.reject(this.terminalError); - } else if (this.done) { - return Promise.resolve({ done: true, value: undefined }); - } else { - const toQueue = new Deferred(); - this.requestQueue.push(toQueue); - return toQueue.promise; + @postConstruct() + protected init(): void { + this.messages = [...this.options.messages]; + if (this.options.cancellationToken) { + this.toDispose.push(this.options.cancellationToken.onCancellationRequested(() => this.currentStream?.controller.abort())); } + this.startIteration(); } protected get cancellationRequested(): boolean { @@ -126,7 +108,7 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat if (this.cancellationRequested) { this.terminalError = new CancellationError(); } else { - console.error('Error in OpenAI chat completion stream:', error); + this.logger.error('Error in OpenAI chat completion stream:', error); this.terminalError = error instanceof Error ? error : new Error(String(error)); } this.dispose(); @@ -247,28 +229,12 @@ export class ChatCompletionStreamingAsyncIterator implements AsyncIterableIterat return typeof result === 'string' ? result : JSON.stringify(result); } - protected handleIncoming(message: LanguageModelStreamResponsePart): void { - if (this.messageCache.length && this.requestQueue.length) { - throw new Error('Assertion error: cache and queue should not both be populated.'); - } - if (this.requestQueue.length) { - this.requestQueue.shift()!.resolve({ done: false, value: message }); - } else { - this.messageCache.push(message); - } - } - - dispose(): void { - this.done = true; - this.toDispose.dispose(); - // No more messages will arrive; resolve or reject any outstanding requests. - if (this.terminalError) { - this.requestQueue.forEach(request => request.reject(this.terminalError)); - } else { - this.requestQueue.forEach(request => request.resolve({ done: true, value: undefined })); - } - // Leave the message cache intact: if it is populated the request queue was empty, and we - // still want to deliver those messages when asked. - this.requestQueue.length = 0; - } } + +export const ChatCompletionStreamingAsyncIteratorFactory = Symbol('ChatCompletionStreamingAsyncIteratorFactory'); +/** + * Creates the iterator that drives a tool-calling chat-completion turn. Rebind in the DI container + * to substitute a customized {@link ChatCompletionStreamingAsyncIterator} implementation. + */ +export type ChatCompletionStreamingAsyncIteratorFactory = + (options: ChatCompletionToolLoopOptions) => ChatCompletionStreamingAsyncIterator; diff --git a/packages/ai-openai/src/node/openai-language-model.spec.ts b/packages/ai-openai/src/node/openai-language-model.spec.ts index dedca6267ab75..2fefbfde0d33b 100644 --- a/packages/ai-openai/src/node/openai-language-model.spec.ts +++ b/packages/ai-openai/src/node/openai-language-model.spec.ts @@ -15,9 +15,14 @@ // ***************************************************************************** import { expect } from 'chai'; -import { LanguageModelRequest, ReasoningSupport } from '@theia/ai-core'; -import { OpenAiModel, OpenAiModelUtils } from './openai-language-model'; +import { Container, injectable } from '@theia/core/shared/inversify'; +import { ILogger } from '@theia/core'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; +import { LanguageModelRequest, ReasoningSupport, ToolCallExecutor, ToolCallExecutorImpl } from '@theia/ai-core'; +import { OpenAiModel, OpenAiModelParams } from './openai-language-model'; +import { OpenAiModelUtils } from './openai-model-utils'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; +import { ChatCompletionStreamingAsyncIteratorFactory } from './openai-chat-completion-stream'; const GPT5_REASONING_SUPPORT: ReasoningSupport = { supportedLevels: ['off', 'minimal', 'low', 'medium', 'high', 'auto'], @@ -29,6 +34,7 @@ const O_SERIES_REASONING_SUPPORT: ReasoningSupport = { defaultLevel: 'auto' }; +@injectable() class TestableOpenAiModel extends OpenAiModel { public callGetSettings(request: LanguageModelRequest, forResponseApi: boolean = false): Record { return this.getSettings(request, forResponseApi); @@ -39,13 +45,31 @@ class TestableOpenAiModel extends OpenAiModel { } function createModel(modelId: string, reasoningSupport?: ReasoningSupport): TestableOpenAiModel { - return new TestableOpenAiModel( - 'test-id', modelId, { status: 'ready' }, true, - () => 'test-key', () => undefined, - false, undefined, undefined, - new OpenAiModelUtils(), new OpenAiResponseApiUtils(), - 'developer', 3, false, undefined, reasoningSupport - ); + const parent = new Container(); + parent.bind(OpenAiModelUtils).toSelf(); + parent.bind(OpenAiResponseApiUtils).toSelf(); + parent.bind(ToolCallExecutor).to(ToolCallExecutorImpl); + parent.bind(ILogger).to(MockLogger); + // These tests never issue a streaming request, so the iterator factory is never invoked. + const iteratorFactory: ChatCompletionStreamingAsyncIteratorFactory = () => { throw new Error('iterator not used in these tests'); }; + parent.bind(ChatCompletionStreamingAsyncIteratorFactory).toConstantValue(iteratorFactory); + parent.bind(TestableOpenAiModel).toSelf().inTransientScope(); + + const child = new Container(); + child.parent = parent; + child.bind(OpenAiModelParams).toConstantValue({ + id: 'test-id', + model: modelId, + status: { status: 'ready' }, + enableStreaming: true, + apiKey: () => 'test-key', + apiVersion: () => undefined, + supportsStructuredOutput: false, + url: undefined, + deployment: undefined, + reasoningSupport + }); + return child.get(TestableOpenAiModel); } describe('OpenAiModel reasoning translation', () => { diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index 3db0315abc9bd..5112ecb04972e 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -22,22 +22,21 @@ import { LanguageModelResponse, LanguageModelTextResponse, UserRequest, - ImageContent, LanguageModelStatus, ReasoningSupport, - ToolCallExecutor, - ToolCallExecutorImpl + ToolCallExecutor } from '@theia/ai-core'; +import { OpenAiModelUtils } from './openai-model-utils'; import { CancellationToken } from '@theia/core'; -import { injectable } from '@theia/core/shared/inversify'; +import { inject, injectable, postConstruct } from '@theia/core/shared/inversify'; import { OpenAI, AzureOpenAI } from 'openai'; -import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; +import { ChatCompletionMessageParam, ChatCompletionTool } from 'openai/resources'; import { StreamingAsyncIterator } from './openai-streaming-iterator'; -import { ChatCompletionStreamingAsyncIterator } from './openai-chat-completion-stream'; +import { ChatCompletionStreamingAsyncIteratorFactory } from './openai-chat-completion-stream'; import { OPENAI_PROVIDER_ID } from '../common'; import type { FinalRequestOptions } from 'openai/internal/request-options'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; -import { OpenAiResponseApiUtils, processSystemMessages } from './openai-response-api-utils'; +import { OpenAiResponseApiUtils } from './openai-response-api-utils'; import { openAiReasoningFor } from './openai-reasoning'; import { createProxyFetch } from '@theia/ai-core/lib/node'; @@ -86,11 +85,30 @@ export interface OpenAiModelParams { maxInputTokens?: number; } +export const OpenAiModelParams = Symbol('OpenAiModelParams'); + export const OpenAiLanguageModelFactory = Symbol('OpenAiLanguageModelFactory'); export type OpenAiLanguageModelFactory = (params: OpenAiModelParams) => OpenAiModel; +@injectable() export class OpenAiModel implements LanguageModel { + id: string; + model: string; + status: LanguageModelStatus; + enableStreaming: boolean; + apiKey: () => string | undefined; + apiVersion: () => string | undefined; + supportsStructuredOutput: boolean; + url: string | undefined; + deployment: string | undefined; + developerMessageSettings: DeveloperMessageSettings; + maxRetries: number; + useResponseApi: boolean; + proxy?: string; + reasoningSupport?: ReasoningSupport; + maxInputTokens?: number; + /** * The options for the OpenAI runner. */ @@ -101,37 +119,40 @@ export class OpenAiModel implements LanguageModel { maxChatCompletions: 100, }; - /** - * @param id the unique id for this language model. It will be used to identify the model in the UI. - * @param model the model id as it is used by the OpenAI API - * @param enableStreaming whether the streaming API shall be used - * @param apiKey a function that returns the API key to use for this model, called on each request - * @param apiVersion a function that returns the OpenAPI version to use for this model, called on each request - * @param developerMessageSettings how to handle system messages - * @param url the OpenAI API compatible endpoint where the model is hosted. If not provided the default OpenAI endpoint will be used. - * @param maxRetries the maximum number of retry attempts when a request fails - * @param useResponseApi whether to use the newer OpenAI Response API instead of the Chat Completion API - */ - constructor( - public readonly id: string, - public model: string, - public status: LanguageModelStatus, - public enableStreaming: boolean, - public apiKey: () => string | undefined, - public apiVersion: () => string | undefined, - public supportsStructuredOutput: boolean, - public url: string | undefined, - public deployment: string | undefined, - public openAiModelUtils: OpenAiModelUtils, - public responseApiUtils: OpenAiResponseApiUtils, - public developerMessageSettings: DeveloperMessageSettings = 'developer', - public maxRetries: number = 3, - public useResponseApi: boolean = false, - public proxy?: string, - public reasoningSupport?: ReasoningSupport, - public maxInputTokens?: number, - protected readonly toolCallExecutor: ToolCallExecutor = new ToolCallExecutorImpl() - ) { } + @inject(OpenAiModelParams) + protected readonly params: OpenAiModelParams; + + @inject(OpenAiModelUtils) + protected readonly openAiModelUtils: OpenAiModelUtils; + + @inject(OpenAiResponseApiUtils) + protected readonly responseApiUtils: OpenAiResponseApiUtils; + + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + + @inject(ChatCompletionStreamingAsyncIteratorFactory) + protected readonly chatCompletionStreamFactory: ChatCompletionStreamingAsyncIteratorFactory; + + @postConstruct() + protected init(): void { + const params = this.params; + this.id = params.id; + this.model = params.model; + this.status = params.status; + this.enableStreaming = params.enableStreaming; + this.apiKey = params.apiKey; + this.apiVersion = params.apiVersion; + this.supportsStructuredOutput = params.supportsStructuredOutput; + this.url = params.url; + this.deployment = params.deployment; + this.developerMessageSettings = params.developerMessageSettings ?? 'developer'; + this.maxRetries = params.maxRetries ?? 3; + this.useResponseApi = params.useResponseApi ?? false; + this.proxy = params.proxy; + this.reasoningSupport = params.reasoningSupport; + this.maxInputTokens = params.maxInputTokens; + } /** Reasoning-level translation lives in {@link openAiReasoningFor}. */ protected getSettings(request: LanguageModelRequest, forResponseApi: boolean = false): Record { @@ -171,7 +192,7 @@ export class OpenAiModel implements LanguageModel { if (tools) { return { - stream: new ChatCompletionStreamingAsyncIterator({ + stream: this.chatCompletionStreamFactory({ openai, model: this.model, request, @@ -302,128 +323,3 @@ export class OpenAiModel implements LanguageModel { return this.openAiModelUtils.processMessages(messages, this.developerMessageSettings, this.model); } } - -/** - * Utility class for processing messages for the OpenAI language model. - * - * Adopters can rebind this class to implement custom message processing behavior. - */ -@injectable() -export class OpenAiModelUtils { - - protected processSystemMessages( - messages: LanguageModelMessage[], - developerMessageSettings: DeveloperMessageSettings - ): LanguageModelMessage[] { - return processSystemMessages(messages, developerMessageSettings); - } - - protected toOpenAiRole( - message: LanguageModelMessage, - developerMessageSettings: DeveloperMessageSettings - ): 'developer' | 'user' | 'assistant' | 'system' { - if (message.actor === 'system') { - if (developerMessageSettings === 'user' || developerMessageSettings === 'system' || developerMessageSettings === 'developer') { - return developerMessageSettings; - } else { - return 'developer'; - } - } else if (message.actor === 'ai') { - return 'assistant'; - } - return 'user'; - } - - protected toOpenAIMessage( - message: LanguageModelMessage, - developerMessageSettings: DeveloperMessageSettings - ): ChatCompletionMessageParam { - if (LanguageModelMessage.isTextMessage(message)) { - return { - role: this.toOpenAiRole(message, developerMessageSettings), - content: message.text - }; - } - if (LanguageModelMessage.isToolUseMessage(message)) { - return { - role: 'assistant', - tool_calls: [{ id: message.id, function: { name: message.name, arguments: JSON.stringify(message.input) }, type: 'function' }] - }; - } - if (LanguageModelMessage.isToolResultMessage(message)) { - return { - role: 'tool', - tool_call_id: message.tool_use_id, - // content only supports text content so we need to stringify any potential data we have, e.g., images - content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content) - }; - } - if (LanguageModelMessage.isImageMessage(message) && message.actor === 'user') { - return { - role: 'user', - content: [{ - type: 'image_url', - image_url: { - url: - ImageContent.isBase64(message.image) ? - `data:${message.image.mimeType};base64,${message.image.base64data}` : - message.image.url - } - }] - }; - } - throw new Error(`Unknown message type:'${JSON.stringify(message)}'`); - } - - /** - * Processes the provided list of messages by applying system message adjustments and converting - * them to the format expected by the OpenAI API. - * - * Adopters can rebind this processing to implement custom behavior. - * - * @param messages the list of messages to process. - * @param developerMessageSettings how system and developer messages are handled during processing. - * @param model the OpenAI model identifier. Currently not used, but allows subclasses to implement model-specific behavior. - * @returns an array of messages formatted for the OpenAI API. - */ - processMessages( - messages: LanguageModelMessage[], - developerMessageSettings: DeveloperMessageSettings, - model?: string - ): ChatCompletionMessageParam[] { - const processed = this.processSystemMessages(messages, developerMessageSettings); - const converted = processed.filter(m => m.type !== 'thinking').map(m => this.toOpenAIMessage(m, developerMessageSettings)); - return this.mergeConsecutiveAssistantMessages(converted); - } - - protected mergeConsecutiveAssistantMessages(messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[] { - const result: ChatCompletionMessageParam[] = []; - for (const message of messages) { - const previous = result[result.length - 1]; - if (previous?.role === 'assistant' && message.role === 'assistant') { - const merged: ChatCompletionAssistantMessageParam = { ...previous, role: 'assistant' }; - - const previousContent = typeof previous.content === 'string' ? previous.content : undefined; - const nextContent = typeof message.content === 'string' ? message.content : undefined; - if (previousContent !== undefined && nextContent !== undefined) { - merged.content = `${previousContent}\n${nextContent}`; - } else if (nextContent !== undefined) { - merged.content = nextContent; - } else if (previousContent !== undefined) { - merged.content = previousContent; - } - - const toolCalls = [...(previous.tool_calls ?? []), ...(message.tool_calls ?? [])]; - if (toolCalls.length > 0) { - merged.tool_calls = toolCalls; - } - - result[result.length - 1] = merged; - } else { - result.push(message); - } - } - return result; - } - -} diff --git a/packages/ai-openai/src/node/openai-model-utils.spec.ts b/packages/ai-openai/src/node/openai-model-utils.spec.ts index 49934bdb1999e..06e687285baf4 100644 --- a/packages/ai-openai/src/node/openai-model-utils.spec.ts +++ b/packages/ai-openai/src/node/openai-model-utils.spec.ts @@ -14,7 +14,7 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** import { expect } from 'chai'; -import { OpenAiModelUtils } from './openai-language-model'; +import { OpenAiModelUtils } from './openai-model-utils'; import { LanguageModelMessage } from '@theia/ai-core'; import { OpenAiResponseApiUtils, recursiveStrictJSONSchema } from './openai-response-api-utils'; import type { JSONSchema, JSONSchemaDefinition } from 'openai/lib/jsonschema'; diff --git a/packages/ai-openai/src/node/openai-model-utils.ts b/packages/ai-openai/src/node/openai-model-utils.ts new file mode 100644 index 0000000000000..e08380c6b1bc6 --- /dev/null +++ b/packages/ai-openai/src/node/openai-model-utils.ts @@ -0,0 +1,145 @@ +// ***************************************************************************** +// Copyright (C) 2024 EclipseSource GmbH. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { ImageContent, LanguageModelMessage } from '@theia/ai-core'; +import { injectable } from '@theia/core/shared/inversify'; +import { ChatCompletionAssistantMessageParam, ChatCompletionMessageParam } from 'openai/resources'; +import { DeveloperMessageSettings } from './openai-language-model'; +import { processSystemMessages } from './openai-response-api-utils'; + +/** + * Utility class for processing messages for the OpenAI language model. + * + * Adopters can rebind this class to implement custom message processing behavior. + */ +@injectable() +export class OpenAiModelUtils { + + protected processSystemMessages( + messages: LanguageModelMessage[], + developerMessageSettings: DeveloperMessageSettings + ): LanguageModelMessage[] { + return processSystemMessages(messages, developerMessageSettings); + } + + protected toOpenAiRole( + message: LanguageModelMessage, + developerMessageSettings: DeveloperMessageSettings + ): 'developer' | 'user' | 'assistant' | 'system' { + if (message.actor === 'system') { + if (developerMessageSettings === 'user' || developerMessageSettings === 'system' || developerMessageSettings === 'developer') { + return developerMessageSettings; + } else { + return 'developer'; + } + } else if (message.actor === 'ai') { + return 'assistant'; + } + return 'user'; + } + + protected toOpenAIMessage( + message: LanguageModelMessage, + developerMessageSettings: DeveloperMessageSettings + ): ChatCompletionMessageParam { + if (LanguageModelMessage.isTextMessage(message)) { + return { + role: this.toOpenAiRole(message, developerMessageSettings), + content: message.text + }; + } + if (LanguageModelMessage.isToolUseMessage(message)) { + return { + role: 'assistant', + tool_calls: [{ id: message.id, function: { name: message.name, arguments: JSON.stringify(message.input) }, type: 'function' }] + }; + } + if (LanguageModelMessage.isToolResultMessage(message)) { + return { + role: 'tool', + tool_call_id: message.tool_use_id, + // content only supports text content so we need to stringify any potential data we have, e.g., images + content: typeof message.content === 'string' ? message.content : JSON.stringify(message.content) + }; + } + if (LanguageModelMessage.isImageMessage(message) && message.actor === 'user') { + return { + role: 'user', + content: [{ + type: 'image_url', + image_url: { + url: + ImageContent.isBase64(message.image) ? + `data:${message.image.mimeType};base64,${message.image.base64data}` : + message.image.url + } + }] + }; + } + throw new Error(`Unknown message type:'${JSON.stringify(message)}'`); + } + + /** + * Processes the provided list of messages by applying system message adjustments and converting + * them to the format expected by the OpenAI API. + * + * Adopters can rebind this processing to implement custom behavior. + * + * @param messages the list of messages to process. + * @param developerMessageSettings how system and developer messages are handled during processing. + * @param model the OpenAI model identifier. Currently not used, but allows subclasses to implement model-specific behavior. + * @returns an array of messages formatted for the OpenAI API. + */ + processMessages( + messages: LanguageModelMessage[], + developerMessageSettings: DeveloperMessageSettings, + model?: string + ): ChatCompletionMessageParam[] { + const processed = this.processSystemMessages(messages, developerMessageSettings); + const converted = processed.filter(m => m.type !== 'thinking').map(m => this.toOpenAIMessage(m, developerMessageSettings)); + return this.mergeConsecutiveAssistantMessages(converted); + } + + protected mergeConsecutiveAssistantMessages(messages: ChatCompletionMessageParam[]): ChatCompletionMessageParam[] { + const result: ChatCompletionMessageParam[] = []; + for (const message of messages) { + const previous = result[result.length - 1]; + if (previous?.role === 'assistant' && message.role === 'assistant') { + const merged: ChatCompletionAssistantMessageParam = { ...previous, role: 'assistant' }; + + const previousContent = typeof previous.content === 'string' ? previous.content : undefined; + const nextContent = typeof message.content === 'string' ? message.content : undefined; + if (previousContent !== undefined && nextContent !== undefined) { + merged.content = `${previousContent}\n${nextContent}`; + } else if (nextContent !== undefined) { + merged.content = nextContent; + } else if (previousContent !== undefined) { + merged.content = previousContent; + } + + const toolCalls = [...(previous.tool_calls ?? []), ...(message.tool_calls ?? [])]; + if (toolCalls.length > 0) { + merged.tool_calls = toolCalls; + } + + result[result.length - 1] = merged; + } else { + result.push(message); + } + } + return result; + } +} diff --git a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts index a3b86163b81b7..8c759ac8ea858 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.spec.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.spec.ts @@ -16,10 +16,20 @@ import { expect } from 'chai'; import { isToolCallResponsePart, isUsageResponsePart, LanguageModelStreamResponsePart, ToolCallExecutorImpl, UserRequest } from '@theia/ai-core'; +import { ILogger } from '@theia/core'; +import { Container } from '@theia/core/shared/inversify'; +import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; import { Deferred } from '@theia/core/lib/common/promise-util'; -import { OpenAiModelUtils } from './openai-language-model'; +import { OpenAiModelUtils } from './openai-model-utils'; import { OpenAiResponseApiUtils } from './openai-response-api-utils'; +function toolCallExecutor(): ToolCallExecutorImpl { + const container = new Container(); + container.bind(ILogger).to(MockLogger); + container.bind(ToolCallExecutorImpl).toSelf(); + return container.get(ToolCallExecutorImpl); +} + async function* toStream(events: unknown[]): AsyncIterable { for (const event of events) { yield event; @@ -36,7 +46,7 @@ function functionCallItem(id: string, name: string, args: string): unknown { describe('OpenAiResponseApiUtils', () => { it('emits per-iteration usage for Response API tool calls instead of accumulated usage', async () => { const utils = new OpenAiResponseApiUtils(); - utils.toolCallExecutor = new ToolCallExecutorImpl(); + utils.toolCallExecutor = toolCallExecutor(); const streams = [ [ { @@ -118,7 +128,7 @@ describe('OpenAiResponseApiUtils', () => { it('executes the tool calls of a single turn concurrently', async () => { const utils = new OpenAiResponseApiUtils(); - utils.toolCallExecutor = new ToolCallExecutorImpl(); + utils.toolCallExecutor = toolCallExecutor(); const streams = [ [ functionCallItem('call-a', 'a', '{}'), diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index 363f80def5e5d..517291e499da5 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -39,7 +39,8 @@ import type { ResponseStreamEvent } from 'openai/resources/responses/responses'; import type { ResponsesModel } from 'openai/resources/shared'; -import { DeveloperMessageSettings, OpenAiModelUtils } from './openai-language-model'; +import { DeveloperMessageSettings } from './openai-language-model'; +import type { OpenAiModelUtils } from './openai-model-utils'; import { JSONSchema, JSONSchemaDefinition } from 'openai/lib/jsonschema'; interface ToolCall { diff --git a/packages/ai-openai/src/node/openai-streaming-iterator.ts b/packages/ai-openai/src/node/openai-streaming-iterator.ts index 8ab06d5061858..b8ddf2ca68981 100644 --- a/packages/ai-openai/src/node/openai-streaming-iterator.ts +++ b/packages/ai-openai/src/node/openai-streaming-iterator.ts @@ -15,24 +15,18 @@ // ***************************************************************************** import { LanguageModelStreamResponsePart, ToolCallResult, ToolCallTextResult } from '@theia/ai-core'; -import { CancellationError, CancellationToken, Disposable, DisposableCollection } from '@theia/core'; -import { Deferred } from '@theia/core/lib/common/promise-util'; +import { CancellationError, CancellationToken } from '@theia/core'; import { ChatCompletionStream, ChatCompletionStreamEvents } from 'openai/lib/ChatCompletionStream'; import { ChatCompletionContentPartText } from 'openai/resources'; +import { AbstractStreamingResponseIterator } from './streaming-response-iterator'; -type IterResult = IteratorResult; - -export class StreamingAsyncIterator implements AsyncIterableIterator, Disposable { - protected readonly requestQueue = new Array>(); - protected readonly messageCache = new Array(); - protected done = false; - protected terminalError: Error | undefined = undefined; - protected readonly toDispose = new DisposableCollection(); +export class StreamingAsyncIterator extends AbstractStreamingResponseIterator { constructor( protected readonly stream: ChatCompletionStream, cancellationToken?: CancellationToken, ) { + super(); this.registerStreamListener('error', error => { console.error('Error in OpenAI chat completion stream:', error); this.terminalError = error; @@ -97,46 +91,6 @@ export class StreamingAsyncIterator implements AsyncIterableIterator { return this; } - - next(): Promise { - if (this.messageCache.length && this.requestQueue.length) { - throw new Error('Assertion error: cache and queue should not both be populated.'); - } - // Deliver all the messages we got, even if we've since terminated. - if (this.messageCache.length) { - return Promise.resolve({ - done: false, - value: this.messageCache.shift()! - }); - } else if (this.terminalError) { - return Promise.reject(this.terminalError); - } else if (this.done) { - return Promise.resolve({ - done: true, - value: undefined - }); - } else { - const toQueue = new Deferred(); - this.requestQueue.push(toQueue); - return toQueue.promise; - } - } - - protected handleIncoming(message: LanguageModelStreamResponsePart): void { - if (this.messageCache.length && this.requestQueue.length) { - throw new Error('Assertion error: cache and queue should not both be populated.'); - } - if (this.requestQueue.length) { - this.requestQueue.shift()!.resolve({ - done: false, - value: message - }); - } else { - this.messageCache.push(message); - } - } - protected registerStreamListener(eventType: Event, handler: ChatCompletionStreamEvents[Event], once?: boolean): void { if (once) { this.stream.once(eventType, handler); @@ -145,19 +99,6 @@ export class StreamingAsyncIterator implements AsyncIterableIterator this.stream.off(eventType, handler) }); } - - dispose(): void { - this.done = true; - this.toDispose.dispose(); - // We will be receiving no more messages. Any outstanding requests have to be handled. - if (this.terminalError) { - this.requestQueue.forEach(request => request.reject(this.terminalError)); - } else { - this.requestQueue.forEach(request => request.resolve({ done: true, value: undefined })); - } - // Leave the message cache alone - if it was populated, then the request queue was empty, but we'll still try to deliver the messages if asked. - this.requestQueue.length = 0; - } } function tryParseToolResult(result: string | ChatCompletionContentPartText[]): ToolCallResult { diff --git a/packages/ai-openai/src/node/streaming-response-iterator.ts b/packages/ai-openai/src/node/streaming-response-iterator.ts new file mode 100644 index 0000000000000..2cae186e5e208 --- /dev/null +++ b/packages/ai-openai/src/node/streaming-response-iterator.ts @@ -0,0 +1,83 @@ +// ***************************************************************************** +// Copyright (C) 2026 EclipseSource GmbH and others. +// +// This program and the accompanying materials are made available under the +// terms of the Eclipse Public License v. 2.0 which is available at +// http://www.eclipse.org/legal/epl-2.0. +// +// This Source Code may also be made available under the following Secondary +// Licenses when the conditions for such availability set forth in the Eclipse +// Public License v. 2.0 are satisfied: GNU General Public License, version 2 +// with the GNU Classpath Exception which is available at +// https://www.gnu.org/software/classpath/license.html. +// +// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 +// ***************************************************************************** + +import { LanguageModelStreamResponsePart } from '@theia/ai-core'; +import { Disposable, DisposableCollection } from '@theia/core'; +import { Deferred } from '@theia/core/lib/common/promise-util'; + +type IterResult = IteratorResult; + +/** + * Base class for async iterators that deliver {@link LanguageModelStreamResponsePart}s produced + * asynchronously from some upstream source. It implements the producer/consumer queue: subclasses + * push parts via {@link handleIncoming} as they arrive and signal completion by setting + * {@link terminalError} (for failures) and/or calling {@link dispose}. Parts produced before the + * consumer asks for them are buffered, and requests made before parts are available are queued. + */ +export abstract class AbstractStreamingResponseIterator implements AsyncIterableIterator, Disposable { + protected readonly requestQueue = new Array>(); + protected readonly messageCache = new Array(); + protected done = false; + protected terminalError: Error | undefined = undefined; + protected readonly toDispose = new DisposableCollection(); + + [Symbol.asyncIterator](): AsyncIterableIterator { + return this; + } + + next(): Promise { + if (this.messageCache.length && this.requestQueue.length) { + throw new Error('Assertion error: cache and queue should not both be populated.'); + } + // Deliver all the messages we got, even if we've since terminated. + if (this.messageCache.length) { + return Promise.resolve({ done: false, value: this.messageCache.shift()! }); + } else if (this.terminalError) { + return Promise.reject(this.terminalError); + } else if (this.done) { + return Promise.resolve({ done: true, value: undefined }); + } else { + const toQueue = new Deferred(); + this.requestQueue.push(toQueue); + return toQueue.promise; + } + } + + protected handleIncoming(message: LanguageModelStreamResponsePart): void { + if (this.messageCache.length && this.requestQueue.length) { + throw new Error('Assertion error: cache and queue should not both be populated.'); + } + if (this.requestQueue.length) { + this.requestQueue.shift()!.resolve({ done: false, value: message }); + } else { + this.messageCache.push(message); + } + } + + dispose(): void { + this.done = true; + this.toDispose.dispose(); + // No more messages will arrive; resolve or reject any outstanding requests. + if (this.terminalError) { + this.requestQueue.forEach(request => request.reject(this.terminalError)); + } else { + this.requestQueue.forEach(request => request.resolve({ done: true, value: undefined })); + } + // Leave the message cache intact: if it is populated the request queue was empty, and we + // still want to deliver those messages when asked. + this.requestQueue.length = 0; + } +} From 395fa57eec60325ffb5b095105c03cfadfe1b1e5 Mon Sep 17 00:00:00 2001 From: "Christian W. Damus" Date: Tue, 1 Sep 2026 18:11:37 +0000 Subject: [PATCH 10/10] fix(ai): tighten chat-completion tool-loop cancellation and address review feedback Addresses the EclipseSourceAI review and additional off-line review: - ai-openai: withhold the "open" tool-call stream part until both id and name are known, so a call streamed with the name after the id never gets stuck nameless in the UI/replay. - ai-openai: fold the duplicated producer/consumer queue of ResponseApiToolCallIterator onto the extracted AbstractStreamingResponseIterator base class. - ai-openai: have ChatCompletionStreamingAsyncIterator inject ToolCallExecutor itself instead of threading it through ChatCompletionToolLoopOptions and every provider model. - ai-openai: terminate ChatCompletionStreamingAsyncIterator promptly on cancellation instead of only aborting the provider stream, closing the race where a pending next() could stall behind an uninterruptible tool handler, where cancellation could still let queued tool calls start, and where a late tool result could leak out after cancellation. - ai-ollama: thread the request's CancellationToken through processToolCalls into ToolCallExecutor and forward ToolInvocationContext through the tool handler adapter, which previously dropped both. - ai-ollama: make processToolCalls protected (matching the other provider models) instead of casting around its private visibility in tests. - CHANGELOG.md: list the additional breaking API changes the review flagged as missing (renamed ModelFactory symbols, moved OpenAiModelUtils, removed toolCallExecutor fields). Signed-off-by: Christian W. Damus --- CHANGELOG.md | 4 + .../src/node/copilot-language-model.ts | 5 - .../src/node/ollama-language-model.ts | 14 +- packages/ai-ollama/src/package.spec.ts | 34 ++++- .../openai-chat-completion-stream.spec.ts | 120 +++++++++++++++++- .../src/node/openai-chat-completion-stream.ts | 49 ++++++- .../src/node/openai-language-model.ts | 5 - .../src/node/openai-response-api-utils.ts | 71 +---------- 8 files changed, 204 insertions(+), 98 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 079dd2d6de3ed..59a253c5ba03d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ - [ai-openai, ai-copilot] `OpenAiModel.createTools()` and `CopilotLanguageModel.createTools()` now return `ChatCompletionTool[]` instead of `RunnableToolFunctionWithoutParse[]`, because the OpenAI SDK `runTools` runner is no longer used [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-copilot] removed the `protected runnerOptions` field from `CopilotLanguageModel`; its only purpose was the `maxChatCompletions` turn cap, which is gone now that the OpenAI SDK `runTools` runner is unused, so the tool loop runs until the model stops requesting tools. Subclasses that read or overrode `runnerOptions` must adapt. `OpenAiModel.runnerOptions` is retained, but its `maxChatCompletions` now bounds only the Response API path, not the Chat Completions tool loop [#17623](https://github.com/eclipse-theia/theia/pull/17623) - [ai-anthropic, ai-google, ai-ollama, ai-openai, ai-copilot] the provider language model classes (`AnthropicModel`, `GoogleModel`, `OllamaModel`, `OpenAiModel`, `CopilotLanguageModel`) no longer expose public constructors; they are `@injectable` and receive their configuration through an injected `ModelParams` object (a symbol) plus injected service dependencies. Instantiate them via the corresponding `LanguageModelFactory` (or the DI container) instead of `new`, and drop constructor overrides in subclasses [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai] renamed the exported `OpenAiModelFactory` symbol/type to `OpenAiLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-ollama] renamed the exported `OllamaModelFactory` symbol/type to `OllamaLanguageModelFactory` [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai] moved `OpenAiModelUtils` out of `openai-language-model.ts` into a new `openai-model-utils.ts` module; deep imports of the old path break [#17623](https://github.com/eclipse-theia/theia/pull/17623) +- [ai-openai, ai-copilot] removed the `protected toolCallExecutor` field from `OpenAiModel` and `CopilotLanguageModel`; `ChatCompletionStreamingAsyncIterator` now injects `ToolCallExecutor` itself instead of receiving it via `ChatCompletionToolLoopOptions`, whose `toolCallExecutor` property is also gone [#17623](https://github.com/eclipse-theia/theia/pull/17623) ## 1.75.0 - 8/27/2026 diff --git a/packages/ai-copilot/src/node/copilot-language-model.ts b/packages/ai-copilot/src/node/copilot-language-model.ts index ee1ba46bd2ce4..2114790a75165 100644 --- a/packages/ai-copilot/src/node/copilot-language-model.ts +++ b/packages/ai-copilot/src/node/copilot-language-model.ts @@ -24,7 +24,6 @@ import { LanguageModelResponse, LanguageModelStatus, LanguageModelTextResponse, - ToolCallExecutor, UserRequest } from '@theia/ai-core'; import { CancellationToken, ILogger } from '@theia/core'; @@ -72,9 +71,6 @@ export class CopilotLanguageModel implements LanguageModel { @inject(CopilotLanguageModelParams) protected readonly params: CopilotLanguageModelParams; - @inject(ToolCallExecutor) - protected readonly toolCallExecutor: ToolCallExecutor; - @inject(ChatCompletionStreamingAsyncIteratorFactory) protected readonly chatCompletionStreamFactory: ChatCompletionStreamingAsyncIteratorFactory; @@ -132,7 +128,6 @@ export class CopilotLanguageModel implements LanguageModel { settings, tools, maxRetries: this.maxRetries, - toolCallExecutor: this.toolCallExecutor, cancellationToken }) }; diff --git a/packages/ai-ollama/src/node/ollama-language-model.ts b/packages/ai-ollama/src/node/ollama-language-model.ts index e85c252e32857..3fb56fa1bd1d0 100644 --- a/packages/ai-ollama/src/node/ollama-language-model.ts +++ b/packages/ai-ollama/src/node/ollama-language-model.ts @@ -26,6 +26,7 @@ import { ToolCall, ToolCallExecutor, ToolCallResult, + ToolInvocationContext, ToolRequest, ToolRequestParameterProperty, ToolRequestParametersProperties, @@ -223,7 +224,7 @@ export class OllamaModel implements LanguageModel { yield { tool_calls: toolCallsForResponse }; // Now handle the tool calls - const processedToolCallsForResponse = await that.processToolCalls(toolCallsForResponse, chatRequest); + const processedToolCallsForResponse = await that.processToolCalls(toolCallsForResponse, chatRequest, cancellation); yield { tool_calls: processedToolCallsForResponse }; // Continue the conversation with tool results @@ -356,7 +357,7 @@ export class OllamaModel implements LanguageModel { }); const preparedToolCalls = this.createToolCalls(toolCalls, lastUpdated); - await this.processToolCalls(preparedToolCalls, chatRequest); + await this.processToolCalls(preparedToolCalls, chatRequest, cancellation); if (cancellation?.isCancellationRequested) { return { text: '' }; } @@ -393,18 +394,19 @@ export class OllamaModel implements LanguageModel { return toolCallsForResponse; } - private async processToolCalls(toolCalls: ToolCall[], chatRequest: ExtendedChatRequest): Promise { + protected async processToolCalls(toolCalls: ToolCall[], chatRequest: ExtendedChatRequest, cancellation?: CancellationToken): Promise { const tools: ToolWithHandler[] = chatRequest.tools ?? []; const toolRequests: ToolRequest[] = tools.map(tool => ({ id: tool.function.name ?? '', name: tool.function.name ?? '', parameters: { type: 'object', properties: {} }, - handler: async argString => (await tool.handler(argString)) as ToolCallResult + handler: async (argString, ctx) => (await tool.handler(argString, ctx)) as ToolCallResult })); const results = await this.toolCallExecutor.executeToolCalls( toolCalls.map(call => ({ id: call.id ?? call.function!.name!, name: call.function!.name!, arguments: call.function!.arguments! })), - toolRequests + toolRequests, + { cancellationToken: cancellation } ); // Build the messages and response entries from the input-ordered results so that the @@ -574,7 +576,7 @@ export class OllamaModel implements LanguageModel { * Extended Tool containing a handler * @see Tool */ -type ToolWithHandler = Tool & { handler: (arg_string: string) => Promise }; +type ToolWithHandler = Tool & { handler: (arg_string: string, ctx?: ToolInvocationContext) => Promise }; /** * Extended chat request with mandatory messages and ToolWithHandler tools diff --git a/packages/ai-ollama/src/package.spec.ts b/packages/ai-ollama/src/package.spec.ts index 0878768114f06..b7e35f0ff6716 100644 --- a/packages/ai-ollama/src/package.spec.ts +++ b/packages/ai-ollama/src/package.spec.ts @@ -14,10 +14,10 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ToolCall, ToolCallExecutor, ToolCallExecutorImpl, ToolRequest } from '@theia/ai-core'; +import { ToolCall, ToolCallExecutor, ToolCallExecutorImpl, ToolInvocationContext, ToolRequest } from '@theia/ai-core'; import { Deferred } from '@theia/core/lib/common/promise-util'; import { Container, injectable } from '@theia/core/shared/inversify'; -import { ILogger } from '@theia/core'; +import { CancellationToken, CancellationTokenSource, ILogger } from '@theia/core'; import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; import { OllamaModel, OllamaModelParams } from './node/ollama-language-model'; import { Tool } from 'ollama'; @@ -71,6 +71,28 @@ describe('ai-ollama package', () => { const result = await model.runProcessToolCalls([{ id: '1', function: { name: 'missing', arguments: '{}' } }], chatRequest); expect(result[0].result).to.equal('error: Tool not found'); }); + + it('forwards the request cancellation token into the tool invocation context', async () => { + const model = createModel(); + const source = new CancellationTokenSource(); + let observedToken: CancellationToken | undefined; + const chatRequest = { + messages: [], + tools: [{ + function: { name: 'a' }, + handler: async (_argString: string, ctx?: ToolInvocationContext) => { + observedToken = ToolInvocationContext.getCancellationToken(ctx); + return 'a-result'; + } + }] + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any; + const toolCalls: ToolCall[] = [{ id: '1', function: { name: 'a', arguments: '{}' } }]; + + await model.runProcessToolCalls(toolCalls, chatRequest, source.token); + + expect(observedToken).to.equal(source.token); + }); }); function createModel(): OllamaModelUnderTest { @@ -92,14 +114,14 @@ function createModel(): OllamaModelUnderTest { @injectable() class OllamaModelUnderTest extends OllamaModel { - override toOllamaTool(tool: ToolRequest): Tool & { handler: (arg_string: string) => Promise } { + override toOllamaTool(tool: ToolRequest): Tool & { handler: (arg_string: string, ctx?: ToolInvocationContext) => Promise } { return super.toOllamaTool(tool); } - // Exposes the private processToolCalls for testing concurrent tool execution. - runProcessToolCalls(toolCalls: ToolCall[], chatRequest: unknown): Promise { + // Exposes the protected processToolCalls for testing concurrent tool execution and cancellation forwarding. + runProcessToolCalls(toolCalls: ToolCall[], chatRequest: unknown, cancellation?: CancellationToken): Promise { // eslint-disable-next-line @typescript-eslint/no-explicit-any - return (this as any).processToolCalls(toolCalls, chatRequest); + return this.processToolCalls(toolCalls, chatRequest as any, cancellation); } } function createToolRequest(): ToolRequest { diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts index 1504bceb17317..45236495159dc 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.spec.ts @@ -16,7 +16,7 @@ import { expect } from 'chai'; import * as sinon from 'sinon'; -import { CancellationTokenSource, ILogger } from '@theia/core'; +import { CancellationError, CancellationTokenSource, ILogger } from '@theia/core'; import { Container, injectable, interfaces } from '@theia/core/shared/inversify'; import { MockLogger } from '@theia/core/lib/common/test/mock-logger'; import { Deferred } from '@theia/core/lib/common/promise-util'; @@ -28,6 +28,7 @@ import { LanguageModelStreamResponsePart, ToolInvocation, ToolCallExecutionOptions, + ToolCallExecutor, ToolCallOutcome, ToolCallExecutorImpl, ToolRequest, @@ -59,6 +60,16 @@ function toolChunk(index: number, id: string, name: string, args: string): any { return { choices: [{ delta: { tool_calls: [{ index, id, function: { name, arguments: args } }] } }] }; } +/** A delta carrying only the tool call's ID, as a provider might stream it before the function name. */ +function toolIdOnlyChunk(index: number, id: string): any { + return { choices: [{ delta: { tool_calls: [{ index, id }] } }] }; +} + +/** A delta carrying only (a fragment of) the tool call's function name. */ +function toolNameChunk(index: number, name: string): any { + return { choices: [{ delta: { tool_calls: [{ index, function: { name } }] } }] }; +} + function usageChunk(inputTokens: number, outputTokens: number): any { return { choices: [{ delta: {} }], usage: { prompt_tokens: inputTokens, completion_tokens: outputTokens } }; } @@ -96,7 +107,11 @@ function withLogger(constructor: interfaces.Newable): T { return container.get(constructor); } -function makeIterator(openai: FakeOpenAi, overrides: Partial = {}): ChatCompletionStreamingAsyncIterator { +function makeIterator( + openai: FakeOpenAi, + overrides: Partial = {}, + toolCallExecutor: ToolCallExecutor = withLogger(ToolCallExecutorImpl) +): ChatCompletionStreamingAsyncIterator { const defaultRequest: UserRequest = { sessionId: 'session', requestId: 'request', @@ -110,11 +125,11 @@ function makeIterator(openai: FakeOpenAi, overrides: Partial => new Promise(resolve => setImmediate(resolve)); +/** Like {@link drain}, but resolves with whatever was collected so far plus the error, instead of rejecting. */ +async function collectUntilSettled( + iterator: AsyncIterableIterator +): Promise<{ parts: LanguageModelStreamResponsePart[]; error?: unknown }> { + const parts: LanguageModelStreamResponsePart[] = []; + try { + // eslint-disable-next-line no-constant-condition + while (true) { + const next = await iterator.next(); + if (next.done) { + return { parts }; + } + parts.push(next.value); + } + } catch (error) { + return { parts, error }; + } +} + /** Captures the batches of tool calls passed to the executor, to assert single-turn batching. */ @injectable() class RecordingExecutor extends ToolCallExecutorImpl { @@ -179,7 +213,7 @@ describe('ChatCompletionStreamingAsyncIterator', () => { new FakeStream([textChunk('done')]) ]); - const parts = await drain(makeIterator(openai, { request, toolCallExecutor: executor })); + const parts = await drain(makeIterator(openai, { request }, executor)); // Both tool calls were handed to the executor together (single turn => single batch). expect(executor.batches).to.have.lengthOf(1); @@ -214,6 +248,26 @@ describe('ChatCompletionStreamingAsyncIterator', () => { expect(finished?.result).to.equal('a-result'); }); + it('withholds the open tool-call part until the function name is known, even if the ID streams first', async () => { + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => 'a-result')] + }; + const openai = fakeOpenAi([ + new FakeStream([toolIdOnlyChunk(0, 'call-a'), toolNameChunk(0, 'a'), toolChunk(0, 'call-a', '', '{}')]), + new FakeStream([textChunk('done')]) + ]); + + const parts = await drain(makeIterator(openai, { request })); + const opens = parts.filter(isToolCallResponsePart).flatMap(p => p.tool_calls).filter(c => c.finished === false); + + // Exactly one "open" part, and it must already carry the (by then complete) name, never an empty one. + expect(opens).to.have.lengthOf(1); + expect(opens[0].function?.name).to.equal('a'); + }); + it('threads the assistant tool_calls and a matching tool message into the next turn', async () => { const request: UserRequest = { sessionId: 'session', @@ -286,4 +340,62 @@ describe('ChatCompletionStreamingAsyncIterator', () => { gate.resolve(); await drained; }); + + it('terminates promptly on cancellation even while an uninterruptible tool handler is still running, and drops its late result', async () => { + const handlerStarted = new Deferred(); + const handlerGate = new Deferred(); + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', async () => { handlerStarted.resolve(); return handlerGate.promise; })] + }; + const openai = fakeOpenAi([new FakeStream([toolChunk(0, 'call-a', 'a', '{}')])]); + const source = new CancellationTokenSource(); + const iterator = makeIterator(openai, { request, cancellationToken: source.token }); + + const settled = collectUntilSettled(iterator); + // The handler ignores its cancellation token and just keeps running: it is up to the iterator, not the + // handler, to stop delivering parts to the consumer. + await handlerStarted.promise; + + source.cancel(); + const { parts, error } = await settled; + + expect(error).to.be.instanceOf(CancellationError); + // Cancellation happened while the handler was still running: no finished tool-call part can have been seen. + expect(parts.filter(isToolCallResponsePart).flatMap(p => p.tool_calls).some(c => c.finished)).to.equal(false); + + // The handler eventually resolves, but the iterator is done: it must not surface a finished part for it, + // only keep rejecting with the same terminal error. + handlerGate.resolve('late-result'); + await flush(); + const after = await iterator.next().catch(caught => caught); + expect(after).to.be.instanceOf(CancellationError); + }); + + it('never starts collected tool calls when cancellation is observed right after the stream ends', async () => { + const handler = sinon.stub().resolves('a-result'); + const request: UserRequest = { + sessionId: 'session', + requestId: 'request', + messages: [{ actor: 'user', type: 'text', text: 'hi' }], + tools: [toolRequest('a', handler)] + }; + const source = new CancellationTokenSource(); + // Cancel lazily, exactly when the stream is drained of its one chunk: after the stream itself has + // finished (so `processStream()` is about to return) but before the iterator moves on to executing the + // tool calls it collected from it. + function* chunksThenCancel(): Iterator { + yield toolChunk(0, 'call-a', 'a', '{}'); + source.cancel(); + } + const stream = new FakeStream(chunksThenCancel() as unknown as any[]); + const openai = fakeOpenAi([stream]); + + const { error } = await collectUntilSettled(makeIterator(openai, { request, cancellationToken: source.token })); + + expect(error).to.be.instanceOf(CancellationError); + expect(handler.called).to.equal(false); + }); }); diff --git a/packages/ai-openai/src/node/openai-chat-completion-stream.ts b/packages/ai-openai/src/node/openai-chat-completion-stream.ts index 95a1c8ba78951..d392f80ee592b 100644 --- a/packages/ai-openai/src/node/openai-chat-completion-stream.ts +++ b/packages/ai-openai/src/node/openai-chat-completion-stream.ts @@ -14,7 +14,7 @@ // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0 // ***************************************************************************** -import { ToolCallExecutor, ToolCallResult, UserRequest } from '@theia/ai-core'; +import { LanguageModelStreamResponsePart, ToolCallExecutor, ToolCallResult, UserRequest } from '@theia/ai-core'; import { CancellationError, CancellationToken, ILogger } from '@theia/core'; import { inject, injectable, named, postConstruct } from '@theia/core/shared/inversify'; import { OpenAI } from 'openai'; @@ -53,7 +53,6 @@ export interface ChatCompletionToolLoopOptions { readonly settings: Record; readonly tools: ChatCompletionTool[]; readonly maxRetries: number; - readonly toolCallExecutor: ToolCallExecutor; readonly cancellationToken?: CancellationToken; } @@ -75,6 +74,9 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo @inject(ChatCompletionToolLoopOptions) protected readonly options: ChatCompletionToolLoopOptions; + @inject(ToolCallExecutor) + protected readonly toolCallExecutor: ToolCallExecutor; + @inject(ILogger) @named('ai-openai:ChatCompletionStreamingAsyncIterator') protected readonly logger: ILogger; @@ -82,7 +84,7 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo protected init(): void { this.messages = [...this.options.messages]; if (this.options.cancellationToken) { - this.toDispose.push(this.options.cancellationToken.onCancellationRequested(() => this.currentStream?.controller.abort())); + this.toDispose.push(this.options.cancellationToken.onCancellationRequested(() => this.cancel())); } this.startIteration(); } @@ -91,10 +93,28 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo return !!this.options.cancellationToken?.isCancellationRequested; } + /** + * Aborts the active provider stream and immediately terminates the iterator, so a `next()` call pending on + * tool execution (which is only cooperatively cancelled via the token forwarded to handlers) settles promptly + * instead of stalling until an uninterruptible handler eventually finishes. + */ + protected cancel(): void { + this.currentStream?.controller.abort(); + if (this.done) { + return; + } + this.terminalError = new CancellationError(); + this.dispose(); + } + protected async startIteration(): Promise { try { while (!this.cancellationRequested) { const { assistantText, toolCalls } = await this.processStream(); + if (this.cancellationRequested) { + // Cancelled while the stream was wrapping up: don't start executing the collected tool calls. + break; + } if (toolCalls.length === 0) { // No tool calls: the conversation is complete. this.dispose(); @@ -102,7 +122,8 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo } await this.executeAndAppendToolCalls(assistantText, toolCalls); } - // Cancelled before the model stopped requesting tools. + // Cancelled before the model stopped requesting tools. `cancel()` (via the cancellation listener) has + // already disposed the iterator; this is a no-op unless cancellation was observed some other way. this.dispose(); } catch (error) { if (this.cancellationRequested) { @@ -176,8 +197,10 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo if (toolCallDelta.function?.name) { slot.name += toolCallDelta.function.name; } - // Open the tool call (emit a `finished: false` part) as soon as we know its ID. - if (!slot.opened && slot.id) { + // Open the tool call (emit a `finished: false` part) once we know both its ID and name. `merge` on the + // receiving `ToolCallChatResponseContentImpl` never updates the name afterwards, so opening early with an + // empty name would leave the call nameless for good. + if (!slot.opened && slot.id && slot.name) { slot.opened = true; this.handleIncoming({ tool_calls: [{ id: slot.id, finished: false, function: { name: slot.name, arguments: '' } }] }); } @@ -189,8 +212,20 @@ export class ChatCompletionStreamingAsyncIterator extends AbstractStreamingRespo } } + /** + * Drops parts arriving after the iterator is done (in particular after {@link cancel} disposed it while tool + * execution was still in flight), so an uninterruptible handler that finishes late cannot surface a + * finished tool-call part - or any other part - as if it were still part of the live turn. + */ + protected override handleIncoming(message: LanguageModelStreamResponsePart): void { + if (this.done) { + return; + } + super.handleIncoming(message); + } + protected async executeAndAppendToolCalls(assistantText: string, toolCalls: CollectedToolCall[]): Promise { - const results = await this.options.toolCallExecutor.executeToolCalls( + const results = await this.toolCallExecutor.executeToolCalls( toolCalls.map(toolCall => ({ id: toolCall.id, name: toolCall.name, arguments: toolCall.arguments || '{}' })), this.options.request.tools, { cancellationToken: this.options.cancellationToken } diff --git a/packages/ai-openai/src/node/openai-language-model.ts b/packages/ai-openai/src/node/openai-language-model.ts index 20618001867e5..ead69f449b530 100644 --- a/packages/ai-openai/src/node/openai-language-model.ts +++ b/packages/ai-openai/src/node/openai-language-model.ts @@ -24,7 +24,6 @@ import { UserRequest, LanguageModelStatus, ReasoningSupport, - ToolCallExecutor, resolveCompactionTokenThreshold, resolveServerSideCompaction, ServerToolDescriptor @@ -142,9 +141,6 @@ export class OpenAiModel implements LanguageModel { @inject(OpenAiResponseApiUtils) protected readonly responseApiUtils: OpenAiResponseApiUtils; - @inject(ToolCallExecutor) - protected readonly toolCallExecutor: ToolCallExecutor; - @inject(ChatCompletionStreamingAsyncIteratorFactory) protected readonly chatCompletionStreamFactory: ChatCompletionStreamingAsyncIteratorFactory; @@ -221,7 +217,6 @@ export class OpenAiModel implements LanguageModel { settings, tools, maxRetries: this.maxRetries, - toolCallExecutor: this.toolCallExecutor, cancellationToken }) }; diff --git a/packages/ai-openai/src/node/openai-response-api-utils.ts b/packages/ai-openai/src/node/openai-response-api-utils.ts index de142bdd4aff2..562e5b4ac7cbd 100644 --- a/packages/ai-openai/src/node/openai-response-api-utils.ts +++ b/packages/ai-openai/src/node/openai-response-api-utils.ts @@ -26,7 +26,6 @@ import { UserRequest } from '@theia/ai-core'; import { CancellationToken, nls, unreachable, ILogger } from '@theia/core'; -import { Deferred } from '@theia/core/lib/common/promise-util'; import { inject, injectable, named } from '@theia/core/shared/inversify'; import { OpenAI } from 'openai'; import type { RunnerOptions } from 'openai/lib/AbstractChatCompletionRunner'; @@ -46,6 +45,7 @@ import type { ResponsesModel } from 'openai/resources/shared'; import { DeveloperMessageSettings } from './openai-language-model'; import type { OpenAiModelUtils } from './openai-model-utils'; import { OPENAI_WEB_SEARCH, OPENAI_WEB_SEARCH_REPLAY_DATA_KEY } from './openai-server-tools'; +import { AbstractStreamingResponseIterator } from './streaming-response-iterator'; export const OPENAI_FUNCTION_CALL_REASONING_DATA_KEY = 'openAiFunctionCallReasoning'; @@ -393,11 +393,7 @@ export class OpenAiResponseApiUtils { * Iterator for handling Response API streaming with tool calls. * Based on the pattern from openai-streaming-iterator.ts but adapted for Response API. */ -class ResponseApiToolCallIterator implements AsyncIterableIterator { - protected readonly requestQueue = new Array>>(); - protected readonly messageCache = new Array(); - protected done = false; - protected terminalError: Error | undefined = undefined; +class ResponseApiToolCallIterator extends AbstractStreamingResponseIterator { // Current iteration state protected currentInput: ResponseInputItem[]; @@ -426,6 +422,7 @@ class ResponseApiToolCallIterator implements AsyncIterableIterator { - return this; - } - - async next(): Promise> { - if (this.messageCache.length && this.requestQueue.length) { - throw new Error('Assertion error: cache and queue should not both be populated.'); - } - - // Deliver all the messages we got, even if we've since terminated. - if (this.messageCache.length) { - return { - done: false, - value: this.messageCache.shift()! - }; - } else if (this.terminalError) { - throw this.terminalError; - } else if (this.done) { - return { - done: true, - value: undefined - }; - } else { - const deferred = new Deferred>(); - this.requestQueue.push(deferred); - return deferred.promise; - } - } - protected async startIteration(): Promise { try { while (this.iteration < this.maxIterations && !this.cancellationToken?.isCancellationRequested) { @@ -475,7 +443,7 @@ class ResponseApiToolCallIterator implements AsyncIterableIterator { - this.done = true; - - // Resolve any outstanding requests - if (this.terminalError) { - this.requestQueue.forEach(request => request.reject(this.terminalError)); - } else { - this.requestQueue.forEach(request => request.resolve({ done: true, value: undefined })); - } - this.requestQueue.length = 0; - } } export function processSystemMessages(