Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion lib/chains/llm_chain/modes/until_tool_used.ex
Original file line number Diff line number Diff line change
Expand Up @@ -53,11 +53,18 @@ defmodule LangChain.Chains.LLMChain.Modes.UntilToolUsed do
end

defp do_run(chain, opts) do
# Order matters: `check_max_runs` must come AFTER `check_until_tool`,
# otherwise the very LLM call that successfully invokes the target
# tool can have its result discarded — `call_llm` increments the run
# counter, the check fires and short-circuits the pipeline before
# `execute_tools`/`check_until_tool` get to see the result. With this
# order, a successful run terminates first; `check_max_runs` is a
# no-op on terminal pipeline states.
{:continue, chain}
|> call_llm()
|> check_max_runs(opts)
|> execute_tools()
|> check_until_tool(opts)
|> check_max_runs(opts)
|> continue_or_done(&do_run/2, opts)
end
end
27 changes: 27 additions & 0 deletions test/chains/llm_chain_test.exs
Original file line number Diff line number Diff line change
Expand Up @@ -3201,6 +3201,33 @@ defmodule LangChain.Chains.LLMChainTest do
assert error.message == "Exceeded maximum number of runs (3/3)"
end

# Regression: a successful tool call on the LLM call that hits the
# max_runs ceiling must terminate with success, not be discarded.
# Previously `check_max_runs` ran before `check_until_tool` and a
# max_runs=1 + first-call success path errored out instead of
# returning the tool result.
test "succeeds when the target tool is called on the same LLM call that hits max_runs",
%{greet: greet, sync: do_thing} do
expect(ChatOpenAI, :call, fn _model, _messages, _tools ->
{:ok,
new_function_calls!([
ToolCall.new!(%{call_id: "call_doThing", name: "do_thing", arguments: nil})
])}
end)

{:ok, updated_chain, tool_result} =
%{llm: ChatOpenAI.new!(%{stream: false}), verbose: false}
|> LLMChain.new!()
|> LLMChain.add_tools([greet, do_thing])
|> LLMChain.add_message(Message.new_system!())
|> LLMChain.add_message(Message.new_user!("Call do_thing right away."))
|> LLMChain.run_until_tool_used("do_thing", max_runs: 1)

assert updated_chain.last_message.role == :tool
assert %ToolResult{is_error: false} = tool_result
assert tool_result.name == "do_thing"
end

test "returns error when tool_name does not exist in available tools", %{greet: greet} do
{:error, _updated_chain, error} =
%{llm: ChatOpenAI.new!(%{stream: false}), verbose: false}
Expand Down