From 71b39be620f2aaafef62582be867c42a7a6a3a2b Mon Sep 17 00:00:00 2001 From: Desel72 <6442298+Desel72@users.noreply.github.com> Date: Wed, 4 Mar 2026 18:53:13 +0000 Subject: [PATCH 1/2] feat: merge question_confirm and summary_task into single LLM call --- backend/app/model/chat.py | 21 ++ backend/app/service/chat_service.py | 226 ++++++++++++++---- .../tests/app/service/test_chat_service.py | 69 +++++- 3 files changed, 262 insertions(+), 54 deletions(-) diff --git a/backend/app/model/chat.py b/backend/app/model/chat.py index 8f13a5aa4..01f683c64 100644 --- a/backend/app/model/chat.py +++ b/backend/app/model/chat.py @@ -46,6 +46,27 @@ class QuestionAnalysisResult(BaseModel): ) +class TaskAnalysisResult(BaseModel): + """Result of combined task analysis (complexity + summary). + + Merges question_confirm (complexity) and summary_task (name/summary) + into one LLM call. See issue #1427. + """ + + is_complex: bool = Field( + description="True if complex task requiring tools/workforce, " + "False if simple question answerable directly." + ) + task_name: str | None = Field( + default=None, + description="Short descriptive task name. Only when is_complex=True.", + ) + summary: str | None = Field( + default=None, + description="Concise task summary. Only when is_complex=True.", + ) + + McpServers = dict[Literal["mcpServers"], dict[str, dict]] diff --git a/backend/app/service/chat_service.py b/backend/app/service/chat_service.py index 41c912ab6..390c0638a 100644 --- a/backend/app/service/chat_service.py +++ b/backend/app/service/chat_service.py @@ -14,6 +14,7 @@ import asyncio import datetime +import json import logging import platform from pathlib import Path @@ -43,7 +44,14 @@ from app.agent.toolkit.skill_toolkit import SkillToolkit from app.agent.toolkit.terminal_toolkit import TerminalToolkit from app.agent.tools import get_mcp_tools, get_toolkits -from app.model.chat import Chat, NewAgent, Status, TaskContent, sse_json +from app.model.chat import ( + Chat, + NewAgent, + Status, + TaskAnalysisResult, + TaskContent, + sse_json, +) from app.service.task import ( Action, ActionDecomposeProgressData, @@ -501,17 +509,28 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock): is_complex_task: bool if len(attaches_to_use) > 0: is_complex_task = True + task_lock._prefetched_summary = None logger.info( "[NEW-QUESTION] Has attachments" ", treating as complex task" ) else: - is_complex_task = await question_confirm( + analysis = await analyze_task( question_agent, question, task_lock ) + is_complex_task = analysis.is_complex + if ( + is_complex_task + and analysis.task_name + and analysis.summary + ): + task_lock._prefetched_summary = ( + f"{analysis.task_name}|{analysis.summary}" + ) + else: + task_lock._prefetched_summary = None logger.info( - "[NEW-QUESTION] question_confirm" - " result: is_complex=" + "[NEW-QUESTION] analyze_task result: is_complex=" f"{is_complex_task}" ) @@ -736,57 +755,73 @@ async def run_decomposition(): except Exception: pass - # Generate task summary - summary_task_agent = task_summary_agent(options) - try: - summary_task_content = await asyncio.wait_for( - summary_task( - summary_task_agent, camel_task - ), - timeout=10, - ) - task_lock.summary_generated = True - except TimeoutError: - logger.warning( - "summary_task timeout", - extra={ - "project_id": options.project_id, - "task_id": options.task_id, - }, - ) - task_lock.summary_generated = True - content_preview = ( - camel_task.content - if hasattr(camel_task, "content") - else "" - ) - if content_preview is None: - content_preview = "" - if len(content_preview) > 80: - cp = content_preview[:80] - summary_task_content = cp + "..." - else: - summary_task_content = content_preview - summary_task_content = ( - f"Task|{summary_task_content}" - ) - except Exception: + # Generate task summary (use prefetched if available) + prefetched = getattr( + task_lock, "_prefetched_summary", None + ) + if prefetched: + summary_task_content = prefetched task_lock.summary_generated = True - content_preview = ( - camel_task.content - if hasattr(camel_task, "content") - else "" + logger.debug( + "Using prefetched task summary from " + "analyze_task (issue #1427)" ) - if content_preview is None: - content_preview = "" - if len(content_preview) > 80: - cp = content_preview[:80] - summary_task_content = cp + "..." - else: - summary_task_content = content_preview - summary_task_content = ( - f"Task|{summary_task_content}" + else: + summary_task_agent = task_summary_agent( + options ) + try: + summary_task_content = ( + await asyncio.wait_for( + summary_task( + summary_task_agent, + camel_task, + ), + timeout=10, + ) + ) + task_lock.summary_generated = True + except TimeoutError: + logger.warning( + "summary_task timeout", + extra={ + "project_id": options.project_id, + "task_id": options.task_id, + }, + ) + task_lock.summary_generated = True + content_preview = ( + camel_task.content + if hasattr(camel_task, "content") + else "" + ) + if content_preview is None: + content_preview = "" + if len(content_preview) > 80: + cp = content_preview[:80] + summary_task_content = cp + "..." + else: + summary_task_content = content_preview + summary_task_content = ( + f"Task|{summary_task_content}" + ) + except Exception: + task_lock.summary_generated = True + content_preview = ( + camel_task.content + if hasattr(camel_task, "content") + else "" + ) + if content_preview is None: + content_preview = "" + if len(content_preview) > 80: + cp = content_preview[:80] + summary_task_content = cp + "..." + else: + summary_task_content = content_preview + summary_task_content = ( + f"Task|{summary_task_content}" + ) state_holder["summary_task"] = summary_task_content try: @@ -1927,6 +1962,93 @@ def add_sub_tasks( return added_tasks +async def analyze_task( + agent: ListenChatAgent, + question: str, + task_lock: TaskLock | None = None, +) -> TaskAnalysisResult: + """Analyze user query in one LLM call: complexity + task name/summary. + + Merges question_confirm and summary_task into a single request (issue #1427). + Falls back to question_confirm when structured output parsing fails. + """ + context_prompt = "" + if task_lock: + context_prompt = build_conversation_context( + task_lock, header="=== Previous Conversation ===" + ) + + full_prompt = f"""{context_prompt}User Query: {question} + +Analyze this user query and return structured JSON with these fields: +- is_complex (boolean): True if this requires tools, code execution, file operations, + multi-step planning, or creating/modifying content. False if it can be answered + directly (greetings, fact queries, clarifications, status checks). +- task_name (string, only when is_complex): A short descriptive name for the task. +- summary (string, only when is_complex): A concise summary of the task's main points. + +Examples of complex: "create a file", "search for X", "implement feature Y". +Examples of simple: "hello", "what is X?", "how are you?" + +Return valid JSON only, no other text.""" + + try: + resp = agent.step(full_prompt, response_format=TaskAnalysisResult) + + if not resp or not resp.msgs or len(resp.msgs) == 0: + logger.warning( + "analyze_task: no response, falling back to question_confirm" + ) + is_complex = await question_confirm(agent, question, task_lock) + return TaskAnalysisResult( + is_complex=is_complex, task_name=None, summary=None + ) + + content = resp.msgs[0].content + parsed = getattr(resp.msgs[0], "parsed", None) + + if parsed is not None and isinstance(parsed, TaskAnalysisResult): + logger.info( + "analyze_task: got structured result", + extra={"is_complex": parsed.is_complex}, + ) + return parsed + + if content: + try: + data = json.loads(content.strip()) + result = TaskAnalysisResult( + is_complex=bool(data.get("is_complex", True)), + task_name=data.get("task_name"), + summary=data.get("summary"), + ) + logger.info( + "analyze_task: parsed JSON from content", + extra={"is_complex": result.is_complex}, + ) + return result + except (json.JSONDecodeError, TypeError): + pass + + logger.warning( + "analyze_task: could not parse response, falling back to question_confirm" + ) + is_complex = await question_confirm(agent, question, task_lock) + return TaskAnalysisResult( + is_complex=is_complex, task_name=None, summary=None + ) + + except Exception as e: + logger.warning( + f"analyze_task failed: {e}, falling back to question_confirm", + exc_info=True, + ) + is_complex = await question_confirm(agent, question, task_lock) + return TaskAnalysisResult( + is_complex=is_complex, task_name=None, summary=None + ) + + async def question_confirm( agent: ListenChatAgent, prompt: str, task_lock: TaskLock | None = None ) -> bool: diff --git a/backend/tests/app/service/test_chat_service.py b/backend/tests/app/service/test_chat_service.py index ab666f3fd..f64b9cf07 100644 --- a/backend/tests/app/service/test_chat_service.py +++ b/backend/tests/app/service/test_chat_service.py @@ -18,9 +18,10 @@ from camel.tasks import Task from camel.tasks.task import TaskState -from app.model.chat import Chat, NewAgent +from app.model.chat import Chat, NewAgent, TaskAnalysisResult from app.service.chat_service import ( add_sub_tasks, + analyze_task, build_context_for_workforce, collect_previous_task_context, construct_workforce, @@ -646,6 +647,65 @@ async def test_summary_task(self, mock_camel_agent): ) mock_camel_agent.step.assert_called_once() + @pytest.mark.asyncio + async def test_analyze_task_structured_output(self, mock_camel_agent): + """Test analyze_task with structured parsed output.""" + mock_camel_agent.step.return_value.msgs[0].parsed = TaskAnalysisResult( + is_complex=True, + task_name="Web App Creation", + summary="Create a modern web application with auth", + ) + mock_camel_agent.step.return_value.msgs[0].content = "{}" + mock_camel_agent.chat_history = [] + + result = await analyze_task( + mock_camel_agent, "Create a web app with authentication" + ) + + assert result.is_complex is True + assert result.task_name == "Web App Creation" + assert result.summary == "Create a modern web application with auth" + mock_camel_agent.step.assert_called_once() + + @pytest.mark.asyncio + async def test_analyze_task_json_content(self, mock_camel_agent): + """Test analyze_task parses JSON from content when parsed is None.""" + mock_camel_agent.step.return_value.msgs[0].parsed = None + mock_camel_agent.step.return_value.msgs[ + 0 + ].content = '{"is_complex": false, "task_name": null, "summary": null}' + mock_camel_agent.chat_history = [] + + result = await analyze_task(mock_camel_agent, "hello") + + assert result.is_complex is False + assert result.task_name is None + assert result.summary is None + + @pytest.mark.asyncio + async def test_analyze_task_fallback_to_question_confirm( + self, mock_camel_agent + ): + """Test analyze_task falls back to question_confirm when parsing fails.""" + first_response = MagicMock() + first_response.msgs = [MagicMock()] + first_response.msgs[0].parsed = None + first_response.msgs[0].content = "invalid json" + + second_response = MagicMock() + second_response.msgs = [MagicMock()] + second_response.msgs[0].content = "yes" + + mock_camel_agent.step.side_effect = [first_response, second_response] + mock_camel_agent.chat_history = [] + + result = await analyze_task(mock_camel_agent, "Create a file") + + assert result.is_complex is True + assert result.task_name is None + assert result.summary is None + assert mock_camel_agent.step.call_count == 2 + @pytest.mark.asyncio async def test_new_agent_model_creation(self, sample_chat_data): """Test new_agent_model creates agent with proper configuration.""" @@ -893,7 +953,12 @@ async def test_step_solve_basic_workflow( "app.service.chat_service.task_summary_agent" ) as mock_summary_agent, patch( - "app.service.chat_service.question_confirm", return_value=True + "app.service.chat_service.analyze_task", + return_value=TaskAnalysisResult( + is_complex=True, + task_name="Test Task", + summary="Test summary", + ), ), patch( "app.service.chat_service.summary_task", From abeeaed535f51a47f5c55bcf8ccb6e4c2d0d2d16 Mon Sep 17 00:00:00 2001 From: Desel72 <6442298+Desel72@users.noreply.github.com> Date: Thu, 5 Mar 2026 10:08:31 +0000 Subject: [PATCH 2/2] address review feedback --- backend/app/model/chat.py | 19 +++++- backend/app/service/chat_service.py | 61 ++++++++++++------- .../tests/app/service/test_chat_service.py | 41 +++++++++++++ 3 files changed, 99 insertions(+), 22 deletions(-) diff --git a/backend/app/model/chat.py b/backend/app/model/chat.py index 01f683c64..2a7144a05 100644 --- a/backend/app/model/chat.py +++ b/backend/app/model/chat.py @@ -59,13 +59,30 @@ class TaskAnalysisResult(BaseModel): ) task_name: str | None = Field( default=None, - description="Short descriptive task name. Only when is_complex=True.", + description="Short descriptive task name. For complex tasks: describe " + "the task. For simple questions: short label (e.g. 'Greeting', " + "'Fact Query').", ) summary: str | None = Field( default=None, description="Concise task summary. Only when is_complex=True.", ) + def has_valid_prefetch_data(self) -> bool: + """Return True if prefetched summary can be used (skip summary_task). + + When is_complex=True, task_name and summary must both be non-empty. + Otherwise we must fall back to calling summary_task. + """ + if not self.is_complex: + return False + return bool( + self.task_name + and self.task_name.strip() + and self.summary + and self.summary.strip() + ) + McpServers = dict[Literal["mcpServers"], dict[str, dict]] diff --git a/backend/app/service/chat_service.py b/backend/app/service/chat_service.py index 390c0638a..0319239b3 100644 --- a/backend/app/service/chat_service.py +++ b/backend/app/service/chat_service.py @@ -510,6 +510,7 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock): if len(attaches_to_use) > 0: is_complex_task = True task_lock._prefetched_summary = None + task_lock._fallback_task_name = "Task" logger.info( "[NEW-QUESTION] Has attachments" ", treating as complex task" @@ -519,16 +520,18 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock): question_agent, question, task_lock ) is_complex_task = analysis.is_complex - if ( - is_complex_task - and analysis.task_name - and analysis.summary - ): + if analysis.has_valid_prefetch_data(): task_lock._prefetched_summary = ( f"{analysis.task_name}|{analysis.summary}" ) + task_lock._fallback_task_name = None else: task_lock._prefetched_summary = None + task_lock._fallback_task_name = ( + analysis.task_name + if analysis.task_name + else "Task" + ) logger.info( "[NEW-QUESTION] analyze_task result: is_complex=" f"{is_complex_task}" @@ -566,21 +569,29 @@ async def step_solve(options: Chat, request: Request, task_lock: TaskLock): task_lock.add_conversation("assistant", answer_content) + wait_confirm_payload: dict[str, Any] = { + "content": answer_content, + "question": question, + } + if analysis.task_name: + wait_confirm_payload["task_name"] = ( + analysis.task_name + ) yield sse_json( "wait_confirm", - {"content": answer_content, "question": question}, + wait_confirm_payload, ) except Exception as e: logger.error(f"Error generating simple answer: {e}") - yield sse_json( - "wait_confirm", - { - "content": "I encountered an error" - " while processing " - "your question.", - "question": question, - }, - ) + error_payload: dict[str, Any] = { + "content": "I encountered an error" + " while processing " + "your question.", + "question": question, + } + if analysis.task_name: + error_payload["task_name"] = analysis.task_name + yield sse_json("wait_confirm", error_payload) # Clean up empty folder if it was created for this task if ( @@ -802,9 +813,12 @@ async def run_decomposition(): summary_task_content = cp + "..." else: summary_task_content = content_preview - summary_task_content = ( - f"Task|{summary_task_content}" + fallback_name = getattr( + task_lock, + "_fallback_task_name", + "Task", ) + summary_task_content = f"{fallback_name}|{summary_task_content}" except Exception: task_lock.summary_generated = True content_preview = ( @@ -819,9 +833,12 @@ async def run_decomposition(): summary_task_content = cp + "..." else: summary_task_content = content_preview - summary_task_content = ( - f"Task|{summary_task_content}" + fallback_name = getattr( + task_lock, + "_fallback_task_name", + "Task", ) + summary_task_content = f"{fallback_name}|{summary_task_content}" state_holder["summary_task"] = summary_task_content try: @@ -1969,7 +1986,6 @@ async def analyze_task( ) -> TaskAnalysisResult: """Analyze user query in one LLM call: complexity + task name/summary. - Merges question_confirm and summary_task into a single request (issue #1427). Falls back to question_confirm when structured output parsing fails. """ context_prompt = "" @@ -1984,8 +2000,11 @@ async def analyze_task( - is_complex (boolean): True if this requires tools, code execution, file operations, multi-step planning, or creating/modifying content. False if it can be answered directly (greetings, fact queries, clarifications, status checks). -- task_name (string, only when is_complex): A short descriptive name for the task. +- task_name (string, always): A short descriptive name. For complex tasks: describe + the task. For simple questions: use a short label (e.g. "Greeting", "Fact Query", + "Clarification"). - summary (string, only when is_complex): A concise summary of the task's main points. + Omit or use null when is_complex is false. Examples of complex: "create a file", "search for X", "implement feature Y". Examples of simple: "hello", "what is X?", "how are you?" diff --git a/backend/tests/app/service/test_chat_service.py b/backend/tests/app/service/test_chat_service.py index f64b9cf07..99abc3e36 100644 --- a/backend/tests/app/service/test_chat_service.py +++ b/backend/tests/app/service/test_chat_service.py @@ -706,6 +706,47 @@ async def test_analyze_task_fallback_to_question_confirm( assert result.summary is None assert mock_camel_agent.step.call_count == 2 + def test_task_analysis_result_has_valid_prefetch_data(self): + """Test has_valid_prefetch_data post-validation.""" + assert ( + TaskAnalysisResult( + is_complex=True, + task_name="Task", + summary="Summary", + ).has_valid_prefetch_data() + is True + ) + + assert ( + TaskAnalysisResult( + is_complex=True, task_name=None, summary="Summary" + ).has_valid_prefetch_data() + is False + ) + + assert ( + TaskAnalysisResult( + is_complex=True, task_name="Task", summary=None + ).has_valid_prefetch_data() + is False + ) + + assert ( + TaskAnalysisResult( + is_complex=True, task_name="", summary="Summary" + ).has_valid_prefetch_data() + is False + ) + + assert ( + TaskAnalysisResult( + is_complex=False, + task_name="Greeting", + summary=None, + ).has_valid_prefetch_data() + is False + ) + @pytest.mark.asyncio async def test_new_agent_model_creation(self, sample_chat_data): """Test new_agent_model creates agent with proper configuration."""