fix(sdk): resolve async response secrets outside the event loop

.pr/secret-masking/README.md

@@ -0,0 +1,24 @@
+# Live Canvas: async secret masking
+
+| Before #4967 | After #4967 |
+| --- | --- |
+| ![Before: Canvas waits for the blocked secret lookup](before.gif) | ![After: reply finishes and Settings remains responsive](after.gif) |
+
+The same text response reached its final assistant event in **31.014 seconds before** and **0.957 seconds after**. Before, Canvas remained on its loading skeleton and Settings navigation stalled; the server logged a 30-second `ReadTimeout`. It then persisted the harmless fixture value unmasked. After, the final event contains `Evidence value: <secret-hidden>`, the conversation finishes, and Settings loads normally. Canvas hides the mask's HTML-like marker when rendering Markdown; the underlying event verifies masking.
+
+After SDK: `bcd72405db0be3c7d8823920eef49870ce9d98d4`.
+
+## Reproduce
+
+1. Start a local Agent Server and a Canvas static build pointing its `/api`, `/server_info`, `/health`, and `/sockets` routes at that server. Use distinct empty persistence and conversation directories for each SDK version. Do not include #5017, whose eager secret materialization would hide this lazy-lookup trigger.
+2. Start the [provider fixture](provider.py.txt): `mkdir -p /tmp/secret-evidence; echo masked > /tmp/secret-evidence/mode; python3 provider.py.txt --root /tmp/secret-evidence --port 19118`.
+3. Through Canvas onboarding, choose OpenHands; Advanced model `openai/gpt-4o-mini`, base URL `http://127.0.0.1:19118/v1`, dummy API key. Close onboarding after saving.
+4. Through Settings → Secrets, add `EVIDENCE_SECRET` with the public dummy value `evidence-only-placeholder-4967`.
+5. Start a new chat: `SDK_STALL_EVIDENCE: reply with the ordinary text supplied by the evidence provider. Do not call tools.` The fixture returns text without tools, exercising async response masking. Navigate to Settings while waiting.
+6. Repeat on the PR head. Canvas's unchanged `LookupSecret` request points back to the same real Agent Server; the fix allows that server to answer its own lookup.
+
+The browser, Canvas build, Agent Server, SDK execution, HTTP transport, and event persistence are real. A disclosed local OpenAI-compatible HTTP provider supplies deterministic responses; no external LM or GitHub credentials are used. Both isolated backends start with empty settings. Model/secret changes and messages were entered through Canvas UI, with no API configuration seeding.
+
+Canvas build: `a3c7915db5f47800f5240a25987b2af75b0d8d04`. SDK base: `c37007429be8b4465a83487dc1fd0914df0ea734`. Only the owning SDK PR changes between each before/after pair. Dependencies: LiteLLM 1.93.0, HTTPX 0.28.1, OpenAI 2.33.0, Pydantic 2.12.5.
+
+The GIFs use selected original screenshots, two seconds per frame; playback time is condensed and is not a latency measurement. Measured timing and actual conversation configuration are in [evidence.json](evidence.json). Original screenshots and traces are retained under `/home/gneubig/work/factory-state/evidence/sdk-stalls/`. Each comparison ran sequentially with one SDK/Canvas/provider stack. All three services were stopped after capture.

.pr/secret-masking/evidence.json

@@ -0,0 +1,42 @@
+{
+  "captured_at": "2026-09-13",
+  "before_manifest": {
+    "case": "4967-before",
+    "variant": "base",
+    "sdk_commit": "c37007429be8b4465a83487dc1fd0914df0ea734",
+    "canvas_commit": "a3c7915db5f47800f5240a25987b2af75b0d8d04",
+    "canvas_url": "http://127.0.0.1:9108",
+    "provider": "disclosed local HTTP fixture; no external requests",
+    "started_at": 1789311240.769199,
+    "memory_available_kib": 7462060,
+    "config": "isolated empty settings; dummy credentials only; UI config to follow"
+  },
+  "after_manifest": {
+    "case": "4967-after",
+    "variant": "4967",
+    "sdk_commit": "bcd72405db0be3c7d8823920eef49870ce9d98d4",
+    "canvas_commit": "a3c7915db5f47800f5240a25987b2af75b0d8d04",
+    "canvas_url": "http://127.0.0.1:9108",
+    "provider": "disclosed local HTTP fixture; no external requests",
+    "started_at": 1789311501.6147518,
+    "memory_available_kib": 6943924,
+    "config": "isolated empty settings; dummy credentials only; UI config to follow"
+  },
+  "results": {
+    "4967-before": {
+      "model_request_to_final_event_seconds": 31.014,
+      "assistant_value_masked": false
+    },
+    "4967-after": {
+      "model_request_to_final_event_seconds": 0.957,
+      "assistant_value_masked": true
+    }
+  },
+  "dependencies": {
+    "litellm": "1.93.0",
+    "httpx": "0.28.1",
+    "openai": "2.33.0",
+    "pydantic": "2.12.5",
+    "uvicorn": "0.37.0"
+  }
+}

.pr/secret-masking/provider.py.txt

@@ -0,0 +1,38 @@
+"""Disclosed local OpenAI-compatible fault fixture; no external model requests."""
+import argparse, json, time, threading
+from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
+from pathlib import Path
+p=argparse.ArgumentParser();p.add_argument('--port',type=int,default=19118);p.add_argument('--root',type=Path,required=True);a=p.parse_args()
+a.root.mkdir(parents=True,exist_ok=True)
+lock=threading.Lock()
+def log(**row):
+ with lock:
+  with (a.root/'provider-events.jsonl').open('a') as f:f.write(json.dumps({'time':time.time(),**row})+'\n')
+class Handler(BaseHTTPRequestHandler):
+ def log_message(self,*args):pass
+ def do_GET(self):
+  body=json.dumps({'object':'list','data':[{'id':'gpt-4o-mini','object':'model','owned_by':'evidence-fixture'}]}).encode();self.send_response(200);self.send_header('Content-Type','application/json');self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body)
+ def do_POST(self):
+  data=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0'))));mode=(a.root/'mode').read_text().strip();stream=data.get('stream',False)
+  messages=data.get('messages',[]);text=json.dumps(messages);is_title='generates concise' in text and 'descriptive titles' in text;target='SDK_STALL_EVIDENCE' in text and not is_title
+  log(event='request',path=self.path,mode=mode,stream=stream,target=target)
+  content='SDK evidence conversation' if is_title else ('EVIDENCE_RECOVERED' if mode=='healthy' else 'Evidence value: evidence-only-placeholder-4967')
+  if mode in ('continuous','idle') and target and stream:
+   self.send_response(200);self.send_header('Content-Type','text/event-stream');self.send_header('Cache-Control','no-cache');self.end_headers()
+   try:
+    for i in range(240):
+     chunk={'id':'chatcmpl-evidence','object':'chat.completion.chunk','created':int(time.time()),'model':'gpt-4o-mini','choices':[{'index':0,'delta':{'content':' waiting'},'finish_reason':None}]}
+     self.wfile.write(('data: '+json.dumps(chunk)+'\n\n').encode());self.wfile.flush();log(event='chunk',index=i)
+     if mode=='idle':time.sleep(30);break
+     time.sleep(.25)
+   except (BrokenPipeError,ConnectionResetError):log(event='client_closed')
+   return
+  if stream:
+   self.send_response(200);self.send_header('Content-Type','text/event-stream');self.end_headers()
+   for delta,finish in [({'role':'assistant','content':content},None),({},'stop')]:
+    chunk={'id':'chatcmpl-evidence','object':'chat.completion.chunk','created':int(time.time()),'model':'gpt-4o-mini','choices':[{'index':0,'delta':delta,'finish_reason':finish}]};self.wfile.write(('data: '+json.dumps(chunk)+'\n\n').encode())
+   self.wfile.write(b'data: [DONE]\n\n');self.wfile.flush()
+  else:
+   body=json.dumps({'id':'chatcmpl-evidence','object':'chat.completion','created':int(time.time()),'model':'gpt-4o-mini','choices':[{'index':0,'message':{'role':'assistant','content':content},'finish_reason':'stop'}],'usage':{'prompt_tokens':1,'completion_tokens':1,'total_tokens':2}}).encode();self.send_response(200);self.send_header('Content-Type','application/json');self.send_header('Content-Length',str(len(body)));self.end_headers();self.wfile.write(body)
+  log(event='response_finished')
+ThreadingHTTPServer(('127.0.0.1',a.port),Handler).serve_forever()

openhands-sdk/openhands/sdk/agent/agent.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import json
 import re
 from collections.abc import Callable
@@ -1022,6 +1023,18 @@ async def _astep(
 
         message: Message = llm_response.message
         response_type = classify_response(message)
+        if response_type is not LLMResponseType.TOOL_CALLS:
+            # Resolve outside the event loop and state lock. A lookup may call
+            # this server, and update_secrets() may register another source while
+            # we await it. Repeat until the registry is stable under the lock.
+            while True:
+                sources = dict(state.secret_registry.secret_sources)
+                async with conversation._released_state_lock_during_io():
+                    message = await asyncio.to_thread(
+                        self._mask_secrets, message, conversation
+                    )
+                if sources == state.secret_registry.secret_sources:
+                    break
 
         match response_type:
             case LLMResponseType.TOOL_CALLS:
@@ -1030,7 +1043,13 @@ async def _astep(
                 )
             case LLMResponseType.CONTENT:
                 self._handle_content_response(
-                    message, llm_response, conversation, state, on_event, stream
+                    message,
+                    llm_response,
+                    conversation,
+                    state,
+                    on_event,
+                    stream,
+                    mask_secrets=False,
                 )
             case LLMResponseType.REASONING_ONLY | LLMResponseType.EMPTY:
                 self._handle_no_content_response(
@@ -1041,6 +1060,7 @@ async def _astep(
                     on_event,
                     stream,
                     response_type=response_type,
+                    mask_secrets=False,
                 )
 
     def _requires_user_confirmation(

openhands-sdk/openhands/sdk/agent/response_dispatch.py

@@ -253,9 +253,18 @@ def _handle_content_response(
         state: ConversationState,
         on_event: ConversationCallbackType,
         stream: StreamContext | None = None,
+        *,
+        mask_secrets: bool = True,
     ) -> None:
         """Handle LLM response with text content — finishes conversation."""
-        self._emit_message_event(message, llm_response, conversation, on_event, stream)
+        self._emit_message_event(
+            message,
+            llm_response,
+            conversation,
+            on_event,
+            stream,
+            mask_secrets=mask_secrets,
+        )
         self._maybe_emit_vllm_tokens(llm_response, on_event)
         logger.debug("LLM produced a message response - awaits user input")
         state.execution_status = ConversationExecutionStatus.FINISHED
@@ -270,6 +279,7 @@ def _handle_no_content_response(
         stream: StreamContext | None = None,
         *,
         response_type: LLMResponseType,
+        mask_secrets: bool = True,
     ) -> None:
         """Handle LLM response with no user-facing content.
 
@@ -279,7 +289,14 @@ def _handle_no_content_response(
         """
         if response_type is LLMResponseType.EMPTY:
             logger.warning("LLM produced empty response - continuing agent loop")
-        self._emit_message_event(message, llm_response, conversation, on_event, stream)
+        self._emit_message_event(
+            message,
+            llm_response,
+            conversation,
+            on_event,
+            stream,
+            mask_secrets=mask_secrets,
+        )
         self._maybe_emit_vllm_tokens(llm_response, on_event)
         self._send_corrective_nudge(on_event)
 
@@ -290,6 +307,8 @@ def _emit_message_event(
         conversation: LocalConversation,
         on_event: ConversationCallbackType,
         stream: StreamContext | None = None,
+        *,
+        mask_secrets: bool = True,
     ) -> MessageEvent:
         """Create and emit a MessageEvent, running critic if configured.
 
@@ -302,7 +321,9 @@ def _emit_message_event(
         msg_event = MessageEvent(
             **minted,
             source="agent",
-            llm_message=self._mask_secrets(message, conversation),
+            llm_message=self._mask_secrets(message, conversation)
+            if mask_secrets
+            else message,
             llm_response_id=llm_response.id,
         )
         if self.critic is not None and self.critic.mode == "finish_and_message":

openhands-sdk/openhands/sdk/conversation/impl/local_conversation.py

@@ -2769,8 +2769,8 @@ def update_secrets(self, secrets: Mapping[str, SecretValue]) -> None:
                      SecretValue = str | Callable[[], str]. Callables are invoked lazily
                      when a command references the secret key.
         """
-        secret_registry = self._state.secret_registry
-        secret_registry.update_secrets(secrets)
+        with self._state:
+            self._state.secret_registry.update_secrets(secrets)
         logger.info(f"Added {len(secrets)} secrets to conversation")
 
     def set_security_analyzer(self, analyzer: SecurityAnalyzerBase | None) -> None:

tests/sdk/agent/test_async_secret_masking.py

@@ -0,0 +1,167 @@
+import asyncio
+
+import pytest
+
+from openhands.sdk import Agent, Conversation
+from openhands.sdk.event import ActionEvent, MessageEvent
+from openhands.sdk.llm import Message, TextContent
+from openhands.sdk.llm.message import MessageToolCall
+from openhands.sdk.secret import LookupSecret
+from openhands.sdk.testing import TestLLM
+
+
+@pytest.mark.asyncio
+async def test_async_response_masks_loopback_lookup_secret_without_blocking(tmp_path):
+    requested = asyncio.Event()
+
+    async def serve_secret(reader, writer):
+        await reader.readuntil(b"\r\n\r\n")
+        requested.set()
+        body = b"loopback-secret-value"
+        writer.write(
+            b"HTTP/1.1 200 OK\r\nContent-Length: "
+            + str(len(body)).encode()
+            + b"\r\nConnection: close\r\n\r\n"
+            + body
+        )
+        await writer.drain()
+        writer.close()
+        await writer.wait_closed()
+
+    server = await asyncio.start_server(serve_secret, "127.0.0.1", 0)
+    port = server.sockets[0].getsockname()[1]
+    llm = TestLLM.from_messages(
+        [
+            Message(
+                role="assistant",
+                content=[TextContent(text="value loopback-secret-value")],
+            )
+        ]
+    )
+    conversation = Conversation(
+        agent=Agent(llm=llm, tools=[]),
+        workspace=str(tmp_path),
+        visualizer=None,
+        secrets={"TEST_TOKEN": LookupSecret(url=f"http://127.0.0.1:{port}/secret")},
+    )
+    try:
+        conversation.send_message("hello")
+        await asyncio.wait_for(conversation.arun(), timeout=5)
+        assert requested.is_set()
+        messages = [
+            event
+            for event in conversation.state.events
+            if isinstance(event, MessageEvent) and event.source == "agent"
+        ]
+        assert messages[-1].llm_message.content == [
+            TextContent(text="value <secret-hidden>")
+        ]
+    finally:
+        await asyncio.to_thread(conversation.close)
+        server.close()
+        await server.wait_closed()
+
+
+@pytest.mark.asyncio
+async def test_async_tool_thought_matches_sync_behavior(tmp_path):
+    thoughts = []
+    for use_async in [False, True]:
+        message = Message(
+            role="assistant",
+            content=[TextContent(text="existing tool thought")],
+            tool_calls=[
+                MessageToolCall(
+                    id="finish-call",
+                    origin="completion",
+                    name="finish",
+                    arguments='{"message":"done"}',
+                )
+            ],
+        )
+        conversation = Conversation(
+            agent=Agent(llm=TestLLM.from_messages([message]), tools=[]),
+            workspace=str(tmp_path),
+            visualizer=None,
+            secrets={"TOKEN": "existing tool thought"},
+        )
+        try:
+            conversation.send_message("finish")
+            if use_async:
+                await conversation.arun()
+            else:
+                await asyncio.to_thread(conversation.run)
+            thoughts.append(
+                [
+                    event.thought
+                    for event in conversation.state.events
+                    if isinstance(event, ActionEvent)
+                ]
+            )
+        finally:
+            await asyncio.to_thread(conversation.close)
+    assert thoughts[0]
+    assert thoughts[0] == thoughts[1]
+
+
+@pytest.mark.asyncio
+async def test_async_response_masks_secrets_registered_during_lookup(tmp_path):
+    requested = []
+    values = ["first-secret-value", "second-secret-value", "third-secret-value"]
+
+    async def serve_secret(reader, writer):
+        request = await reader.readuntil(b"\r\n\r\n")
+        index = int(request.split(b" ")[1].removeprefix(b"/"))
+        requested.append(index)
+        if index + 1 < len(values):
+            await asyncio.to_thread(
+                conversation.update_secrets,
+                {
+                    f"TOKEN_{index + 1}": LookupSecret(
+                        url=f"http://127.0.0.1:{port}/{index + 1}"
+                    )
+                },
+            )
+        body = values[index].encode()
+        writer.write(
+            b"HTTP/1.1 200 OK\r\nContent-Length: "
+            + str(len(body)).encode()
+            + b"\r\nConnection: close\r\n\r\n"
+            + body
+        )
+        await writer.drain()
+        writer.close()
+        await writer.wait_closed()
+
+    server = await asyncio.start_server(serve_secret, "127.0.0.1", 0)
+    port = server.sockets[0].getsockname()[1]
+    conversation = Conversation(
+        agent=Agent(
+            llm=TestLLM.from_messages(
+                [
+                    Message(
+                        role="assistant", content=[TextContent(text=" ".join(values))]
+                    )
+                ]
+            ),
+            tools=[],
+        ),
+        workspace=str(tmp_path),
+        visualizer=None,
+        secrets={"TOKEN_0": LookupSecret(url=f"http://127.0.0.1:{port}/0")},
+    )
+    try:
+        conversation.send_message("hello")
+        await asyncio.wait_for(conversation.arun(), timeout=5)
+        assert requested == [0, 1, 2]
+        messages = [
+            event
+            for event in conversation.state.events
+            if isinstance(event, MessageEvent) and event.source == "agent"
+        ]
+        assert messages[-1].llm_message.content == [
+            TextContent(text="<secret-hidden> <secret-hidden> <secret-hidden>")
+        ]
+    finally:
+        await asyncio.to_thread(conversation.close)
+        server.close()
+        await server.wait_closed()