diff --git a/caw/providers/codex.py b/caw/providers/codex.py index 64ecf6c..9f4c101 100644 --- a/caw/providers/codex.py +++ b/caw/providers/codex.py @@ -5,6 +5,7 @@ import atexit import json import logging +import os import re import subprocess import threading @@ -131,6 +132,21 @@ def _cleanup_processes() -> None: atexit.register(_cleanup_processes) +# -- Prompt argv / stdin ------------------------------------------------------ + +# Linux ARG_MAX includes the process environment, so keep a conservative +# threshold well below SC_ARG_MAX. Large prompts (e.g. CodeWiki repository +# overviews) must use ``codex exec … -`` and stdin. +_PROMPT_STDIN_THRESHOLD_BYTES = 100_000 + + +def _prompt_exceeds_argv_budget(cmd: list[str], prompt: str, *, threshold: int = _PROMPT_STDIN_THRESHOLD_BYTES) -> bool: + """Return True when placing ``prompt`` on argv would risk E2BIG.""" + cmd_bytes = sum(len(part.encode("utf-8", "replace")) + 1 for part in cmd) + prompt_bytes = len(prompt.encode("utf-8", "replace")) + return (cmd_bytes + prompt_bytes) > threshold + + # -- MCP config helpers ------------------------------------------------------- @@ -255,8 +271,13 @@ def send(self, message: str) -> Turn: cmd += self._mcp_config_args() - # Prompt as positional arg (last) - cmd.append(prompt) + # Prompt as positional arg, or stdin ("-") when argv would exceed ARG_MAX. + if _prompt_exceeds_argv_budget(cmd, prompt): + cmd.append("-") + stdin_data = prompt + else: + cmd.append(prompt) + stdin_data = None # Accumulated state for event processing blocks: list[ContentBlock] = [] @@ -267,7 +288,7 @@ def send(self, message: str) -> Turn: try: proc = subprocess.Popen( cmd, - stdin=subprocess.DEVNULL, + stdin=subprocess.PIPE if stdin_data is not None else subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -277,6 +298,9 @@ def send(self, message: str) -> Turn: _register_process(proc) try: + if stdin_data is not None and proc.stdin is not None: + proc.stdin.write(stdin_data) + proc.stdin.close() # Stream stdout line by line for line in proc.stdout: # type: ignore[union-attr] line = line.rstrip("\n") diff --git a/tests/test_codex_large_prompt_stdin.py b/tests/test_codex_large_prompt_stdin.py new file mode 100644 index 0000000..1206c70 --- /dev/null +++ b/tests/test_codex_large_prompt_stdin.py @@ -0,0 +1,80 @@ +"""Large prompts must go through stdin to avoid Linux ARG_MAX / E2BIG.""" + +from __future__ import annotations + +import io +from unittest.mock import MagicMock, patch + +from caw.providers.codex import ( + CodexSession, + _PROMPT_STDIN_THRESHOLD_BYTES, + _prompt_exceeds_argv_budget, +) + + +class TestPromptArgvBudget: + def test_small_prompt_fits_on_argv(self): + assert _prompt_exceeds_argv_budget(["codex", "exec"], "hello") is False + + def test_large_prompt_requires_stdin(self): + prompt = "x" * (_PROMPT_STDIN_THRESHOLD_BYTES + 1) + assert _prompt_exceeds_argv_budget(["codex", "exec"], prompt) is True + + def test_custom_threshold(self): + assert _prompt_exceeds_argv_budget(["codex"], "abcd", threshold=3) is True + assert _prompt_exceeds_argv_budget(["codex"], "ab", threshold=100) is False + + +class TestCodexSessionSendStdin: + def test_send_pipes_large_prompt_via_stdin(self): + session = CodexSession(mcp_servers=[]) + large = "P" * (_PROMPT_STDIN_THRESHOLD_BYTES + 50) + + stdout = io.StringIO( + '{"type":"thread.started","thread_id":"t1"}\n' + '{"type":"item.completed","item":{"type":"agent_message","text":"ok"}}\n' + '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' + ) + proc = MagicMock() + proc.stdin = MagicMock() + proc.stdout = stdout + proc.stderr = io.StringIO("") + proc.returncode = 0 + + with patch("caw.providers.codex.subprocess.Popen", return_value=proc) as popen: + turn = session.send(large) + + cmd = popen.call_args.args[0] + kwargs = popen.call_args.kwargs + assert cmd[-1] == "-" + assert kwargs["stdin"] is not None # PIPE, not DEVNULL + proc.stdin.write.assert_called_once_with(large) + proc.stdin.close.assert_called_once() + assert "ok" in turn.result + + def test_send_keeps_small_prompt_on_argv(self): + from subprocess import DEVNULL + + session = CodexSession(mcp_servers=[]) + small = "short prompt" + + stdout = io.StringIO( + '{"type":"thread.started","thread_id":"t1"}\n' + '{"type":"item.completed","item":{"type":"agent_message","text":"ok"}}\n' + '{"type":"turn.completed","usage":{"input_tokens":1,"cached_input_tokens":0,"output_tokens":1}}\n' + ) + proc = MagicMock() + proc.stdin = MagicMock() + proc.stdout = stdout + proc.stderr = io.StringIO("") + proc.returncode = 0 + + with patch("caw.providers.codex.subprocess.Popen", return_value=proc) as popen: + session.send(small) + + cmd = popen.call_args.args[0] + kwargs = popen.call_args.kwargs + assert cmd[-1] == small + assert kwargs["stdin"] is DEVNULL + proc.stdin.write.assert_not_called() + proc.stdin.close.assert_not_called()