From 9622f4fd25587050cb8d38df4373999cb2bea7ac Mon Sep 17 00:00:00 2001 From: "fujiajie.168" Date: Tue, 11 Aug 2026 11:07:48 +0800 Subject: [PATCH 01/12] fix(service): break startup circular imports with lazy exports --- openviking/service/__init__.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/openviking/service/__init__.py b/openviking/service/__init__.py index 46090c8fe9..f99f970aed 100644 --- a/openviking/service/__init__.py +++ b/openviking/service/__init__.py @@ -7,14 +7,29 @@ enabling reuse across HTTP Server and CLI. """ -from openviking.service.core import OpenVikingService -from openviking.service.debug_service import ComponentStatus, DebugService, SystemStatus -from openviking.service.fs_service import FSService -from openviking.service.pack_service import PackService -from openviking.service.relation_service import RelationService -from openviking.service.resource_service import ResourceService -from openviking.service.search_service import SearchService -from openviking.service.session_service import SessionService +import importlib + +_LAZY_IMPORTS = { + "OpenVikingService": "openviking.service.core", + "ComponentStatus": "openviking.service.debug_service", + "DebugService": "openviking.service.debug_service", + "SystemStatus": "openviking.service.debug_service", + "FSService": "openviking.service.fs_service", + "PackService": "openviking.service.pack_service", + "RelationService": "openviking.service.relation_service", + "ResourceService": "openviking.service.resource_service", + "SearchService": "openviking.service.search_service", + "SessionService": "openviking.service.session_service", +} + + +def __getattr__(name: str): + module_path = _LAZY_IMPORTS.get(name) + if module_path is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(importlib.import_module(module_path), name) + globals()[name] = value + return value __all__ = [ "OpenVikingService", From 0887ea9212222b856cd348b19a22821ec2c4cc4d Mon Sep 17 00:00:00 2001 From: "fujiajie.168" Date: Tue, 11 Aug 2026 11:08:05 +0800 Subject: [PATCH 02/12] fix(fs): avoid root semantic refresh when removing resource scope --- openviking/service/fs_service.py | 2 +- .../storage/queuefs/semantic_processor.py | 6 ++++++ tests/service/test_fs_service.py | 18 ++++++++++++++++++ tests/storage/test_memory_semantic_stall.py | 11 +++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py index 8ee7a9e5e0..52c62e5cdd 100644 --- a/openviking/service/fs_service.py +++ b/openviking/service/fs_service.py @@ -314,7 +314,7 @@ def _semantic_refresh_parent_uri(uri: str, context_type: str) -> Optional[str]: if context_type != "resource": return None parent = VikingURI(uri).parent - return parent.uri if parent else None + return parent.uri if parent and parent.scope else None @staticmethod def _memory_overview_parent_uri(uri: str, context_type: str) -> Optional[str]: diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py index d3da70d82d..88c91d363c 100644 --- a/openviking/storage/queuefs/semantic_processor.py +++ b/openviking/storage/queuefs/semantic_processor.py @@ -329,6 +329,12 @@ async def on_dequeue( assert data is not None msg = SemanticMsg.from_dict(data) + if VikingURI(msg.uri).parent is None: + logger.warning("Skipping semantic generation for root URI: %s", msg.uri) + if msg.telemetry_id and msg.id: + get_request_wait_tracker().mark_semantic_done(msg.telemetry_id, msg.id) + self.report_success() + return None if is_semantic_msg_stale(msg): logger.info( "Skipping stale semantic message: uri=%s version=%s", diff --git a/tests/service/test_fs_service.py b/tests/service/test_fs_service.py index fd55fd1808..1a4dffbe8f 100644 --- a/tests/service/test_fs_service.py +++ b/tests/service/test_fs_service.py @@ -279,6 +279,24 @@ async def test_resource_rm_without_wait_only_queues_refresh(request_context): assert result["semantic_status"] == "queued" +@pytest.mark.asyncio +async def test_resource_scope_rm_does_not_refresh_global_root(request_context): + service = FSService(viking_fs=_FakeVikingFS()) + service._enqueue_delete_refresh = AsyncMock() + service._wait_for_refresh = AsyncMock() + + result = await service.rm( + "viking://resources", + ctx=request_context, + recursive=True, + wait=True, + ) + + service._enqueue_delete_refresh.assert_not_awaited() + service._wait_for_refresh.assert_not_awaited() + assert "semantic_root_uri" not in result + + @pytest.mark.asyncio async def test_resource_rm_deactivates_watch_tasks(request_context): viking_fs = _FakeVikingFS() diff --git a/tests/storage/test_memory_semantic_stall.py b/tests/storage/test_memory_semantic_stall.py index 099fa35f50..b872c72401 100644 --- a/tests/storage/test_memory_semantic_stall.py +++ b/tests/storage/test_memory_semantic_stall.py @@ -40,6 +40,17 @@ def _build_data(msg: SemanticMsg) -> dict: return msg.to_dict() +@pytest.mark.asyncio +async def test_root_semantic_message_is_acknowledged_without_processing(): + processor = SemanticProcessor() + success = MagicMock() + processor.set_callbacks(success, MagicMock(), MagicMock()) + + await processor.on_dequeue(_build_data(_make_msg(uri="viking://", context_type="resource"))) + + success.assert_called_once_with() + + @pytest.mark.asyncio async def test_memory_empty_dir_still_reports_success(): """When viking_fs.ls returns an empty list, report_success() must be called.""" From 7aca501f509427aba5930d73c3f873032c8634c9 Mon Sep 17 00:00:00 2001 From: "fujiajie.168" Date: Tue, 11 Aug 2026 11:08:21 +0800 Subject: [PATCH 03/12] fix(compile): preserve existing wiki links --- bot/tests/test_compile.py | 63 ++++++++++++++++++- bot/vikingbot/agent/tools/compile.py | 25 ++++---- bot/vikingbot/compile/renderer.py | 9 +-- .../session/memory/utils/link_renderer.py | 47 ++++++++++++++ tests/test_link_renderer.py | 45 +++++++++++++ 5 files changed, 171 insertions(+), 18 deletions(-) diff --git a/bot/tests/test_compile.py b/bot/tests/test_compile.py index b4218a3d63..4fdc57b1cd 100644 --- a/bot/tests/test_compile.py +++ b/bot/tests/test_compile.py @@ -321,6 +321,31 @@ def test_renderer_creates_okf_pages_links_and_citations(): assert "[1] [source](viking://resources/source)" in first["content"] +def test_renderer_preserves_existing_link_and_keeps_relationship(): + bundle = WikiBundleDraft.model_validate( + { + "pages": [ + _page(1, "Overview", body_markdown="Read [Details](./details.md) next."), + _page(2, "Details"), + ], + "links": [{"f": 1, "t": 2, "match_text": "Details"}], + } + ) + + rendered = WikiRenderer().render( + bundle=bundle, + target_uri="viking://resources/wiki", + source_roots={"src_1": "viking://resources/source"}, + catalog_uris=set(), + existing_raw={}, + ) + operations = {operation["uri"]: operation["content"] for operation in rendered.operations} + + assert operations["viking://resources/wiki/overview.md"].count("[Details](./details.md)") == 1 + assert "- [Overview](./overview.md)" in operations["viking://resources/wiki/details.md"] + assert rendered.link_count == 0 + + def test_wiki_page_title_path_normalizes_spaced_dashes_only(): assert ( wiki_page_path_from_title("Experimental Designs - Residual Networks") @@ -758,6 +783,40 @@ async def test_submit_tool_rejects_protected_anchor_and_path_collision(): assert tool.bundle is not None and tool.bundle.pages == [] +@pytest.mark.asyncio +async def test_submit_tool_accepts_existing_link_only_when_target_matches(): + tool = SubmitWikiBundleTool( + source_ids={"src_1"}, + catalog_uris=set(), + target_uri="viking://resources/wiki", + limits=CompileLimits(), + ) + context = ToolContext() + + accepted = await tool.execute( + context, + pages=[ + _page(1, "One", body_markdown="参见 [L2 行为标签库](./two.md)。"), + _page(2, "Two"), + ], + links=[{"f": 1, "t": 2, "match_text": "行为标签库"}], + ) + assert accepted.startswith("Wiki bundle accepted") + assert tool.bundle is not None and len(tool.bundle.links) == 1 + + rejected = await tool.execute( + context, + pages=[ + _page(1, "One", body_markdown="参见 [行为标签库](./three.md)。"), + _page(2, "Two"), + _page(3, "Three"), + ], + links=[{"f": 1, "t": 2, "match_text": "行为标签库"}], + ) + assert "unsatisfied anchor '行为标签库'" in rejected + assert tool.bundle is None + + @pytest.mark.asyncio async def test_submit_tool_checks_size_before_parsing_okf_artifact(): tool = SubmitWikiBundleTool( @@ -869,8 +928,8 @@ async def test_submit_tool_reports_all_invalid_links(): ) assert result.startswith("Error: Invalid Wiki bundle: 2 invalid link(s):") - assert "links[0] from page 1 has non-linkable anchor 'Missing One'" in result - assert "links[1] from page 1 has non-linkable anchor 'Missing Two'" in result + assert "links[0] from page 1 has unsatisfied anchor 'Missing One'" in result + assert "links[1] from page 1 has unsatisfied anchor 'Missing Two'" in result assert tool.bundle is None diff --git a/bot/vikingbot/agent/tools/compile.py b/bot/vikingbot/agent/tools/compile.py index d368eb8431..87298d7743 100644 --- a/bot/vikingbot/agent/tools/compile.py +++ b/bot/vikingbot/agent/tools/compile.py @@ -279,8 +279,9 @@ def parameters(self) -> dict[str, Any]: match_schema = link_def.get("properties", {}).get("match_text") if isinstance(match_schema, dict): match_schema["description"] = ( - "Exact anchor text that must appear in the source page draft body outside " - "frontmatter, code, existing Markdown links, and Citations." + "Exact anchor text that must either appear in the source page draft body " + "outside frontmatter, code, existing Markdown links, and Citations, or " + "already be part of a Markdown link to the target page." ) schema.pop("title", None) return schema @@ -490,6 +491,7 @@ async def _validate_bundle( "using workspace_path instead of inline content" ) page_ids: set[int] = set() + page_uris: dict[int, str] = {} final_uris: set[str] = set() total_bytes = 0 for page in bundle.pages: @@ -532,6 +534,7 @@ async def _validate_bundle( if final_uri in final_uris: raise ValueError(f"duplicate final Wiki path: {final_uri}") final_uris.add(final_uri) + page_uris[page.page_id] = final_uri total_bytes += len(page.body_markdown.encode("utf-8")) file_payloads: list[bytes | None] = [] @@ -600,18 +603,16 @@ async def _validate_bundle( link_errors.append(f"{prefix} match_text is required") continue source_page = page_by_id[link.f] - if ( - LinkRenderer._find_match_span( - source_page.body_markdown, - link.match_text, - LinkRenderer.protected_markdown_spans(source_page.body_markdown), - ) - is None + if not LinkRenderer.can_render_link( + source_page.body_markdown, + link.match_text, + page_uris[link.f], + page_uris[link.t], ): link_errors.append( - f"{prefix} from page {link.f} has non-linkable anchor " - f"{link.match_text!r}; remove the link or use exact unprotected " - "text from that page body" + f"{prefix} from page {link.f} has unsatisfied anchor " + f"{link.match_text!r}; use exact unprotected text or an existing " + f"Markdown link to page {link.t}" ) if link_errors: raise ValueError(f"{len(link_errors)} invalid link(s): " + "; ".join(link_errors)) diff --git a/bot/vikingbot/compile/renderer.py b/bot/vikingbot/compile/renderer.py index 17a5c8332f..5dbdfbc444 100644 --- a/bot/vikingbot/compile/renderer.py +++ b/bot/vikingbot/compile/renderer.py @@ -453,13 +453,14 @@ def render( raise ValueError(f"WikiLink references an unknown page_id: f={link.f}, t={link.t}") if not link.match_text: raise ValueError("WikiLink match_text is required") - if LinkRenderer._find_match_span( + if not LinkRenderer.can_render_link( source_page.body_markdown, link.match_text, - LinkRenderer.protected_markdown_spans(source_page.body_markdown), - ) is None: + page_uris[link.f][0], + page_uris[link.t][0], + ): raise ValueError( - f"WikiLink match_text is not a linkable body anchor: {link.match_text!r}" + f"WikiLink match_text is not a satisfiable body anchor: {link.match_text!r}" ) resolved_links = resolve_wiki_links(bundle.links, page_uris, strict=True) diff --git a/openviking/session/memory/utils/link_renderer.py b/openviking/session/memory/utils/link_renderer.py index eb0250f296..2e9b77b04e 100644 --- a/openviking/session/memory/utils/link_renderer.py +++ b/openviking/session/memory/utils/link_renderer.py @@ -1,5 +1,7 @@ +import posixpath import re from typing import Dict, List, Optional +from urllib.parse import unquote from openviking.core.namespace import uri_parts @@ -79,6 +81,51 @@ def _overlaps_protected_span(start: int, end: int) -> bool: return start, end return None + @staticmethod + def _normalize_markdown_target(target: str) -> str: + target = unquote(target.strip()) + if target.startswith("<") and target.endswith(">"): + target = target[1:-1] + target = target.split("#", 1)[0] + return target.rstrip("/") if "://" in target else posixpath.normpath(target) + + @staticmethod + def can_render_link( + content: str, + match_text: str, + source_uri: str, + target_uri: str, + ) -> bool: + """Return whether rendering can insert or preserve the requested link.""" + protected_spans = LinkRenderer.protected_markdown_spans(content) + if LinkRenderer._find_match_span(content, match_text, protected_spans) is not None: + return True + + markdown_links = list(LinkRenderer._RELATIVE_LINK_RE.finditer(content)) + link_spans = {(match.start(), match.end()) for match in markdown_links} + non_link_protected = [span for span in protected_spans if span not in link_spans] + relative_target = LinkRenderer.relative_path(source_uri, target_uri) + expected_targets = { + LinkRenderer._normalize_markdown_target(target_uri), + LinkRenderer._normalize_markdown_target( + relative_target if relative_target is not None else target_uri + ), + } + + for link in markdown_links: + if link.start() > 0 and content[link.start() - 1] == "!": + continue + if any( + not (link.end() <= start or link.start() >= end) + for start, end in non_link_protected + ): + continue + if LinkRenderer._find_match_span(link.group("text"), match_text) is None: + continue + if LinkRenderer._normalize_markdown_target(link.group("target")) in expected_targets: + return True + return False + @staticmethod def render_links(content: str, source_uri: str, links: List[Dict]) -> str: """Replace match_text in content with relative markdown links. diff --git a/tests/test_link_renderer.py b/tests/test_link_renderer.py index b77efbcf0d..6b9ff2fdfd 100644 --- a/tests/test_link_renderer.py +++ b/tests/test_link_renderer.py @@ -63,6 +63,51 @@ def test_same_file_returns_empty(self): assert result == "" +class TestLinkSatisfaction: + source_uri = "viking://resources/wiki/overview.md" + target_uri = "viking://resources/wiki/tags.md" + + def test_unprotected_anchor_can_be_linked(self): + assert LinkRenderer.can_render_link( + "Read the behavior tags.", + "behavior tags", + self.source_uri, + self.target_uri, + ) + + def test_existing_link_to_target_is_already_satisfied(self): + assert LinkRenderer.can_render_link( + "参见 [L2 行为标签库](./tags.md)。", + "行为标签库", + self.source_uri, + self.target_uri, + ) + + def test_equivalent_encoded_target_with_fragment_is_satisfied(self): + assert LinkRenderer.can_render_link( + "See [ByteDance](../concepts/byte%20dance.md#facts).", + "ByteDance", + "viking://resources/wiki/sections/overview.md", + "viking://resources/wiki/concepts/byte dance.md", + ) + + def test_existing_link_to_other_target_is_not_satisfied(self): + assert not LinkRenderer.can_render_link( + "参见 [行为标签库](./other.md)。", + "行为标签库", + self.source_uri, + self.target_uri, + ) + + def test_link_syntax_in_code_is_not_satisfied(self): + assert not LinkRenderer.can_render_link( + "`[行为标签库](./tags.md)`", + "行为标签库", + self.source_uri, + self.target_uri, + ) + + class TestRenderLinks: def test_single_link(self): content = "Caroline attended a support group meeting." From 85547cad352e3bc2393eadc91acb27d8905bcc2a Mon Sep 17 00:00:00 2001 From: "fujiajie.168" Date: Tue, 11 Aug 2026 11:10:53 +0800 Subject: [PATCH 04/12] fix(sdk): extend HTTP timeout for blocking batch writes --- sdk/python/openviking_sdk/client.py | 8 +++++++- .../tests/test_async_client_behaviors.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.py index 9b2142b9bd..67e4d9e089 100644 --- a/sdk/python/openviking_sdk/client.py +++ b/sdk/python/openviking_sdk/client.py @@ -68,7 +68,6 @@ _SESSION_CONFIG_UNSET = object() - def _image_mime_type(file_name: str = "") -> str: mime_type, _ = mimetypes.guess_type(file_name or "") if mime_type and mime_type.startswith("image/"): @@ -518,6 +517,12 @@ async def _request(self, method: str, url: str, **kwargs: Any) -> httpx.Response retry_headers[GATEWAY_TOKEN_HEADER] = self._gateway_token return await self._send_http_request(method, url, retry_headers, request_kwargs) + def _wait_request_kwargs(self, *, wait: bool, timeout: Optional[float]) -> Dict[str, Any]: + if not wait or timeout is None: + return {} + read_timeout = max(self._timeout, timeout + 30.0) + return {"timeout": httpx.Timeout(self._timeout, read=read_timeout)} + async def close(self) -> None: if self._http: try: @@ -1207,6 +1212,7 @@ async def batch_write( "timeout": timeout, "telemetry": telemetry, }, + **self._wait_request_kwargs(wait=wait, timeout=timeout), ) return self._handle_response_data(response).get("result", {}) diff --git a/sdk/python/tests/test_async_client_behaviors.py b/sdk/python/tests/test_async_client_behaviors.py index 71be100729..857bc5e44c 100644 --- a/sdk/python/tests/test_async_client_behaviors.py +++ b/sdk/python/tests/test_async_client_behaviors.py @@ -961,6 +961,24 @@ async def test_rm_uses_delete_request_with_timeout_when_provided(): ) +@pytest.mark.asyncio +async def test_batch_write_http_timeout_outlives_server_wait_timeout(): + client = AsyncHTTPClient(url="http://localhost:1933", timeout=180.0) + client._request = AsyncMock(return_value=object()) + client._handle_response_data = lambda _response: {"result": {}} + + await client.batch_write( + "viking://resources/wiki", + [], + wait=True, + timeout=300.0, + ) + + request_timeout = client._request.await_args.kwargs["timeout"] + assert request_timeout.read == 330.0 + assert request_timeout.connect == 180.0 + + @pytest.mark.asyncio async def test_link_normalizes_single_and_multiple_target_uris(): client = AsyncHTTPClient(url="http://localhost:1933") From 6116c076f417617df3f1414a28fd976679ef7ac4 Mon Sep 17 00:00:00 2001 From: "fujiajie.168" Date: Tue, 11 Aug 2026 15:21:01 +0800 Subject: [PATCH 05/12] fix(compile): salvage workspace output on runtime timeout --- bot/tests/test_compile.py | 290 +++++++++++++++++++++++++++++++ bot/vikingbot/compile/models.py | 2 +- bot/vikingbot/compile/service.py | 279 ++++++++++++++++++++++++++++- 3 files changed, 565 insertions(+), 6 deletions(-) diff --git a/bot/tests/test_compile.py b/bot/tests/test_compile.py index 4fdc57b1cd..e44a1c6c28 100644 --- a/bot/tests/test_compile.py +++ b/bot/tests/test_compile.py @@ -15,6 +15,7 @@ CompileFailure, CompileLimits, CompileRequest, + CompileResult, CompileTask, SanitizedCompileRequest, WikiBundleDraft, @@ -2506,6 +2507,295 @@ async def test_compile_queue_wait_has_a_deadline(tmp_path: Path): assert service._target_locks == {} +@pytest.mark.asyncio +async def test_timeout_salvage_copies_workspace_and_repairs_links(tmp_path: Path): + service = _compile_service( + tmp_path, + auth_mode="api_key", + backend=SandboxBackend.DIRECT, + ) + workspace = tmp_path / "workspace" + page_root = workspace / "__compile_staging__" / "wiki_pages" + (page_root / "guide").mkdir(parents=True) + (workspace / "__compile_staging__" / "work").mkdir(parents=True) + (workspace / "__compile_staging__" / "tmp").mkdir(parents=True) + (workspace / "meta").mkdir() + (workspace / "skills" / "wiki").mkdir(parents=True) + (workspace / "empty").mkdir() + (page_root / "home.md").write_text("# Home\n", encoding="utf-8") + (page_root / "guide" / "topic.md").write_text( + "\n".join( + [ + "[Home](home.md#top)", + "[Meta](meta/readme.md)", + "[Existing](../existing.md)", + "[Missing](missing.md)", + "![Missing image](missing.png)", + '[Titled](../meta/title.md "Title")', + "[Paren](../meta/foo(1).md)", + "[Web](https://example.com)", + "[Source](viking://resources/source)", + "[Anchor](#local)", + "`[Code](missing.md)`", + ] + ), + encoding="utf-8", + ) + (workspace / "meta" / "readme.md").write_text("# Meta\n", encoding="utf-8") + (workspace / "meta" / "title.md").write_text("# Title\n", encoding="utf-8") + (workspace / "meta" / "foo(1).md").write_text("# Paren\n", encoding="utf-8") + (workspace / "meta" / "events.jsonl").write_bytes(b"") + (workspace / "artifact.bin").write_bytes(b"\x00\x01") + (workspace / "home.md").write_text("# Artifact Home\n", encoding="utf-8") + (workspace / "caseonly.md").write_text("case mismatch", encoding="utf-8") + (workspace / "Foo.md").write_text("first", encoding="utf-8") + (workspace / "foo.md").write_text("duplicate", encoding="utf-8") + (workspace / "bad#name.txt").write_text("unsafe URI", encoding="utf-8") + (workspace / "__compile_staging__" / "work" / "notes.txt").write_text("notes", encoding="utf-8") + (workspace / "__compile_staging__" / "tmp" / "check.txt").write_text("check", encoding="utf-8") + (workspace / "skills" / "wiki" / "SKILL.md").write_text("do not copy", encoding="utf-8") + + class Client: + operations = [] + + async def tree(self, uri, *, node_limit): + assert uri == "viking://resources/wiki" + assert node_limit == service.limits.target_inventory_entries + 1 + return [ + {"uri": f"{uri}/existing.md", "isDir": False}, + {"uri": f"{uri}/CaseOnly.md", "isDir": False}, + {"uri": f"{uri}/meta/events.jsonl", "isDir": False}, + ] + + async def download_bytes(self, uri): + return { + "viking://resources/wiki/CaseOnly.md": b"old case", + "viking://resources/wiki/meta/events.jsonl": b"old", + }[uri] + + async def batch_write(self, *, root_uri, operations, wait): + assert root_uri == "viking://resources/wiki" + assert wait is False + self.operations = operations + updated = [ + operation["uri"] + for operation in operations + if operation["precondition"]["kind"] == "replace_if_hash" + ] + created = [ + operation["uri"] + for operation in operations + if operation["precondition"]["kind"] == "create_if_absent" + ] + return {"created": created, "updated": updated, "unchanged": []} + + client = Client() + result = await service._salvage_workspace( + client=client, + request=_sanitized_compile_request(), + workspace=workspace, + ) + + assert result is not None + payloads = { + operation["uri"].removeprefix("viking://resources/wiki/"): base64.b64decode( + operation["content_base64"] + ) + for operation in client.operations + } + assert "skills/wiki/SKILL.md" not in payloads + assert "empty" not in payloads + assert "__compile_staging__/wiki_pages/home.md" not in payloads + assert payloads["home.md"] == b"# Artifact Home\n" + assert payloads["artifact.bin"] == b"\x00\x01" + assert payloads["CaseOnly.md"] == b"case mismatch" + assert payloads["meta/events.jsonl"] == b"" + assert "__compile_staging__/work/notes.txt" not in payloads + assert "__compile_staging__/tmp/check.txt" not in payloads + assert sum(path.casefold() == "foo.md" for path in payloads) == 1 + assert "bad#name.txt" not in payloads + topic = payloads["guide/topic.md"].decode() + assert "[Home](../home.md#top)" in topic + assert "[Meta](../meta/readme.md)" in topic + assert "[Existing](../existing.md)" in topic + assert "Missing\nMissing image" in topic + assert '[Titled](../meta/title.md "Title")' in topic + assert "[Paren](../meta/foo(1).md)" in topic + assert "[Web](https://example.com)" in topic + assert "[Source](viking://resources/source)" in topic + assert "[Anchor](#local)" in topic + assert "`[Code](missing.md)`" in topic + assert result.warnings and "partial output" in result.warnings[0] + assert any("Skipped" in warning for warning in result.warnings) + + +@pytest.mark.asyncio +async def test_runtime_deadline_salvages_before_workspace_cleanup(monkeypatch, tmp_path: Path): + observed = [] + + class TaskConfig: + def __init__(self): + self.bot_data_path = tmp_path + self.workspace_path = tmp_path / "host-workspace" + self.skills = [] + self.sandbox = SimpleNamespace( + mode=None, model_copy=lambda *, deep: SimpleNamespace(mode=None) + ) + + def model_copy(self, *, update): + copy = TaskConfig() + for key, value in update.items(): + setattr(copy, key, value) + return copy + + class FakeSandboxManager: + def __init__(self, config, workspace_parent, workspace_path): + del config, workspace_path + self.workspace = workspace_parent / "workspace" + self.workspace.mkdir(parents=True) + + def get_workspace_path(self, session_key): + del session_key + return self.workspace + + async def cleanup_session(self, session_key): + del session_key + observed.append("cleanup") + + class FakeSkillsLoader: + def __init__(self, workspace, *, builtin_skills_dir): + del workspace, builtin_skills_dir + + def load_skills_for_context(self, names): + assert names == ["wiki"] + return "Write Wiki pages." + + def _get_skill_meta(self, name): + assert name == "wiki" + return {} + + class FakeRequestLoop: + def __init__(self, **kwargs): + self.workspace = kwargs["workspace"] + + async def run_structured_task(self, **kwargs): + del kwargs + (self.workspace / "output.md").write_text("partial", encoding="utf-8") + await asyncio.Event().wait() + + class Client: + async def get_skill(self, skill_name, *, target_uri): + assert skill_name == "wiki" + assert target_uri == "viking://agent/skills" + return { + "root_uri": "viking://agent/skills/wiki", + "content": "---\nname: wiki\ndescription: Write Wiki\n---\nWrite it.", + "files": [], + } + + async def close(self): + return None + + async def create_client(**kwargs): + del kwargs + return Client() + + async def no_op(*args, **kwargs): + del args, kwargs + + async def build_sources(*args, **kwargs): + del args, kwargs + return [] + + async def build_catalog(*args, **kwargs): + del args, kwargs + return [], {} + + async def salvage(*, client, request, workspace): + del client + assert request.to == "viking://resources/wiki" + assert (workspace / "output.md").read_text() == "partial" + observed.append("salvage") + return CompileResult( + **{ + "from": request.from_, + "to": request.to, + "skill": request.skill, + "created": [f"{request.to}/output.md"], + "page_count": 1, + "warnings": ["partial output"], + } + ) + + monkeypatch.setattr("vikingbot.compile.service.SandboxManager", FakeSandboxManager) + monkeypatch.setattr("vikingbot.compile.service.SkillsLoader", FakeSkillsLoader) + monkeypatch.setattr("vikingbot.compile.service.AgentLoop", FakeRequestLoop) + monkeypatch.setattr("vikingbot.compile.service.VikingClient.create", create_client) + + host_loop = SimpleNamespace( + config=TaskConfig(), + bus=None, + provider=None, + model=None, + temperature=0, + max_iterations=1, + memory_window=1, + brave_api_key=None, + exa_api_key=None, + gen_image_model=None, + exec_config=None, + ) + service = BotCompileService(agent_loop=host_loop) + monkeypatch.setattr(service, "_materialize_skill", no_op) + monkeypatch.setattr(service, "_check_requirements", no_op) + monkeypatch.setattr(service, "_build_sources", build_sources) + monkeypatch.setattr(service, "_build_catalog", build_catalog) + monkeypatch.setattr( + service, "_build_compile_registry", lambda *args, **kwargs: (object(), set()) + ) + monkeypatch.setattr(service, "_salvage_workspace", salvage) + + request = _sanitized_compile_request() + task = CompileTask( + task_id="cmp_deadline", + principal_scope="owner", + sanitized_request=request, + status="accepted", + stage="queued", + created_at=utc_now(), + updated_at=utc_now(), + ) + await service.store.create(task) + loop = asyncio.get_running_loop() + await asyncio.wait_for( + service._execute_task( + task.task_id, + request, + {"api_key": "secret"}, + runtime_deadline=loop.time() + 0.01, + ), + timeout=0.01, + ) + + completed = await service.store.get(task.task_id) + assert completed is not None + assert completed.status == "completed" + assert completed.stage == "salvaged" + assert completed.result is not None + assert completed.result.created == ["viking://resources/wiki/output.md"] + assert observed == ["salvage", "cleanup"] + assert not (tmp_path / "compile_workspaces" / task.task_id).exists() + + await service._fail( + task.task_id, + CompileFailure("INTERNAL", "late cleanup failure", stage="salvaging"), + ) + still_completed = await service.store.get(task.task_id) + assert still_completed is not None + assert still_completed.status == "completed" + assert still_completed.result is not None + + @pytest.mark.asyncio async def test_task_store_restart_marks_nonterminal_without_persisting_connection(tmp_path: Path): store = CompileTaskStore(tmp_path) diff --git a/bot/vikingbot/compile/models.py b/bot/vikingbot/compile/models.py index 8ad1e35831..0b23e74fe5 100644 --- a/bot/vikingbot/compile/models.py +++ b/bot/vikingbot/compile/models.py @@ -41,7 +41,7 @@ class CompileLimits(BaseModel): accepted_tasks: int = 16 accepted_tasks_per_principal: int = 4 queue_wait_seconds: float = 5 * 60 - task_runtime_seconds: float = 30 * 60 + task_runtime_seconds: float = 360 * 60 terminal_task_retention_seconds: float = 24 * 60 * 60 terminal_task_records: int = 1000 diff --git a/bot/vikingbot/compile/service.py b/bot/vikingbot/compile/service.py index 7a448c981b..d6613fd8d8 100644 --- a/bot/vikingbot/compile/service.py +++ b/bot/vikingbot/compile/service.py @@ -3,7 +3,9 @@ from __future__ import annotations import asyncio +import base64 import json +import posixpath import re import shlex import shutil @@ -13,12 +15,18 @@ from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Mapping +from urllib.parse import unquote from loguru import logger -from openviking.core.namespace import classify_uri, uri_parts +from openviking.core.namespace import classify_uri, relative_uri_path, uri_parts from openviking.core.skill_loader import SkillLoader -from openviking.utils.path_safety import sanitize_relative_viking_path +from openviking.session.memory.utils.link_renderer import LinkRenderer +from openviking.utils.path_safety import ( + safe_join_viking_uri, + sanitize_relative_viking_path, + validate_safe_viking_uri_path, +) from openviking_cli.exceptions import OpenVikingError from vikingbot.agent.loop import AgentLoop from vikingbot.agent.skills import SkillsLoader @@ -42,6 +50,7 @@ ) from vikingbot.compile.renderer import ( WikiRenderer, + content_hash, has_unclosed_frontmatter, validate_declared_okf_markdown, ) @@ -71,10 +80,13 @@ _SKILL_EXCLUDED_FILES = frozenset( {".abstract.md", ".overview.md", ".relations.json", ".source.json"} ) -_CATALOG_EXCLUDED_FILES = _SKILL_EXCLUDED_FILES +_CATALOG_EXCLUDED_FILES = _SKILL_EXCLUDED_FILES _CATALOG_FRONTMATTER_LINES = 128 _TARGET_CATALOG_QUERY_CHARS = 40_000 _REQUIREMENT_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") +_SALVAGE_LINK_RE = re.compile( + r"(?P!?)\[(?P