diff --git a/README.md b/README.md index 28d3f4ba444..690d1cc110d 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ For a disposable DragonFly chroot development shell, use - **scripts/** Shell scripts to generate the final DPorts repository, as well as a copy of the Tinderbox hooks. + Also includes `scripts/dsynth-hooks/` which provides dsynth hook scripts to capture a bounded, high-signal “evidence bundle” (distilled errors + port context) when a port fails to build. + - **ports/** Contains subdirectories corresponding to Ports categories (e.g., `audio`, `editors`, `devel`, etc.). diff --git a/config/agentic-policy.json b/config/agentic-policy.json new file mode 100644 index 00000000000..785420c6322 --- /dev/null +++ b/config/agentic-policy.json @@ -0,0 +1,22 @@ +{ + "tiers": { + "AUTO": {"max_iterations": 2, "max_tokens": 30000}, + "ASSIST": {"max_iterations": 4, "max_tokens": 120000}, + "MANUAL": {} + }, + "classification_to_tier": { + "plist-error": "AUTO", + "fetch-checksum": "AUTO", + "pkg-format": "AUTO", + "compile-error": "ASSIST", + "patch-error": "ASSIST", + "link-error": "ASSIST", + "configure-error": "ASSIST", + "missing-dep": "MANUAL", + "fetch-error": "MANUAL", + "runtime-error": "MANUAL", + "dependency-conflict": "MANUAL", + "unknown": "MANUAL" + }, + "confidence_floor": {"AUTO": "high", "ASSIST": "medium"} +} diff --git a/docs/AGENTIC_BUILDS.md b/docs/AGENTIC_BUILDS.md new file mode 100644 index 00000000000..ee1b1a2f971 --- /dev/null +++ b/docs/AGENTIC_BUILDS.md @@ -0,0 +1,154 @@ +# Agentic build assistance — operator guide + +This document describes how the agentic loop fits into the DeltaPorts +build flow today (post-Phase-5). For the design history behind it, see +the per-phase commits referenced from `docs/agentic-consolidation-plan.md`. + +## What the loop does + +For every dsynth build a per-port hook fires when a port fails. The hook +uploads the failure evidence as a "bundle" (logs, port files, metadata) +to the artifact-store. A queue runner picks the bundle up, runs an LLM +triage pass on it, classifies the failure, and — if the trust tier +permits — runs a patch attempt loop against a writable copy of the +DeltaPorts tree inside a chrooted dev-env. The patch loop ends at a +`rebuild_proof.json` recording whether dsynth liked the change. No +commits, branches, pushes, or PRs are produced; promoting a fix is a +manual operator step. + +``` +dsynth build ──fail──▶ hook ──▶ artifact-store (bundle) + │ + ▼ + agent-queue-runner + │ + ┌────────────┴────────────┐ + ▼ ▼ + dportsv3.agent.triage dportsv3.agent.patch + (LLM classify) (LLM + 13 tools, + attempt loop) + │ + ▼ + bundle.analysis/ + ├── triage.md + ├── patch_audit.json + ├── rebuild_proof.json + └── changes.diff +``` + +## Architecture in one breath + +- **One DB**: `state.db` (WAL + `busy_timeout=5000`, `foreign_keys=ON` + on every connection). Two writers — artifact-store and tracker — + share it. +- **One serve process** for the UI/read surface: FastAPI + `dportsv3 tracker serve` at `:8080` by default. Builds, agentic + bundles/jobs/runner, and HTML views are all routed through it. +- **One hook set**: `scripts/dsynth-hooks/` writes the failure bundle + to artifact-store and the run-state to the tracker. +- **One per-port workspace primitive**: `dportsv3 dev-env` (chroot + + writable copy-on-write overlay). The patch agent edits files in the + overlay; the host tree is never modified. + +## Services to run + +| Service | Purpose | Default port | +|---|---|---| +| `dportsv3 artifact-store` | Receives bundles + `/v1/user-context` POSTs; writes to `state.db`; serves artifact streams under `/v1/artifacts/get` | 8788 | +| `dportsv3 tracker serve` | Read API + HTML views (`/`, `/target/{target}`, `/builds/{id}`, `/agentic/*`); SSE event tail | 8080 | +| `dportsv3 agent-queue-runner --queue-root $LOGS_ROOT/evidence/queue` | Pops bundles from the file queue, runs triage + patch jobs against the LLM provider | — | + +All three read the same `state.db`. Order of startup doesn't matter — +each one is idempotent on the schema. + +## Environment + +Required by the runner: + +| Var | Meaning | Example | +|---|---|---| +| `DPORTSV3_STATE_DB` | Path to state.db (shared with artifact-store) | `/build/synth/logs/evidence/state.db` | +| `DPORTSV3_TRACKER_URL` | Base URL for runner→tracker lookups (bundles, ports) | `http://127.0.0.1:8080` (default) | +| `ARTIFACT_STORE_URL` | Base URL for runner→artifact-store artifact GETs | `http://127.0.0.1:8788` (default) | +| `DP_HARNESS_TRIAGE_MODEL` | LiteLLM model string for triage | `openai/gpt-5-nano` | +| `DP_HARNESS_PATCH_MODEL` | LiteLLM model string for patch | `anthropic/claude-sonnet-4` | +| `DP_HARNESS_TRIAGE_API_KEY` / `_BASE` | Provider key + optional custom endpoint | — | +| `DP_HARNESS_PATCH_API_KEY` / `_BASE` | Same for patch | — | + +Required by the tracker: + +| Var | Meaning | +|---|---| +| `DPORTSV3_STATE_DB` | State DB path (defaults to `$PWD/state.db`) | +| `DPORTSV3_ARTIFACT_ROOT` | Evidence dir for blob streaming (defaults to `/build/synth/logs/evidence`) | + +Required by the hooks (set in `/etc/dsynth/hooks.conf` or wherever your +dsynth profile sources environment from): + +| Var | Meaning | +|---|---| +| `DPORTSV3_TRACKER_URL` | Where to write tracker run state | +| `DPORTSV3_TRACKER_TARGET` | Target (e.g. `@2026Q2`); defaults from `$PROFILE` | +| `ARTIFACT_STORE_URL` | Where to upload bundles | + +## Trust tiers + +`config/agentic-policy.json` decides whether a triaged failure +auto-advances into a patch attempt: + +- **AUTO** (`plist-error`, `fetch-checksum`, `pkg-format`): up to 2 + patch iterations, 30k tokens. +- **ASSIST** (`compile-error`, `patch-error`, `link-error`, + `configure-error`): up to 4 iterations, 120k tokens. +- **MANUAL** (`missing-dep`, `fetch-error`, `runtime-error`, + `dependency-conflict`, `unknown`): triage runs but no patch job is + enqueued. An operator can still hand-fire a patch via the queue. + +A `confidence_floor` downgrades the tier when the triage LLM's +reported confidence is below the floor for that tier. + +## Reading the tracker UI + +`http://:8080/` + +| Route | What | +|---|---| +| `/` | Target grid — one card per target with cumulative stat pills | +| `/target/{target}` | Live dsynth-progress UI for the latest build run on this target | +| `/builds` | All build runs across targets | +| `/builds/{run_id}` | Dsynth-progress UI scoped to one specific run | +| `/diff` | Compare two targets' current port status | +| `/agentic` | Bundles + jobs summary dashboard | +| `/agentic/bundles[?target=]` | Failure-evidence bundles | +| `/agentic/bundles/{id}` | One bundle's metadata + artifact list (links stream files from the artifact-store) | +| `/agentic/jobs[?state=&target=]` | Queue state (pending / inflight / done / failed) | +| `/agentic/runner` | Current runner heartbeat | +| `/agentic/activity` | Stage-transition log | + +The progress UI polls `/api/progress/{target}/summary.json` every 10s +when a build is active; SSE on `/api/events?target=...` streams +per-target events live. + +## When a fix lands + +1. Look at the bundle in `/agentic/bundles/{id}`. Read `analysis/triage.md`, + `analysis/patch_audit.json`, and `analysis/rebuild_proof.json`. +2. If `rebuild_ok` is true, the dirty edits live in the dev-env's + writable overlay at `$(dportsv3 dev-env path NAME --writable)`. +3. Inspect with `git -C $(dportsv3 dev-env path NAME --writable)/work/DeltaPorts diff`. +4. If you accept the change, copy the diff into your own DeltaPorts + clone, review, sign, and commit there. The agentic loop never + touches your authoritative working tree. + +## Troubleshooting + +| Symptom | Check | +|---|---| +| Runner sees no bundles | Hook output (`dsynth` logs); `state.db` `bundles` table | +| Triage 401s | `DP_HARNESS_TRIAGE_API_KEY` and `_API_BASE` | +| Patch loop stalls | Tracker `/agentic/jobs/{id}` → state=inflight + last_error | +| Artifact 404 | `DPORTSV3_ARTIFACT_ROOT` on the tracker matches `--logs-root` on the artifact-store | +| Hook can't reach tracker | `DPORTSV3_TRACKER_URL` from the dsynth env; some chroot setups need a bind-mount or 127.0.0.1 only | + +For deeper triage of the agent runtime itself, see `docs/TESTING_E2E.md` +for the manual fixtures under `scripts/generator/dportsv3/agent/`. diff --git a/docs/OPERATOR_SETUP.md b/docs/OPERATOR_SETUP.md new file mode 100644 index 00000000000..ba344377579 --- /dev/null +++ b/docs/OPERATOR_SETUP.md @@ -0,0 +1,205 @@ +# Operator setup — agentic DeltaPorts builds, zero to running + +This is the single walkthrough from a fresh DragonFly box to a live +agentic dsynth build with the tracker UI open. For architecture see +`docs/AGENTIC_BUILDS.md`; for testing see `docs/TESTING_E2E.md`. + +## 1. System prerequisites + +DragonFly packages required by the generator + tracker venv: + +```sh +pkg install py311-sqlite3 py311-pydantic2 py311-pydantic-core \ + py311-fastapi py311-uvicorn py311-watchfiles \ + py311-uvloop py311-httptools py311-websockets \ + py311-python-dotenv +``` + +(These keep us off Rust-built wheels — the venv is set up with +`--system-site-packages` to pick them up.) + +You also need `dsynth` itself, of course, and a profile set up +(`/usr/local/etc/dsynth/-make.conf` etc). + +## 2. Clone + bootstrap + +```sh +cd /build/synth # or wherever you keep build trees +git clone https://github.com/.../DeltaPorts.git +cd DeltaPorts +./dportsv3 --help # first run builds the venv at scripts/generator/.venv +``` + +The wrapper script: +- creates `scripts/generator/.venv/` with `--system-site-packages` +- `pip install -e scripts/generator` (editable, source changes pick up live) +- caches a stamp so subsequent calls skip re-install + +If you also want pytest + mypy in there: + +```sh +scripts/generator/.venv/bin/pip install -e 'scripts/generator[dev]' +``` + +## 3. Create a dev-env for your target + +The dev-env is a chroot with a writable copy-on-write overlay where +the agent edits files. One per target/origin you want to iterate on. + +```sh +./dportsv3 dev-env create myenv --target @2026Q2 --origin devel/foo +./dportsv3 dev-env status myenv # expect: status=ready, backend=chroot, root_mounted=true +``` + +`dev-env path myenv --writable` will print the overlay path +(`/var/cache/dports-dev/myenv/writable`) — that's where the agent's +dirty edits land. + +For details (mounts, FPORTS pinning, materialization), see +`docs/dev-chroot-environment.md`. + +## 4. Install dsynth hooks into the dev-env + +dsynth runs inside the chroot, so its hooks belong in the env's +writable `/etc/dsynth` overlay rather than on the host: + +```sh +./dportsv3 dev-env hooks-install myenv +``` + +This copies the hook scripts + `dportsv3-hooks.conf.example` → +`dportsv3-hooks.conf` into `${env_dir}/writable/etc_dsynth/`, which +bind-mounts to `/etc/dsynth` inside the chroot. Existing +`dportsv3-hooks.conf` is preserved (pass `--force` to overwrite). + +Edit the conf and set: + +```sh +ARTIFACT_STORE_URL=http://127.0.0.1:8788 +DPORTSV3_TRACKER_URL=http://127.0.0.1:8080 +DPORTSV3_TRACKER_TARGET=@2026Q2 # defaults from $PROFILE if unset +DPORTSV3_BIN=/build/synth/DeltaPorts/dportsv3 +``` + +Verify with: + +```sh +./dportsv3 dev-env hooks-status myenv +``` + +Reports which hooks are present, whether they're executable, and +whether any are stale vs. the in-repo source. Re-run +`hooks-install` after a `git pull` to refresh them. + +## 5. Configure env for the services + +Pick a logs root that artifact-store + tracker share: + +```sh +LOGS_ROOT=/build/synth/logs +STATE_DB=$LOGS_ROOT/evidence/state.db +ARTIFACT_ROOT=$LOGS_ROOT/evidence +QUEUE_ROOT=$LOGS_ROOT/evidence/queue # where hooks write .job files +``` + +LLM credentials — pick a provider for each phase: + +```sh +export DP_HARNESS_TRIAGE_MODEL=deepseek/deepseek-v4-flash +export DP_HARNESS_TRIAGE_API_KEY=... +export DP_HARNESS_PATCH_MODEL=anthropic/claude-sonnet-4 +export DP_HARNESS_PATCH_API_KEY=... +``` + +For DeepSeek thinking-mode the harness keeps `reasoning_content` on +all turns — works out of the box, no extra config. + +## 6. Start the three services + +In three separate shells or under your service manager of choice: + +```sh +# Shell A — artifact-store (receives bundles, writes state.db + blobs) +./dportsv3 artifact-store --logs-root $LOGS_ROOT + +# Shell B — tracker (UI + read API + SSE) +DPORTSV3_STATE_DB=$STATE_DB \ +DPORTSV3_ARTIFACT_ROOT=$ARTIFACT_ROOT \ + ./dportsv3 tracker serve --port 8080 + +# Shell C — queue runner (claims jobs, runs triage/patch) +DPORTSV3_STATE_DB=$STATE_DB \ +DPORTSV3_TRACKER_URL=http://127.0.0.1:8080 \ +ARTIFACT_STORE_URL=http://127.0.0.1:8788 \ + ./dportsv3 agent-queue-runner --queue-root $QUEUE_ROOT +``` + +Order doesn't matter; each is idempotent on schema init. Open +`http://localhost:8080/` in a browser and confirm the dashboard +loads (it'll be empty until a build runs). + +## 7. Run a build + +```sh +dsynth -p 2026Q2 -S -y build devel/known-failing-port +``` + +The dsynth profile must source `/etc/dsynth/dportsv3-hooks.conf` for +the hooks to fire. With hooks live, watch: + +- `http://localhost:8080/target/@2026Q2` — the dsynth-progress view + updates as builders move through phases. Failed ports appear with a + red pill. +- `http://localhost:8080/agentic/bundles?target=@2026Q2` — failure + bundles as they upload. +- `http://localhost:8080/agentic/jobs?target=@2026Q2&state=pending` + — triage jobs as the runner picks them up. + +## 8. Inspect a result + +For a job that landed `rebuild_ok=true`: + +```sh +curl -s http://localhost:8080/api/bundles/ | python3 -m json.tool +``` + +The `artifacts` array points at `analysis/triage.md`, +`analysis/patch_audit.json`, `analysis/rebuild_proof.json`, and +`analysis/changes.diff`. Each is streamable from +`/api/bundles//artifacts/`. + +The actual edits live in the dev-env's writable overlay: + +```sh +ENV_DIR=$(./dportsv3 dev-env path myenv --writable) +git -C $ENV_DIR/work/DeltaPorts diff +``` + +If you accept the change, apply that diff in your own DeltaPorts +clone, review, sign, and commit there. The agentic loop never +touches your authoritative working tree. + +## Common stumbles + +| Symptom | Fix | +|---|---| +| `dportsv3` says "missing DragonFly packages" | install the `pkg install` list from §1 | +| Hooks don't fire on failure | `./dportsv3 dev-env hooks-status myenv` for stale/missing; confirm the env's `/etc/dsynth/dportsv3-hooks.conf` is being sourced by the dsynth profile inside the chroot | +| Tracker 500s on artifact stream | `DPORTSV3_ARTIFACT_ROOT` doesn't match `--logs-root`/evidence on the artifact-store | +| Triage 401s | provider key wrong, or `DP_HARNESS_TRIAGE_API_BASE` needs to be set for non-default endpoints | +| Patch loop stops with `budget-exhausted` | check trust tier classification in `analysis/triage.md`; consider bumping the tier in `config/agentic-policy.json` | +| Runner sees no jobs after a failure | check `bundles` row exists in `state.db` (hook side); check classification didn't resolve to MANUAL | + +## Upgrading + +When you pull new code: + +```sh +git pull +./dportsv3 dev-env hooks-status myenv # are any hooks stale vs. the new source? +./dportsv3 dev-env hooks-install myenv # re-copy (config preserved) +``` + +Then restart the tracker (uvicorn doesn't auto-reload templates or +static files). Artifact-store + runner read state.db schema on +startup; migrations are idempotent. diff --git a/docs/TESTING_E2E.md b/docs/TESTING_E2E.md new file mode 100644 index 00000000000..f487458315e --- /dev/null +++ b/docs/TESTING_E2E.md @@ -0,0 +1,129 @@ +# End-to-end testing — agentic loop + +Three layers of testing exist for the agentic pipeline. Use the layer +that matches the change you're validating. + +## 1. Unit / integration tests (`pytest`) + +Fast, deterministic, no LLM, no real dsynth, no real dev-env. Run as +part of every change. + +``` +scripts/generator/.venv/bin/python -m pytest scripts/generator/tests/ -q +``` + +Coverage: + +| Area | Test module | +|---|---| +| Tracker DB schema + queries | `test_tracker_api.py`, `test_tracker_integration.py`, `test_tracker_queue.py` | +| State DB shared-writer concurrency | `test_state_db_concurrency.py` | +| Agentic read endpoints | `test_tracker_agentic_endpoints.py` | +| Agentic HTML views | `test_tracker_agentic_views.py` | +| Tracker progress UI adapter | `test_tracker_progress.py` | + +The venv at `scripts/generator/.venv` is created by the `dportsv3` +wrapper on first run. To install pytest + mypy into it: +`scripts/generator/.venv/bin/pip install -e 'scripts/generator[dev]'`. + +## 2. Manual harness fixtures + +For changes to the agent runtime (`dportsv3.agent.*`) where a real +LLM + a real dev-env round-trip is the only way to be sure. Each +fixture takes its config from environment variables and prints what +landed on disk. + +| Fixture | What it exercises | +|---|---| +| `scripts/generator/dportsv3/agent/_manual_test_tool_loop.py` | LLM + tool dispatch loop against a real dev-env, one-shot inspection task | +| `scripts/generator/dportsv3/agent/_manual_test_triage_tier.py` | Triage flow + tier dispatch on fabricated bundles (compile / plist / unknown) | +| `scripts/generator/dportsv3/agent/_manual_test_patch_flow.py` | Full patch flow + attempt loop end-to-end against a real port | + +Common env vars: + +| Var | Meaning | +|---|---| +| `DP_TEST_MODEL` | LiteLLM model string (e.g. `deepseek/deepseek-v4-flash`) | +| `DP_TEST_API_KEY` | Provider key | +| `DP_TEST_API_BASE` | Optional custom endpoint | +| `DP_TEST_ENV` | Name of a prepared `dportsv3 dev-env` (must be `ready`) | + +The patch fixture produces `rebuild_proof.json` + `changes.diff` + +`patch_audit.json` on disk in the bundle output dir; check those to +verify the loop actually built something. + +## 3. Live end-to-end on a real builder + +For shaking out hook / queue / runner / tracker integration as a whole +on the dfly box. Slowest path; use when: + +- A hook contract or bundle schema changed +- Tracker endpoints changed shape +- Trust-tier policy or `agentic-policy.json` was edited + +### Prerequisites + +``` +pkg install py311-sqlite3 py311-pydantic2 py311-pydantic-core \ + py311-fastapi py311-uvicorn py311-watchfiles \ + py311-uvloop py311-httptools py311-websockets \ + py311-python-dotenv +``` + +A prepared dev-env for the target: `dportsv3 dev-env create NAME @TARGET`, +then `dportsv3 dev-env status NAME` must report `ready`. + +### Run + +``` +# 1. Start artifact-store (writes bundles + state.db rows) +dportsv3 artifact-store --logs-root /build/synth/logs & + +# 2. Start tracker (reads state.db, serves UI + read endpoints) +DPORTSV3_STATE_DB=/build/synth/logs/evidence/state.db \ +DPORTSV3_ARTIFACT_ROOT=/build/synth/logs/evidence \ + dportsv3 tracker serve --port 8080 & + +# 3. Start the queue runner +DPORTSV3_STATE_DB=/build/synth/logs/evidence/state.db \ +DPORTSV3_TRACKER_URL=http://127.0.0.1:8080 \ +ARTIFACT_STORE_URL=http://127.0.0.1:8788 \ +DP_HARNESS_TRIAGE_MODEL=... \ +DP_HARNESS_PATCH_MODEL=... \ +DP_HARNESS_TRIAGE_API_KEY=... \ +DP_HARNESS_PATCH_API_KEY=... \ + ./dportsv3 agent-queue-runner --queue-root /build/synth/logs/evidence/queue & + +# 4. Trigger a real build (the dsynth profile should source the hooks) +dsynth -p 2026Q2 -S -y build devel/known-failing-port +``` + +### Verify + +| Step | Check | +|---|---| +| Hook fires on failure | New row in `bundles` for the origin | +| Bundle uploaded | `ls /build/synth/logs/evidence/blobstore/objects/sha256/...` | +| Triage queued | `curl localhost:8080/api/jobs?state=pending` | +| Triage ran | `curl localhost:8080/api/bundles/` → `artifacts` includes `analysis/triage.md` | +| Patch ran (if AUTO/ASSIST) | `analysis/rebuild_proof.json` present, `rebuild_ok` set | +| Tracker UI | Open `http://host:8080/target/@2026Q2` and watch the progress view update | + +### Smoking individual surfaces without a real build + +`docs/agentic-consolidation-plan.md` has a seed-script for state.db that +populates bundles/jobs/runs with mixed targets. Useful for UI smokes +when you don't want to wait for a real dsynth run. The same snippet +works for shaking the agentic read endpoints (`/api/bundles`, +`/api/jobs`, `/api/events`) and the progress adapter +(`/api/progress/{target}/summary.json`). + +## Where things go wrong + +| Symptom | Layer | Diagnosis | +|---|---|---| +| pytest can't import dportsv3 | venv | `pip install -e scripts/generator[dev]` into the venv, not the host Python | +| Manual fixture hangs at the first LLM call | manual harness | API key, custom base URL, or model name routing — check the litellm error string | +| Bundle reaches state.db but no job is enqueued | live e2e | Trust tier resolved to MANUAL, or the runner isn't subscribed to that target | +| Patch loop stops with `budget-exhausted` | manual / live | Token budget too small for the failure class; bump the tier in `config/agentic-policy.json` | +| Tracker UI shows stale data | live e2e | Restart tracker — uvicorn doesn't auto-reload templates/static | diff --git a/docs/agentic-consolidation-plan.md b/docs/agentic-consolidation-plan.md new file mode 100644 index 00000000000..71ffdd42ce6 --- /dev/null +++ b/docs/agentic-consolidation-plan.md @@ -0,0 +1,197 @@ +# Phase 4 — Consolidate state-server + tracker onto one DB + +> **Phases 1–3 shipped.** Phase 1 (`dportsv3 dev-env exec`), Phase 2 +> (`apply-patch` + `process_apply_job` retired), and Phase 3 (opencode +> replaced by `dportsv3.agent`) all landed in the +> `agentic-dsynth-evidence-hooks` branch. Their commits are the +> authoritative record; this document only carries the *current* plan. + +## Goal + +Today the agentic side runs on `state-server` (stdlib `http.server`, +~1500 LOC) reading/writing `state.db`, while build tracking runs on +the FastAPI `dportsv3 tracker serve` against a separate `tracker.db`. +Two processes, two DBs, two SPAs, overlapping concerns. + +Phase 4 collapses this into: + +- **One DB**: `state.db` (the existing one — `tracker.db` retires). + Artifact-store writes failure evidence; tracker writes build runs + + ports status. SQLite WAL + `busy_timeout=5000` + `foreign_keys=ON` + on every connection lets both writers share the file safely. +- **One serve process** for the read/UI surface: the FastAPI tracker + absorbs state-server's read endpoints, then state-server + + `state-server-ui` retire. +- **One hook set**: `dsynth-hooks/` writes both the artifact-store + failure bundle *and* the tracker run state on every event. + +## Status + +| Step | What | State | +|---|---|---| +| 1 | Artifact-store grows tracker tables in `state.db` schema | shipped | +| 1.5 | Artifact-store extracted into `dportsv3.artifact_store` (shim retained) | shipped | +| 2 | `POST /v1/user-context` on artifact-store | shipped | +| 3a | `dsynth-hooks/` + `builderhooks/` merged into one hook set | shipped | +| 4a | `dportsv3 dev-env update` + cache bind-mount inside chroot | shipped | +| 4 | Tracker reads/writes `state.db`; `tracker.db` retires | shipped (26ac0ae1e03) | +| 5 | Tracker agentic-read endpoints + target column on bundles/jobs/runs | shipped (8797550a6ac) | +| 6 | Tracker HTML views (bundle detail, jobs queue, runner) | shipped (fdc7528a24f) | +| 8 | Decommission `state-server` + `state-server-ui` SPA | shipped | + +> **Phase 4 complete.** One DB (`state.db`), one serve process +> (FastAPI tracker), one hook set. Agent queue runner now points +> exclusively at the tracker for bundle/artifact lookups via +> `DPORTSV3_TRACKER_URL` (default `http://127.0.0.1:8080`). + +(There is no step 7 — slot reserved during planning, folded into 6/8.) + +## Step 5 — agentic read endpoints + target column + +Two coupled changes ship together because the endpoints exist to +expose target-scoped views; landing the endpoints without the target +column would harden a known gap that step 8 then can't undo without +schema migration on top of decommission. + +### Schema additions + +`dportsv3.db.schema` gains (idempotent — applied via `MIGRATIONS`, +not `SCHEMA`, since legacy rows exist): + +```sql +ALTER TABLE bundles ADD COLUMN target TEXT; +ALTER TABLE jobs ADD COLUMN target TEXT; +ALTER TABLE runs ADD COLUMN target TEXT; +CREATE INDEX IF NOT EXISTS idx_bundles_target ON bundles(target); +CREATE INDEX IF NOT EXISTS idx_jobs_target ON jobs(target); +CREATE INDEX IF NOT EXISTS idx_runs_target ON runs(target); +``` + +Target is nullable: pre-step-5 rows stay `NULL` and surface as target +`unknown` in filtered queries. Going forward every new row is +written with a target. + +### Hook + writer changes + +`scripts/dsynth-hooks/hook_common.sh` already knows the target — it +defaults `DPORTSV3_TRACKER_TARGET` from `$PROFILE`. The bundle upload +to artifact-store carries it in the POST body. Artifact-store writes +it on the `bundles` row (and on the `runs` row when creating one). +`scripts/agent-queue-runner` writes target on the `jobs` row when +enqueueing. + +### Endpoints to port from state-server + +All read; all live in `dportsv3/tracker/server.py` next to the +existing routes. Per-request SQLite connections (matching the +a14fe9c4dab pattern). + +| state-server route | tracker route | target filter? | +|---|---|---| +| `GET /health` | `GET /api/health` | no | +| `GET /status` | `GET /api/agentic-status` | no (global counts) | +| `GET /activity` | `GET /api/activity?limit=N` | `&target=` | +| `GET /runner-status` | `GET /api/runner-status` | no | +| `GET /runs`, `/runs/` | `GET /api/runs[?target=]`, `/api/runs/{id}` | yes | +| `GET /jobs?state=`, `/jobs/` | `GET /api/jobs[?state=&target=]`, `/api/jobs/{id}` | yes | +| `GET /bundles`, `/bundles/` | `GET /api/bundles[?target=&origin=]`, `/api/bundles/{id}` | yes | +| `GET /ports/` | `GET /api/ports/{origin}[?target=]` | yes | +| `GET /bundles//artifacts/` | `GET /api/bundles/{id}/artifacts/{relpath:path}` | no | +| `GET /events` (SSE) | `GET /api/events[?target=]` (SSE) | yes | + +`user-context*` stays on artifact-store (write path, step 2 settled +it there). + +### Artifact root discovery + +`GET /api/bundles/{id}/artifacts/...` streams from artifact-store's +`blob_root`. The tracker doesn't know that path today. Resolution: +`DPORTSV3_ARTIFACT_ROOT` env var on the tracker process, defaulting +to the artifact-store default (`/build/synth/logs/evidence`). +Symmetric with `DPORTSV3_STATE_DB`. + +### SSE on FastAPI + +State-server hand-rolls SSE on `http.server`. Tracker uses +`StreamingResponse` + async generator that: +- Polls `events` table on a 1s tick. +- Tracks last-seen `id` per connection. +- Emits each new row as `event: \ndata: \n\n`. +- Filters by target if `?target=` is supplied. + +Async polling is fine here — single FastAPI process, low event rate. +No LISTEN/NOTIFY, no external pub/sub. + +### Tests + +One pytest module per route group under `scripts/generator/tests/` +(`test_tracker_agentic_runs.py`, `test_tracker_agentic_bundles.py`, +`test_tracker_agentic_jobs.py`, `test_tracker_agentic_events.py`). +Each seeds `state.db` with rows including a mix of target values +(some `NULL` to verify legacy-row tolerance) and asserts: +- response shape matches the state-server equivalent (no field-name + drift), +- `?target=` filtering works, +- `NULL`-target rows surface only when no filter is set. + +### LOC estimate + +| Area | Added | Notes | +|---|---|---| +| `dportsv3/db/schema.py` migrations | ~10 | three ALTERs + three indexes | +| `dportsv3/tracker/server.py` routes | ~350 | ten read endpoints + SSE | +| `dportsv3/tracker/queries.py` (new) | ~150 | agentic-read SQL helpers | +| `dportsv3/artifact_store.py` target writes | ~20 | bundle/run insert paths | +| `scripts/dsynth-hooks/hook_common.sh` | ~5 | target in POST body | +| `scripts/agent-queue-runner` job-enqueue paths | ~10 | target in jobs insert | +| tests | ~250 | one module per route group | +| **Total** | **~795** | | + +No deletions in this step. State-server keeps running in parallel +until step 8. + +## Step 6 — HTML views + +Add target-scoped views to the tracker: + +- `/agentic/bundles[?target=]` — bundle list +- `/agentic/bundles/{id}` — bundle detail with linked artifacts +- `/agentic/jobs[?target=&state=]` — job queue +- `/agentic/runner` — runner heartbeat / status + +Templates live next to existing `dportsv3/tracker/templates/`. +Server-rendered, no SPA. Style follows the existing tracker +dashboard. + +## Step 8 — decommission state-server + state-server-ui + +- Delete `scripts/state-server` (~1500 LOC). +- Delete `scripts/state-server-ui/` (the React SPA). +- Remove the systemd/rc unit references in `docs/dportsv3-user-guide.md`. +- Update any docs/scripts that point at the state-server port. +- One follow-up commit; predicated on step 5 + step 6 being in + production for at least one build cycle. + +## Out of scope (Phase 4) + +- Tracker UI redesign — Phase 5. +- Authn/authz on agentic-read endpoints — operator-internal for now. +- Migration of historical `state.db` rows on machines that already + ran pre-step-5: those rows keep `target = NULL` and surface in + unfiltered views. No backfill. + +## Verification + +End-to-end check after step 5 lands: + +1. `dportsv3 dev-env exec foo -- /etc/dsynth/05_failed.sh ...` triggers + the hook, uploading a bundle to artifact-store with + `target=$PROFILE` in the POST body. +2. `sqlite3 state.db 'select bundle_id, target from bundles order by + id desc limit 1;'` shows the new row with non-NULL target. +3. `curl tracker:8080/api/bundles?target=@2026Q2` returns it. +4. `curl tracker:8080/api/bundles?target=@main` excludes it. +5. `curl -N tracker:8080/api/events?target=@2026Q2` streams the + `bundle_upserted` event for the same upload. +6. Same checks against `state-server`'s legacy endpoints still pass + (parallel operation until step 8). diff --git a/docs/kedb/README.md b/docs/kedb/README.md new file mode 100644 index 00000000000..0eff328d21d --- /dev/null +++ b/docs/kedb/README.md @@ -0,0 +1,21 @@ +# Known Error Database (KEDB) + +This directory contains markdown files documenting known build issues specific to DragonFlyBSD DPorts. These files are automatically included in the triage agent's context to improve diagnosis accuracy. + +## How It Works + +The `agent-queue-runner` reads all `*.md` files from this directory (except `README.md` and `TEMPLATE.md`) and appends them to the triage payload sent to the `dports-triage` agent. This gives the agent knowledge of known patterns and proven fixes. + +## Adding a New Entry + +1. Copy `TEMPLATE.md` to a new file named after the issue category (e.g., `pthread.md`, `procfs.md`) +2. Fill in the sections with specific patterns, causes, and fixes +3. The runner will automatically pick it up on the next job + +## Guidelines + +1. **Be specific**: Include exact error patterns that can be matched in logs. +2. **Explain the cause**: Help the agent understand *why* this happens on DragonFly. +3. **Provide actionable fixes**: Describe DeltaPorts-style patches (files to modify, flags to add). +4. **Include examples**: Reference actual ports that were fixed this way. +5. **Keep files focused**: One file per issue category. diff --git a/docs/kedb/TEMPLATE.md b/docs/kedb/TEMPLATE.md new file mode 100644 index 00000000000..335e16c594e --- /dev/null +++ b/docs/kedb/TEMPLATE.md @@ -0,0 +1,34 @@ +# Known Issue: + +## Pattern +- `` +- `` + +## Cause +<1-3 sentences explaining why this happens on DragonFlyBSD> + +## Fix + +### Option 1: + + +```makefile +# Example Makefile addition +CFLAGS+= -DSOME_FLAG +``` + +### Option 2: + + +```diff +--- file.orig ++++ file +@@ -1,3 +1,3 @@ + context line +-old line ++new line + context line +``` + +## Examples +- `category/portname`: diff --git a/docs/kedb/dragonfly-source-patches.md b/docs/kedb/dragonfly-source-patches.md new file mode 100644 index 00000000000..54c60ea32d0 --- /dev/null +++ b/docs/kedb/dragonfly-source-patches.md @@ -0,0 +1,88 @@ +# Known Issue: Creating DragonFly-specific source patches + +## Pattern +- When upstream source needs modification for DragonFlyBSD +- Missing preprocessor symbols (e.g., `IFM_IEEE80211_VHT5G`) +- BSD-specific code paths needing conditional compilation +- Header differences from FreeBSD + +## Cause +DragonFlyBSD shares BSD heritage but has diverged from FreeBSD. Some kernel +headers, system macros, and feature flags differ. When upstream code assumes +FreeBSD-specific features, patches are needed to add conditional compilation. + +## Fix + +### Option 1: DragonFly-specific patch in dragonfly/ directory + +Create a patch file at `ports///dragonfly/patch-`: + +```diff +--- src/file.c.orig ++++ src/file.c +@@ -100,6 +100,8 @@ function_name(args) + some_code(); ++#if defined(SOME_DRAGONFLY_SYMBOL) + code_using_symbol(); ++#endif + more_code(); +``` + +**IMPORTANT**: +- Use `.orig` suffix for the original file (standard diff convention) +- The patch file goes in `dragonfly/` NOT `files/` (that's for FreeBSD patches) +- DragonFly patches apply AFTER FreeBSD `files/patch-*` files + +### Option 2: Makefile.DragonFly with CFLAGS + +For simple cases where a preprocessor define suffices: + +```makefile +CFLAGS+= -DMISSING_SYMBOL=0 +``` + +### Option 3: Disable feature via OPTIONS + +If the problematic code is behind an option: + +```makefile +OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NPROBLEMATIC_OPTION} +``` + +## Line Number Considerations + +When creating DragonFly patches, be aware that: + +1. **FreeBSD patches apply first**: If `files/patch-foo` exists and modifies the + same file, your line numbers must account for those changes + +2. **Check the intermediate state**: The source your patch applies to is AFTER + FreeBSD patches but BEFORE DragonFly patches + +3. **Use enough context**: Include 3+ context lines to handle minor line shifts + +## Diff Format for Overlay + +When generating a diff for the DeltaPorts overlay, the path format is: + +```diff +--- /dev/null ++++ b/ports/net/example/dragonfly/patch-vht5g-guard +@@ -0,0 +1,10 @@ ++--- src/drivers/driver_bsd.c.orig +++++ src/drivers/driver_bsd.c ++@@ -638,6 +638,8 @@ bsd_set_freq(void *priv, struct hostapd_freq_params *freq) ++ } else { +++#if defined(IFM_IEEE80211_VHT5G) ++ mode = freq->vht_enabled ? IFM_IEEE80211_VHT5G : +++#endif ++ ... +``` + +Note: This creates a NEW file containing the patch content. + +**Agent guidance (patch jobs):** Do NOT output diff text or write patch files directly. Use the `dports_*` tools (`dupe`/`genpatch`/`install_patches`) to generate one `patch-*` file per source file under `dragonfly/`. + +## Examples +- `net/hostapd`: Missing `IFM_IEEE80211_VHT5G` symbol - wrapped in `#if defined()` guard +- `net/wpa_supplicant`: Similar VHT5G issue on older DragonFly versions diff --git a/docs/kedb/freebsd-only-features.md b/docs/kedb/freebsd-only-features.md new file mode 100644 index 00000000000..d334e066aff --- /dev/null +++ b/docs/kedb/freebsd-only-features.md @@ -0,0 +1,93 @@ +# Known Issue: FreeBSD-Only Base Features + +## Pattern +- `fatal error: blacklist.h: No such file or directory` +- `fatal error: sys/audit.h: No such file or directory` +- `fatal error: sys/capsicum.h: No such file or directory` +- `-DUSE_BLACKLIST` in compile command with missing `blacklist.h` +- `undefined reference to 'blacklist_open'` +- `undefined reference to 'cap_enter'` +- Build environment shows `USE_LIBBLACKLIST=yes` or similar FreeBSD-only option enabled + +## Cause +FreeBSD base system includes several facilities that do not exist in DragonFlyBSD: + +| Feature | FreeBSD Header/Lib | Description | +|---------|-------------------|-------------| +| blacklistd | `blacklist.h`, `libblacklist` | Connection blacklisting daemon | +| Capsicum | `sys/capsicum.h` | Capability-based sandboxing | +| BSM Audit | `sys/audit.h`, `libbsm` | Basic Security Module auditing | +| SCTP | `netinet/sctp.h` | Stream Control Transmission Protocol | +| utmpx | `utmpx.h` | Extended user accounting (partial) | + +When a port has an OPTION that enables these features (often ON by default in FreeBSD ports), the build fails on DragonFly because the headers/libraries are missing. + +## Fix + +### Option 1: Disable the option in Makefile.DragonFly (Preferred) + +If the port defines an OPTION for the feature, disable it by default on DragonFly using the `:N` pattern to remove from OPTIONS_DEFAULT: + +```makefile +# ports/category/portname/Makefile.DragonFly +OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NLIBBLACKLIST} +``` + +The `:N` modifier removes matching items from the list. Common option names: + +| Missing Feature | Likely Option Name(s) | +|-----------------|----------------------| +| blacklistd | `BLACKLIST`, `LIBBLACKLIST` | +| Capsicum | `CAPSICUM` | +| BSM Audit | `AUDIT`, `BSM` | +| SCTP | `SCTP` | +| PulseAudio | `PULSEAUDIO` | +| udev/libudev | `UDEV` | +| systemd/basu | `BASU`, `SYSTEMD` | + +Multiple options can be disabled in one line: + +```makefile +OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NPULSEAUDIO:NUDEV} +``` + +### Option 2: Add replacement option (when alternative exists) + +Sometimes DragonFly has an alternative. Use `:N` to remove the bad option and append the good one: + +```makefile +# Remove MIT krb5 from base, use port instead +OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NGSSAPI_BASE} GSSAPI_MIT +``` + +```makefile +# Use ALSA instead of PulseAudio +OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NPULSEAUDIO} ALSA +``` + +### Option 3: Patch source to make feature optional (Last Resort) + +If the port hardcodes the feature without an OPTION, create `diffs/patch-*.diff` to wrap the code in `#ifdef` guards or remove the problematic includes. + +## Examples +- `net/bsdrcmds`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NLIBBLACKLIST}` - disables blacklistd support +- `www/bozohttpd`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NBLACKLIST}` - same pattern, different option name +- `security/sudo`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NAUDIT}` - disables BSM audit +- `net/yate`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NSCTP}` - disables SCTP protocol support +- `x11/waybar`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NPULSEAUDIO:NUDEV}` - disables multiple missing features +- `x11-wm/sway`: `OPTIONS_DEFAULT:= ${OPTIONS_DEFAULT:NBASU}` - disables systemd/basu integration + +## Triage Classification +This error type is **patchable** when: +1. The error is a missing header/library for a known FreeBSD-only feature +2. The port has an OPTION that controls the feature +3. The feature is not essential to the port's core functionality + +Classify as `freebsd-feature` with `high` confidence when the pattern matches. + +## Detection Hints +To identify the correct OPTION to disable: +1. Check errors.txt for the missing header (e.g., `blacklist.h`) +2. Search the port's Makefile for related OPTION definitions +3. Look for `-DUSE_*` flags in the failing compile command +4. Check if DeltaPorts already has a fix for similar ports diff --git a/docs/kedb/plist-mismatch.md b/docs/kedb/plist-mismatch.md new file mode 100644 index 00000000000..467736df164 --- /dev/null +++ b/docs/kedb/plist-mismatch.md @@ -0,0 +1,109 @@ +# Known Issue: check-plist phase failures (Orphaned / Missing files) + +## Pattern +- `Error: Orphaned: ` +- `Error: Missing: ` +- `===> Error: Plist issues found.` +- Stage succeeds but `check-plist` phase fails + +Common variants: +- `Error: Orphaned: /man/man8/foo.8.gz` (file installed but not in plist) +- `Error: Missing: man/man8/foo.8.gz` (plist expects file but not installed) +- `Error: Orphaned: @dir /some/path` (absolute path in plist causes mismatch) + +## Cause +The `check-plist` phase compares files actually installed in `STAGEDIR` against entries in `pkg-plist` (or `PLIST_FILES`). Mismatches occur when: + +1. **Orphaned**: A file is installed but not listed in the plist (DragonFly builds extra files, or upstream plist is incomplete) +2. **Missing**: A file is listed in plist but not installed (DragonFly build skips some files, or upstream plist has stale entries) +3. **Path mismatch**: Absolute paths like `/etc/...` in plist vs relative `etc/...` expected by the framework + +On DragonFlyBSD, differences in build configuration, enabled options, or platform-specific code paths can cause the installed file set to differ from FreeBSD. + +## Fix + +### Option 1: Single or few missing plist entries → `PLIST_FILES+=` in Makefile.DragonFly + +For **Orphaned** errors (file exists but not in plist), add the missing entry: + +```makefile +# ports///Makefile.DragonFly +PLIST_FILES+= man/man8/foo.8.gz +``` + +For multiple related entries: +```makefile +PLIST_FILES+= libexec/nut/microsol-apc \ + man/man8/microsol-apc.8.gz +``` + +This is the preferred fix when only a small number of files need to be added. + +### Option 2: Multiple changes or removals → `diffs/pkg-plist.diff` + +For complex plist changes (many additions, removals, or path rewrites), create a unified diff: + +```diff +# ports///diffs/pkg-plist.diff +--- pkg-plist.orig 2023-01-02 19:50:10.000000000 +0100 ++++ pkg-plist 2023-01-02 19:50:37.000000000 +0100 +@@ -1,5 +1,5 @@ +-@dir /etc/X11/xrdp +-/etc/X11/xrdp/xorg.conf ++@dir etc/X11/xrdp ++etc/X11/xrdp/xorg.conf + lib/xorg/modules/drivers/xrdpdev_drv.so +``` + +Use this approach when: +- Multiple lines need to be added or removed +- Absolute paths need to be converted to relative paths +- Upstream plist has stale entries that should be removed + +### Important: Avoid overriding FreeBSD-only make targets + +Do NOT override standard targets like `do-install`, `post-install`, `pre-install` in `Makefile.DragonFly` — these can conflict with the FreeBSD port's own definitions. + +Instead, use DeltaPorts-specific hooks: +- `dfly-patch:` — runs after `post-patch` +- `dfly-configure:` — runs after `post-configure` +- `dfly-build:` — runs after `post-build` +- `dfly-install:` — runs after `post-install` (for cleanup/fixups only) + +For plist issues specifically, prefer `PLIST_FILES+=` or `diffs/pkg-plist.diff` over install-phase hacks. + +## Examples + +### Adding missing files via PLIST_FILES +- `sysutils/nut-devel`: Fixed orphaned files (commit `8ffa24d43d6`) + ```makefile + PLIST_FILES+= libexec/nut/microsol-apc \ + man/man8/microsol-apc.8.gz + ``` + +- `www/jira-cli`: Added missing manpage + ```makefile + PLIST_FILES+= man/man7/jira-sprint-add.7.gz + ``` + +- `sysutils/cpu-microcode-intel`: Added directory entry (commit `440707d6763`) + ```makefile + PLIST_FILES+= "@dir /boot/firmware" + ``` + +### Patching pkg-plist via diffs/ +- `x11-drivers/xorgxrdp`: Fixed absolute paths in plist + ```diff + -@dir /etc/X11/xrdp + +@dir etc/X11/xrdp + ``` + +- `net/freeswitch`: Added missing module (commit `2e2d6b59d43`) + ```diff + +lib/freeswitch/mod/mod_av.so + ``` + +- `graphics/qt5-3d`: Added multiple missing cmake/plugin files (commit `9e2be93e20d`) + +### Removing stale overlay entries +- `x11/cinnamon`: Removed `Makefile.DragonFly` that was adding obsolete plist entries (commit `e2753b339e6`) diff --git a/dportsv3 b/dportsv3 index fabf056ac97..4010f5ead62 100755 --- a/dportsv3 +++ b/dportsv3 @@ -39,7 +39,9 @@ generator commands: compose-report Summarize compose JSON report for humans/tools dsl DSL parse/check/plan commands migrate Migration utilities (compose-first workflow) - tracker Build tracker server and queries + tracker Build tracker server and queries + artifact-store Run the artifact-store HTTP service + agent-queue-runner Run the agent queue runner (triage + patch loop) Use 'dportsv3 COMMAND --help' for command-specific help. EOF @@ -168,9 +170,24 @@ if [ "$needs_install" -eq 1 ]; then exit 1 fi + if [ "$(uname -s)" = "DragonFly" ]; then + REQUIRED_DFLY_PKGS="py311-sqlite3 py311-pydantic2 py311-pydantic-core py311-fastapi py311-uvicorn py311-watchfiles py311-uvloop py311-httptools py311-websockets py311-python-dotenv" + missing_pkgs= + for p in $REQUIRED_DFLY_PKGS; do + pkg info -e "$p" >/dev/null 2>&1 || missing_pkgs="$missing_pkgs $p" + done + if [ -n "$missing_pkgs" ]; then + printf '%s\n' "dportsv3: missing DragonFly packages required for the generator/tracker venv:" >&2 + printf '%s\n' " pkg install$missing_pkgs" >&2 + printf '%s\n' "These satisfy pydantic / fastapi / uvicorn[standard] without pulling Rust-built wheels." >&2 + printf '%s\n' "Re-run dportsv3 after installing." >&2 + exit 1 + fi + fi + if [ ! -d "$VENV_DIR" ]; then printf '%s\n' "dportsv3: creating virtual environment at $VENV_DIR" >&2 - "$PYTHON_BIN" -m venv "$VENV_DIR" + "$PYTHON_BIN" -m venv --system-site-packages "$VENV_DIR" fi printf '%s\n' "dportsv3: installing generator ($INSTALL_PROFILE)" >&2 diff --git a/scripts/agent-queue-runner b/scripts/agent-queue-runner new file mode 100755 index 00000000000..bd6f3e470e6 --- /dev/null +++ b/scripts/agent-queue-runner @@ -0,0 +1,1691 @@ +#!/usr/bin/env python3 +""" +agent-queue-runner: Process dsynth failure jobs via the dportsv3.agent harness. + +Usage: + agent-queue-runner --queue-root [--once] [--dry-run] + +Required env vars (one of triage or patch model must be set for the +relevant job type): + DP_HARNESS_TRIAGE_MODEL litellm model string for triage + DP_HARNESS_PATCH_MODEL litellm model string for patch + DP_HARNESS_*_API_KEY provider API keys + DP_HARNESS_*_API_BASE optional custom endpoint + DP_HARNESS_*_PROVIDER optional custom_llm_provider override + DP_HARNESS_ENV dev-env name for patch tool dispatch + DP_HARNESS_POLICY optional path to agentic-policy.json + DP_HARNESS_TIMEOUT triage timeout (default 120) + DP_HARNESS_PATCH_TIMEOUT patch timeout (default 600) + DP_HARNESS_MAX_SNIPPET_ROUNDS default 5 + +Job types: + type=triage (default) — runs dportsv3.agent.triage; classification + + confidence drive trust-tier dispatch. + type=patch — runs dportsv3.agent.patch with the resolved tier. + +Job fields: + iteration=N — current fix iteration (1-based) + max_iterations=N — max iterations before giving up (default: 3) + tier=NAME — pre-resolved trust tier (set by triage step) + dev_env=NAME — dev-env to use (set by triage step or DP_HARNESS_ENV) + previous_bundle=... — bundle from previous failed attempt +""" + +import argparse +import json +import os +import re +import sqlite3 +import subprocess +import sys +import threading +import time +import urllib.request +import urllib.error +import urllib.parse +from datetime import datetime, timezone +from pathlib import Path + +# Make dportsv3.agent.* importable from the standalone runner script. +# The package lives in scripts/generator/dportsv3/; this script lives in scripts/. +_GENERATOR_DIR = Path(__file__).resolve().parent / "generator" +if _GENERATOR_DIR.is_dir() and str(_GENERATOR_DIR) not in sys.path: + sys.path.insert(0, str(_GENERATOR_DIR)) + + +# Max fix iterations before giving up on a port +DEFAULT_MAX_ITERATIONS = 3 + +DEFAULT_ARTIFACT_STORE_URL = "http://127.0.0.1:8788" +DEFAULT_TRACKER_URL = "http://127.0.0.1:8080" + +# Heartbeat interval (seconds) +HEARTBEAT_INTERVAL = 5 + + +# ============================================================================= +# State DB connection (for activity logging and runner status) +# ============================================================================= + +_state_db_conn: sqlite3.Connection | None = None +_state_db_lock = threading.Lock() +_heartbeat_stop_event = threading.Event() +_heartbeat_thread: threading.Thread | None = None +_current_job_id: str | None = None +_current_stage: str | None = None + + +def get_state_db_path(queue_root: Path) -> Path: + """Get path to state.db (same directory as queue).""" + # Queue is at /evidence/queue/, state.db is at /evidence/state.db + return queue_root.parent / "state.db" + + +def init_state_db(queue_root: Path) -> sqlite3.Connection | None: + """Initialize connection to state.db for activity logging.""" + global _state_db_conn + + db_path = get_state_db_path(queue_root) + + if not db_path.exists(): + # State server hasn't created the DB yet - that's ok + return None + + try: + conn = sqlite3.connect(str(db_path), check_same_thread=False) + conn.row_factory = sqlite3.Row + _state_db_conn = conn + return conn + except Exception as e: + print(f"Warning: Could not connect to state.db: {e}", file=sys.stderr) + return None + + +def _artifact_store_url() -> str: + return os.environ.get("ARTIFACT_STORE_URL", DEFAULT_ARTIFACT_STORE_URL) + + +def _tracker_url() -> str: + return os.environ.get("DPORTSV3_TRACKER_URL", DEFAULT_TRACKER_URL) + + +def artifact_store_get(bundle_id: str, relpath: str) -> bytes | None: + url = f"{_artifact_store_url()}/v1/artifacts/get?bundle_id={urllib.parse.quote(bundle_id)}&relpath={urllib.parse.quote(relpath)}" + try: + with urllib.request.urlopen(url, timeout=10) as resp: + return resp.read() + except Exception: + return None + + +def tracker_artifact_get(bundle_id: str, relpath: str) -> bytes | None: + url = ( + f"{_tracker_url()}/api/bundles/{urllib.parse.quote(bundle_id)}" + f"/artifacts/{urllib.parse.quote(relpath, safe='/')}" + ) + try: + with urllib.request.urlopen(url, timeout=10) as resp: + return resp.read() + except Exception: + return None + + +def artifact_store_put(bundle_id: str, relpath: str, data: bytes, kind: str | None = None) -> bool: + url = f"{_artifact_store_url()}/v1/artifacts/put" + headers = { + "Accept": "application/json", + "Content-Type": "application/octet-stream", + "X-Bundle-Id": bundle_id, + "X-Relpath": relpath, + } + if kind: + headers["X-Kind"] = kind + try: + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=20): + return True + except Exception: + return False + + +def bundle_artifact_list(bundle_id: str) -> list[str]: + url = f"{_tracker_url()}/api/bundles/{urllib.parse.quote(bundle_id)}" + try: + with urllib.request.urlopen(url, timeout=10) as resp: + data = json.load(resp) + return [a.get("relpath") for a in data.get("artifacts", []) if a.get("relpath")] + except Exception: + return [] + + +def port_bundle_history(origin: str) -> list[dict]: + # Tracker /api/ports/ returns a flat list (vs. legacy + # state-server's {"origin", "bundles", "jobs"} shape). + url = f"{_tracker_url()}/api/ports/{urllib.parse.quote(origin)}" + try: + with urllib.request.urlopen(url, timeout=10) as resp: + data = json.load(resp) + return data if isinstance(data, list) else data.get("bundles", []) + except Exception: + return [] + + +def read_bundle_text(bundle_dir: Path | None, bundle_id: str | None, relpath: str) -> str | None: + if bundle_dir: + path = bundle_dir / relpath + if path.exists(): + return read_file_if_exists(path) + if bundle_id: + data = artifact_store_get(bundle_id, relpath) + if data is None: + data = tracker_artifact_get(bundle_id, relpath) + if data is not None: + return data.decode("utf-8", errors="replace") + return None + + +def bundle_artifact_exists(bundle_dir: Path | None, bundle_id: str | None, relpath: str) -> bool: + if bundle_dir: + if (bundle_dir / relpath).exists(): + return True + if bundle_id: + return relpath in bundle_artifact_list(bundle_id) + return False + + +def run_cmd(cmd: list[str], cwd: Path | None = None) -> str: + env = os.environ.copy() + env["GIT_TERMINAL_PROMPT"] = "0" + result = subprocess.run( + cmd, + cwd=cwd, + text=True, + capture_output=True, + env=env, + ) + if result.returncode != 0: + stderr = result.stderr.strip() + stdout = result.stdout.strip() + detail = stderr or stdout or "unknown error" + raise RuntimeError(f"command failed ({result.returncode}): {' '.join(cmd)}: {detail}") + return result.stdout + + + +def parse_meta_kv(bundle_dir: Path) -> dict: + """Parse bundle meta.txt into dict (legacy filesystem mode).""" + data = {} + meta_path = bundle_dir / "meta.txt" + if not meta_path.exists(): + return data + try: + for line in meta_path.read_text().splitlines(): + if "=" in line: + key, _, value = line.partition("=") + data[key.strip()] = value.strip() + except OSError: + pass + return data + + +def get_run_profile(run_id: str) -> str: + if _state_db_conn is None: + return "unknown" + try: + with _state_db_lock: + row = _state_db_conn.execute( + "SELECT profile FROM runs WHERE run_id = ?", + (run_id,), + ).fetchone() + if row and row["profile"]: + return row["profile"] + except Exception: + pass + return "unknown" + + +def get_bundle_flavor(bundle_id: str) -> str: + if _state_db_conn is None: + return "" + try: + with _state_db_lock: + row = _state_db_conn.execute( + "SELECT flavor FROM bundles WHERE bundle_id = ?", + (bundle_id,), + ).fetchone() + if row and row["flavor"]: + return row["flavor"] + except Exception: + pass + return "" + + +def get_user_context(run_id: str | None, origin: str | None) -> tuple[str | None, int]: + """Fetch user context for run_id+origin from state.db.""" + if _state_db_conn is None or not run_id or not origin: + return None, 0 + try: + with _state_db_lock: + row = _state_db_conn.execute( + """SELECT context_text, context_rev FROM user_context + WHERE run_id = ? AND origin = ?""", + (run_id, origin) + ).fetchone() + if not row: + return None, 0 + return row["context_text"], int(row["context_rev"]) + except Exception: + return None, 0 + + +def upsert_user_context_request( + queue_root: Path, + run_id: str, + origin: str, + bundle_id: str, + classification: str, + confidence: str, + iteration: int, + max_iterations: int, +): + """Record a request for user context in state.db.""" + if _state_db_conn is None: + return + now = datetime.now(timezone.utc).isoformat() + _, context_rev = get_user_context(run_id, origin) + try: + with _state_db_lock: + row = _state_db_conn.execute( + """SELECT last_context_rev_handled FROM user_context_requests + WHERE run_id = ? AND origin = ? AND bundle_id = ?""", + (run_id, origin, bundle_id) + ).fetchone() + if row: + _state_db_conn.execute( + """UPDATE user_context_requests + SET confidence = ?, classification = ?, iteration = ?, + max_iterations = ?, requested_at = ?, status = 'pending' + WHERE run_id = ? AND origin = ? AND bundle_id = ?""", + (confidence, classification, iteration, max_iterations, now, run_id, origin, bundle_id) + ) + else: + _state_db_conn.execute( + """INSERT INTO user_context_requests + (run_id, origin, bundle_id, confidence, classification, iteration, + max_iterations, requested_at, status, last_context_rev_handled) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'pending', ?)""", + (run_id, origin, bundle_id, confidence, classification, iteration, max_iterations, now, context_rev) + ) + _state_db_conn.commit() + except Exception as e: + print(f"Warning: Failed to write user_context_request: {e}", file=sys.stderr) + + +def find_latest_bundle_id(run_id: str, origin: str) -> str | None: + """Find latest bundle_id for run_id+origin from state.db.""" + if _state_db_conn is None: + return None + try: + with _state_db_lock: + row = _state_db_conn.execute( + """SELECT bundle_id FROM bundles + WHERE run_id = ? AND origin = ? + ORDER BY ts_utc DESC LIMIT 1""", + (run_id, origin), + ).fetchone() + if row: + return row["bundle_id"] + except Exception: + return None + return None + + +def enqueue_triage_job( + queue_root: Path, + bundle_id: str, + run_id: str, + origin: str, + profile: str, + flavor: str, + iteration: int, + max_iterations: int, + previous_bundle: str | None, + context_rev: int, +) -> Path: + """Enqueue a triage job for the given bundle.""" + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%SZ") + origin_safe = origin.replace("/", "_") + pid = os.getpid() + job_name = f"{ts}-{profile}-{origin_safe}-{pid}.job" + pending_dir = queue_root / "pending" + job_path = pending_dir / job_name + + content = [ + "type=triage", + f"created_ts_utc={ts}", + f"profile={profile}", + f"origin={origin}", + f"flavor={flavor}", + f"bundle_id={bundle_id}", + f"run_id={run_id}", + f"iteration={iteration}", + f"max_iterations={max_iterations}", + f"user_context_rev={context_rev}", + ] + if previous_bundle: + content.append(f"previous_bundle={previous_bundle}") + + tmp_path = job_path.with_suffix(".tmp") + with open(tmp_path, "w") as f: + f.write("\n".join(content) + "\n") + tmp_path.rename(job_path) + return job_path + + +def process_user_context_updates(queue_root: Path): + """Enqueue triage jobs when new user context is provided.""" + if _state_db_conn is None: + return + try: + with _state_db_lock: + rows = _state_db_conn.execute( + """SELECT run_id, origin, bundle_id, iteration, max_iterations, + last_context_rev_handled + FROM user_context_requests WHERE status = 'pending' + ORDER BY requested_at ASC""" + ).fetchall() + for row in rows: + run_id = row["run_id"] + origin = row["origin"] + last_handled = int(row["last_context_rev_handled"]) + context_text, context_rev = get_user_context(run_id, origin) + if not context_text or context_rev <= last_handled: + continue + latest_bundle_id = find_latest_bundle_id(run_id, origin) + if not latest_bundle_id: + continue + iteration = int(row["iteration"] or 1) + max_iterations = int(row["max_iterations"] or DEFAULT_MAX_ITERATIONS) + previous_bundle = row["bundle_id"] + profile = get_run_profile(run_id) + flavor = get_bundle_flavor(latest_bundle_id) + job_path = enqueue_triage_job( + queue_root, latest_bundle_id, run_id, origin, + profile, flavor, iteration, max_iterations, previous_bundle, context_rev, + ) + activity_log(queue_root, "retriage_enqueued", + f"Re-running triage for {origin} after user context", + job_id=job_path.name, + extra={"run_id": run_id, "origin": origin, "context_rev": context_rev}) + with _state_db_lock: + _state_db_conn.execute( + """UPDATE user_context_requests + SET last_context_rev_handled = ?, status = 'retriage_enqueued' + WHERE run_id = ? AND origin = ? AND bundle_id = ?""", + (context_rev, run_id, origin, row["bundle_id"]) + ) + _state_db_conn.commit() + except Exception as e: + print(f"Warning: Failed to process user context updates: {e}", file=sys.stderr) + + +def activity_log( + queue_root: Path, + stage: str, + message: str, + job_id: str | None = None, + duration_ms: int | None = None, + extra: dict | None = None +): + """ + Log activity to state.db activity_log table. + Also updates _current_stage for heartbeat. + Keeps only last 10 entries. + """ + global _current_stage + _current_stage = stage + + # Also write to runner.log for backwards compatibility + log(queue_root, "INFO", f"[{stage}] {message}") + + if _state_db_conn is None: + return + + ts = datetime.now(timezone.utc).isoformat() + extra_json = json.dumps(extra) if extra else None + + try: + with _state_db_lock: + _state_db_conn.execute( + """INSERT INTO activity_log (ts, job_id, stage, message, duration_ms, extra_json) + VALUES (?, ?, ?, ?, ?, ?)""", + (ts, job_id, stage, message, duration_ms, extra_json) + ) + + # Prune to keep only last 10 entries + _state_db_conn.execute( + """DELETE FROM activity_log WHERE id NOT IN ( + SELECT id FROM activity_log ORDER BY id DESC LIMIT 10 + )""" + ) + + _state_db_conn.commit() + except Exception as e: + print(f"Warning: Failed to write activity log: {e}", file=sys.stderr) + + +def update_runner_status( + status: str, + job_id: str | None = None, + stage: str | None = None, + extra: dict | None = None +): + """Update runner_status table (singleton row).""" + global _current_job_id, _current_stage + + _current_job_id = job_id + if stage is not None: + _current_stage = stage + + if _state_db_conn is None: + return + + ts = datetime.now(timezone.utc).isoformat() + extra_json = json.dumps(extra) if extra else None + + try: + with _state_db_lock: + # Upsert the singleton row + _state_db_conn.execute( + """INSERT INTO runner_status (id, status, job_id, current_stage, started_at, updated_at, extra_json) + VALUES (1, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + status = excluded.status, + job_id = excluded.job_id, + current_stage = excluded.current_stage, + started_at = CASE WHEN excluded.job_id != runner_status.job_id THEN excluded.started_at ELSE runner_status.started_at END, + updated_at = excluded.updated_at, + extra_json = excluded.extra_json""", + (status, job_id, stage or _current_stage, ts, ts, extra_json) + ) + _state_db_conn.commit() + except Exception as e: + print(f"Warning: Failed to update runner status: {e}", file=sys.stderr) + + +def _heartbeat_loop(): + """Background thread that updates runner_status.updated_at every 5 seconds.""" + while not _heartbeat_stop_event.is_set(): + if _state_db_conn is not None: + try: + ts = datetime.now(timezone.utc).isoformat() + with _state_db_lock: + _state_db_conn.execute( + """UPDATE runner_status SET updated_at = ? WHERE id = 1""", + (ts,) + ) + _state_db_conn.commit() + except Exception: + pass + + _heartbeat_stop_event.wait(HEARTBEAT_INTERVAL) + + +def start_heartbeat(): + """Start the heartbeat thread.""" + global _heartbeat_thread + + if _heartbeat_thread is not None: + return + + _heartbeat_stop_event.clear() + _heartbeat_thread = threading.Thread(target=_heartbeat_loop, daemon=True) + _heartbeat_thread.start() + + +def stop_heartbeat(): + """Stop the heartbeat thread.""" + global _heartbeat_thread + + _heartbeat_stop_event.set() + if _heartbeat_thread is not None: + _heartbeat_thread.join(timeout=2) + _heartbeat_thread = None + + +def log(queue_root: Path, level: str, message: str): + """Log to both stderr and runner.log.""" + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + line = f"{ts} {level:5} {message}" + print(line, file=sys.stderr) + try: + with open(queue_root / "runner.log", "a") as f: + f.write(line + "\n") + except OSError: + pass + + +def parse_job_file(path: Path) -> dict: + """Parse key=value job file into dict.""" + data = {} + with open(path) as f: + for line in f: + line = line.strip() + if "=" in line: + key, _, value = line.partition("=") + data[key] = value + return data + + +def read_file_if_exists(path: Path, max_bytes: int = 200_000) -> str | None: + """Read file contents if it exists, truncate if too large.""" + if not path.exists(): + return None + try: + content = path.read_text(errors="replace") + if len(content) > max_bytes: + content = content[:max_bytes] + "\n[...truncated...]\n" + return content + except OSError: + return None + + +def find_kedb_dir() -> Path | None: + """Find the KEDB directory relative to this script or in DeltaPorts repo.""" + script_dir = Path(__file__).resolve().parent + kedb_dir = script_dir.parent / "docs" / "kedb" + if kedb_dir.exists(): + return kedb_dir + return None + + +def load_kedb(kedb_dir: Path | None) -> str: + """Load all KEDB markdown files into a single context block.""" + if not kedb_dir or not kedb_dir.exists(): + return "" + + kedb_files = sorted(kedb_dir.glob("*.md")) + skip_files = {"readme.md", "template.md"} + kedb_files = [f for f in kedb_files if f.name.lower() not in skip_files] + + if not kedb_files: + return "" + + parts = ["## Known Error Database (KEDB)", ""] + parts.append("The following are known DragonFlyBSD-specific build issues and their fixes:") + parts.append("") + + for kf in kedb_files: + content = read_file_if_exists(kf, max_bytes=50_000) + if content: + parts.append(f"### {kf.stem}") + parts.append(content) + parts.append("") + + return "\n".join(parts) + + +# ----------------------------------------------------------------------------- +# Triage parsing +# ----------------------------------------------------------------------------- + +def parse_triage_output(content: str | None) -> dict: + """Extract Classification and Confidence from triage.md content.""" + result = {"classification": "", "confidence": "", "raw": ""} + + if not content: + return result + + result["raw"] = content + + # Extract Classification + match = re.search(r"^##\s*Classification\s*\n+([^\n#]+)", content, re.MULTILINE | re.IGNORECASE) + if match: + result["classification"] = match.group(1).strip().lower() + + # Extract Confidence + match = re.search(r"^##\s*Confidence\s*\n+([^\n#]+)", content, re.MULTILINE | re.IGNORECASE) + if match: + result["confidence"] = match.group(1).strip().lower() + + return result + + + +def build_snippet_feedback(bundle_dir: Path, round_num: int) -> str: + """Generate feedback section from snippet manifest for agent context.""" + manifest_path = bundle_dir / "analysis" / "snippets" / "manifest.json" + if not manifest_path.exists(): + return "" + + try: + with open(manifest_path) as f: + manifest = json.load(f) + except Exception: + return "" + + rounds = manifest.get("rounds", []) + if not rounds: + return "" + + # Find the latest round + latest_round = None + for r in rounds: + if r.get("round") == round_num: + latest_round = r + break + + if not latest_round: + # Use the last round + latest_round = rounds[-1] + + parts = ["## Snippet Extraction Results", ""] + parts.append(f"**Round {latest_round.get('round', '?')}** | Source: `{latest_round.get('source', 'unknown')}` | Budget remaining: {latest_round.get('budget_remaining', 0)} bytes") + parts.append("") + + requests = latest_round.get("requests", []) + if requests: + parts.append("| Request | Status | Output | Bytes |") + parts.append("|---------|--------|--------|-------|") + for req in requests: + raw = req.get("raw", "?")[:40] + status = req.get("status", "?") + output = req.get("output", "-") + if output and len(output) > 30: + output = "..." + output[-27:] + bytes_ = req.get("bytes", 0) + note = req.get("note", "") + + # Add emoji for status + status_display = { + "ok": "ok", + "not_found": "not_found", + "budget_exceeded": "budget_exceeded", + "empty": "empty", + }.get(status, status) + + parts.append(f"| `{raw}` | {status_display} | {output or '-'} | {bytes_} |") + if note: + parts.append(f"| | *{note}* | | |") + + parts.append("") + + # Add summary + total_rounds = manifest.get("total_rounds", 0) + max_rounds = int(os.environ.get("DP_HARNESS_MAX_SNIPPET_ROUNDS", "5")) + remaining_rounds = max_rounds - total_rounds + + parts.append(f"**Snippet rounds used:** {total_rounds}/{max_rounds} (remaining: {remaining_rounds})") + if remaining_rounds <= 0: + parts.append("**NOTE:** No more snippet rounds available. Work with the information provided.") + parts.append("") + + return "\n".join(parts) + + +def load_snippets_content(bundle_dir: Path, round_num: int, max_bytes: int = 200_000) -> str: + """Load extracted snippet contents for inclusion in payload.""" + round_dir = bundle_dir / "analysis" / "snippets" / f"round_{round_num}" + if not round_dir.exists(): + return "" + + parts = ["## Extracted Snippets", ""] + total_bytes = 0 + + # Load round manifest for context + manifest_path = round_dir / "manifest.json" + if manifest_path.exists(): + try: + with open(manifest_path) as f: + round_manifest = json.load(f) + source_type = round_manifest.get("source", "unknown") + distfile = round_manifest.get("distfile") + if distfile: + parts.append(f"*Source: distfile `{distfile}`*") + else: + parts.append(f"*Source: {source_type}*") + parts.append("") + except Exception: + pass + + # Walk through subdirectories (source, buildsystem, configure, log) + for subdir in sorted(round_dir.iterdir()): + if not subdir.is_dir() or subdir.name.startswith("."): + continue + + for file_path in sorted(subdir.glob("*.txt")): + if total_bytes >= max_bytes: + parts.append(f"*[...truncated, budget exceeded...]*") + break + + try: + content = file_path.read_text(errors="replace") + remaining = max_bytes - total_bytes + if len(content) > remaining: + content = content[:remaining] + "\n[...truncated...]\n" + + # Infer original filename from safe name + original_name = file_path.stem.replace("_", "/") + + parts.append(f"### {subdir.name}/{original_name}") + parts.append("```") + parts.append(content) + parts.append("```") + parts.append("") + + total_bytes += len(content) + except Exception: + continue + + if total_bytes >= max_bytes: + break + + if total_bytes == 0: + return "" + + return "\n".join(parts) + + +def build_triage_payload( + bundle_dir: Path | None, + kedb_dir: Path | None = None, + job: dict | None = None +) -> str: + """Build the triage prompt from bundle contents.""" + parts = [] + + # Include snippet feedback and content if this is a follow-up round + snippet_round = int(job.get("snippet_round", "0")) if job else 0 + has_snippets = job.get("has_snippets", "false") == "true" if job else False + + if bundle_dir is not None and has_snippets and snippet_round > 0: + # Add feedback about previous extraction + feedback = build_snippet_feedback(bundle_dir, snippet_round) + if feedback: + parts.append(feedback) + parts.append("") + + # Add extracted snippet contents + snippet_content = load_snippets_content(bundle_dir, snippet_round) + if snippet_content: + parts.append(snippet_content) + parts.append("") + + # Known Error Database (if available) + kedb_content = load_kedb(kedb_dir) + if kedb_content: + parts.append(kedb_content) + parts.append("") + + # User-provided context (run-scoped) + run_id = job.get("run_id") if job else None + origin = job.get("origin") if job else None + user_context, _ = get_user_context(run_id, origin) + if user_context: + parts.append("## User Context (run-scoped)") + parts.append(user_context) + parts.append("") + + bundle_id = job.get("bundle_id") if job else None + + # Metadata + meta = read_bundle_text(bundle_dir, bundle_id, "meta.txt") + if meta: + parts.append("## Metadata") + parts.append(meta) + parts.append("") + + + # Build errors + errors = read_bundle_text(bundle_dir, bundle_id, "logs/errors.txt") + if errors: + parts.append("## Build Errors") + parts.append(errors) + parts.append("") + + # Port files + parts.append("## Port Files") + + makefile = read_bundle_text(bundle_dir, bundle_id, "port/Makefile") + if makefile: + parts.append("### Makefile") + parts.append("```makefile") + parts.append(makefile) + parts.append("```") + parts.append("") + + plist = read_bundle_text(bundle_dir, bundle_id, "port/pkg-plist") + if plist: + parts.append("### pkg-plist") + parts.append("```") + parts.append(plist) + parts.append("```") + parts.append("") + + distinfo = read_bundle_text(bundle_dir, bundle_id, "port/distinfo") + if distinfo: + parts.append("### distinfo") + parts.append("```") + parts.append(distinfo) + parts.append("```") + parts.append("") + + # Existing patches (if stored) + if bundle_id: + patch_relpaths = [p for p in bundle_artifact_list(bundle_id) if p.startswith("port/files/patch-")] + else: + patch_relpaths = [] + if patch_relpaths: + parts.append("### Existing Patches") + for rel in sorted(patch_relpaths): + content = read_bundle_text(bundle_dir, bundle_id, rel) + if content: + parts.append(f"#### {Path(rel).name}") + parts.append("```diff") + parts.append(content) + parts.append("```") + parts.append("") + + parts.append("---") + parts.append("Analyze this build failure and provide your triage report.") + + return "\n".join(parts) + + +def build_patch_payload( + bundle_dir: Path | None, + kedb_dir: Path | None = None, + job: dict | None = None +) -> str: + """Build the patch generation prompt including triage output.""" + parts = [] + + # Include snippet feedback and content if this is a follow-up round + snippet_round = int(job.get("snippet_round", "0")) if job else 0 + has_snippets = job.get("has_snippets", "false") == "true" if job else False + + if bundle_dir is not None and has_snippets and snippet_round > 0: + # Add feedback about previous extraction + feedback = build_snippet_feedback(bundle_dir, snippet_round) + if feedback: + parts.append(feedback) + parts.append("") + + # Add extracted snippet contents + snippet_content = load_snippets_content(bundle_dir, snippet_round) + if snippet_content: + parts.append(snippet_content) + parts.append("") + + bundle_id = job.get("bundle_id") if job else None + + # Triage summary (most important context) + triage = read_bundle_text(bundle_dir, bundle_id, "analysis/triage.md") + if triage: + parts.append("## Triage Summary") + parts.append(triage) + parts.append("") + + # Prior attempts (last 3 bundles for this origin) + origin = job.get("origin") if job else None + if origin: + history = [] + for entry in port_bundle_history(origin): + bundle = entry.get("bundle_id") + if not bundle or bundle == bundle_id: + continue + history.append(bundle) + if len(history) >= 3: + break + if history: + parts.append("## Prior Attempts (most recent 3)") + for past_bundle in history: + parts.append(f"### Bundle {past_bundle}") + for relpath, title, code_block in [ + ("analysis/patch_plan.json", "Patch Plan", "json"), + ("analysis/patch.log", "Patch Log", None), + ("analysis/rebuild_status.txt", "Rebuild Status", None), + ]: + content = read_bundle_text(None, past_bundle, relpath) + if not content: + continue + parts.append(f"#### {title}") + if code_block: + parts.append(f"```{code_block}") + parts.append(content) + parts.append("```") + else: + parts.append(content) + parts.append("") + + # User-provided context (run-scoped) + run_id = job.get("run_id") if job else None + user_context, _ = get_user_context(run_id, origin) + if user_context: + parts.append("## User Context (run-scoped)") + parts.append(user_context) + parts.append("") + + # Known Error Database (if available) + kedb_content = load_kedb(kedb_dir) + if kedb_content: + parts.append(kedb_content) + parts.append("") + + # Metadata + meta = read_bundle_text(bundle_dir, bundle_id, "meta.txt") + if meta: + parts.append("## Metadata") + parts.append(meta) + parts.append("") + + # Build errors + errors = read_bundle_text(bundle_dir, bundle_id, "logs/errors.txt") + if errors: + parts.append("## Build Errors") + parts.append(errors) + parts.append("") + + # Port files + parts.append("## Port Files") + + makefile = read_bundle_text(bundle_dir, bundle_id, "port/Makefile") + if makefile: + parts.append("### Makefile") + parts.append("```makefile") + parts.append(makefile) + parts.append("```") + parts.append("") + + plist = read_bundle_text(bundle_dir, bundle_id, "port/pkg-plist") + if plist: + parts.append("### pkg-plist") + parts.append("```") + parts.append(plist) + parts.append("```") + parts.append("") + + distinfo = read_bundle_text(bundle_dir, bundle_id, "port/distinfo") + if distinfo: + parts.append("### distinfo") + parts.append("```") + parts.append(distinfo) + parts.append("```") + parts.append("") + + # Existing patches (if stored) + if bundle_id: + patch_relpaths = [p for p in bundle_artifact_list(bundle_id) if p.startswith("port/files/patch-")] + else: + patch_relpaths = [] + if patch_relpaths: + parts.append("### Existing Patches") + for rel in sorted(patch_relpaths): + content = read_bundle_text(bundle_dir, bundle_id, rel) + if content: + parts.append(f"#### {Path(rel).name}") + parts.append("```diff") + parts.append(content) + parts.append("```") + parts.append("") + + parts.append("---") + parts.append("Use the dports tools to apply fixes in the shared workspace and rebuild the target origin.") + parts.append("Return a report with these exact sections:") + parts.append("- ## Patch Log") + parts.append("- ## Rebuild Status") + parts.append("- ## Patch Plan (JSON) with a ```json block") + parts.append("- ## Rebuild Proof (JSON) with a ```json block") + + return "\n".join(parts) + + + +# ----------------------------------------------------------------------------- + +def move_job(job_path: Path, dest_dir: str) -> Path: + """Move job file to destination directory (done/failed). + Also moves any associated .job.error file. + """ + dest = job_path.parent.parent / dest_dir / job_path.name + job_path.rename(dest) + + # Also move error file if it exists + error_file = job_path.with_suffix(".job.error") + if error_file.exists(): + error_dest = dest.with_suffix(".job.error") + try: + error_file.rename(error_dest) + except OSError: + pass # Best effort + + return dest + + +def write_error_note(job_path: Path, error: str): + """Write error note next to failed job.""" + error_path = job_path.with_suffix(".job.error") + with open(error_path, "w") as f: + f.write(f"timestamp={datetime.now(timezone.utc).isoformat()}\n") + f.write(f"error={error}\n") + + +def claim_next_job(queue_root: Path) -> Path | None: + """Claim the oldest pending job by moving to inflight/.""" + pending_dir = queue_root / "pending" + inflight_dir = queue_root / "inflight" + + jobs = sorted(pending_dir.glob("*.job")) + + for job_path in jobs: + try: + dest = inflight_dir / job_path.name + job_path.rename(dest) + return dest + except OSError: + continue + + return None + + +def enqueue_patch_job( + queue_root: Path, + job: dict, + *, + tier_name: str | None = None, + dev_env: str | None = None, +): + """Enqueue a patch job based on completed triage job. + + ``tier_name`` is the trust tier resolved at triage time (AUTO/ASSIST); + propagating it lets the patch worker use the right budget without + re-parsing triage.md. ``dev_env`` is the dev-env name the patch + flow should operate against; omit to let the patch worker fall + back to DP_HARNESS_ENV. + """ + ts = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%SZ") + origin_safe = job.get("origin", "unknown").replace("/", "_") + pid = os.getpid() + + job_name = f"{ts}-{job.get('profile', 'unknown')}-{origin_safe}-{pid}-patch.job" + + pending_dir = queue_root / "pending" + job_path = pending_dir / job_name + + # Inherit iteration from parent job, or start at 1 + iteration = int(job.get("iteration", "1")) + max_iterations = int(job.get("max_iterations", str(DEFAULT_MAX_ITERATIONS))) + + content = [ + f"type=patch", + f"created_ts_utc={ts}", + f"profile={job.get('profile', '')}", + f"origin={job.get('origin', '')}", + f"flavor={job.get('flavor', '')}", + f"bundle_id={job.get('bundle_id', '')}", + f"run_id={job.get('run_id', '')}", + f"triage_relpath=analysis/triage.md", + f"iteration={iteration}", + f"max_iterations={max_iterations}", + ] + if tier_name: + content.append(f"tier={tier_name}") + if dev_env: + content.append(f"dev_env={dev_env}") + + # Include previous_bundle if this is a retry + previous_bundle = job.get("previous_bundle") + if previous_bundle: + content.append(f"previous_bundle={previous_bundle}") + + # Atomic write + tmp_path = job_path.with_suffix(".tmp") + with open(tmp_path, "w") as f: + f.write("\n".join(content) + "\n") + tmp_path.rename(job_path) + + return job_path + + + +# ----------------------------------------------------------------------------- +# Job processing +# ----------------------------------------------------------------------------- + +def _write_triage_audit_harness( + bundle_dir: Path | None, + bundle_id: str | None, + result, # dportsv3.agent.triage.TriageResult + model: str, +) -> None: + """Write the harness-side audit JSON to the bundle. + + The markdown response is already on disk (triage.run writes + analysis/triage.md after each LLM round). This adds an + analysis/triage.json with classification, confidence, usage, and + provenance. + """ + audit = { + "classification": result.classification, + "confidence": result.confidence, + "snippet_rounds": result.snippet_rounds, + "tokens_used": { + "prompt": result.usage.prompt_tokens, + "completion": result.usage.completion_tokens, + "total": result.usage.total_tokens, + }, + "model": model, + "via": "dportsv3.agent.triage", + } + data = (json.dumps(audit, indent=2) + "\n").encode("utf-8") + if bundle_id: + if not artifact_store_put(bundle_id, "analysis/triage.json", data, "json"): + raise RuntimeError("failed to write triage.json to artifact store") + return + if bundle_dir is None: + raise RuntimeError("bundle_dir or bundle_id required") + out = bundle_dir / "analysis" / "triage.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(data) + + +def _process_triage_job_harness( + queue_root: Path, + job_path: Path, + job: dict, + bundle_dir: Path | None, + kedb_dir: Path | None, + payload: str, + origin: str, + job_id: str, + model: str, +) -> tuple[bool, str]: + """Triage path that uses dportsv3.agent.triage instead of opencode.""" + from dportsv3.agent import triage as harness_triage # type: ignore[import-not-found] + + api_base = os.environ.get("DP_HARNESS_TRIAGE_API_BASE") or None + api_key = os.environ.get("DP_HARNESS_TRIAGE_API_KEY") or None + custom_llm_provider = os.environ.get("DP_HARNESS_TRIAGE_PROVIDER") or None + timeout = int(os.environ.get("DP_HARNESS_TIMEOUT", "120")) + max_snippet_rounds = int(os.environ.get("DP_HARNESS_MAX_SNIPPET_ROUNDS", "5")) + + if bundle_dir is None: + return False, "harness triage requires a local bundle_dir" + + activity_log( + queue_root, "api_call_start", + f"Calling harness triage for {origin}", + job_id=job_id, extra={"agent": "dports-triage", "model": model}, + ) + start = time.time() + try: + result = harness_triage.run( + payload, + bundle_dir=bundle_dir, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + max_snippet_rounds=max_snippet_rounds, + ) + except Exception as exc: + activity_log( + queue_root, "api_error", + f"Harness triage failed for {origin}: {str(exc)[:200]}", + job_id=job_id, + ) + write_error_note(job_path, str(exc)) + return False, str(exc) + duration_ms = int((time.time() - start) * 1000) + activity_log( + queue_root, "api_call_complete", + f"Harness triage response received for {origin} (rounds={result.snippet_rounds}, tokens={result.usage.total_tokens})", + job_id=job_id, duration_ms=duration_ms, + ) + + bundle_id = job.get("bundle_id") + _write_triage_audit_harness(bundle_dir, bundle_id, result, model) + activity_log( + queue_root, "write_output", + f"Wrote harness triage outputs for {origin}", + job_id=job_id, + ) + + triage = { + "classification": result.classification, + "confidence": result.confidence, + "raw": result.text, + } + + # Resolve trust tier from triage outcome. The policy's + # confidence_floor downgrades automatically (AUTO with low + # confidence -> ASSIST -> MANUAL), so low confidence is captured + # naturally as MANUAL without a separate needs_user_context check. + from dportsv3.agent import policy as harness_policy # type: ignore[import-not-found] + policy_path = os.environ.get( + "DP_HARNESS_POLICY", + str(Path(__file__).resolve().parent.parent / "config" / "agentic-policy.json"), + ) + try: + pol = harness_policy.load_policy(policy_path) + except Exception as exc: + activity_log( + queue_root, "policy_error", + f"Failed to load harness policy at {policy_path}: {exc}", + job_id=job_id, + ) + return False, f"policy load failed: {exc}" + tier = harness_policy.tier_for(pol, result.classification, result.confidence) + + if tier.name == "MANUAL": + # Surface low-confidence + non-patchable cases to the operator + # via the user_context request channel (UI picks them up). + run_id = job.get("run_id", "") + iteration = int(job.get("iteration", "1")) + max_iterations = int(job.get("max_iterations", str(DEFAULT_MAX_ITERATIONS))) + upsert_user_context_request( + queue_root, + run_id=run_id, + origin=origin, + bundle_id=bundle_id or (bundle_dir.name if bundle_dir else ""), + classification=result.classification, + confidence=result.confidence, + iteration=iteration, + max_iterations=max_iterations, + ) + activity_log( + queue_root, "triage_manual", + f"Triage tier MANUAL for {origin} (classification={result.classification}, confidence={result.confidence}); no auto-enqueue", + job_id=job_id, + extra={ + "classification": result.classification, + "confidence": result.confidence, + "tier": tier.name, + "run_id": run_id, + }, + ) + update_runner_status("waiting", job_id=job_id, stage="waiting_user_context", + extra={"origin": origin, "type": "triage", "tier": tier.name}) + return True, "manual_tier" + + # AUTO or ASSIST: auto-enqueue a patch job, carrying the resolved + # tier + dev_env so the patch worker doesn't re-resolve. + enqueue_patch_job( + queue_root, job, + tier_name=tier.name, + dev_env=os.environ.get("DP_HARNESS_ENV") or None, + ) + activity_log( + queue_root, "enqueue_patch", + f"Auto-enqueued patch job for {origin} (tier={tier.name}, classification={result.classification})", + job_id=job_id, + extra={ + "classification": result.classification, + "confidence": result.confidence, + "tier": tier.name, + "max_iterations": tier.max_iterations, + "max_tokens": tier.max_tokens, + }, + ) + return True, "done" + + +def process_triage_job( + queue_root: Path, + job_path: Path, + job: dict, + bundle_dir: Path | None, + kedb_dir: Path | None, +) -> tuple[bool, str]: + """Process a triage job via the dportsv3.agent harness.""" + harness_model = os.environ.get("DP_HARNESS_TRIAGE_MODEL") + if not harness_model: + msg = "DP_HARNESS_TRIAGE_MODEL not set; cannot run triage" + write_error_note(job_path, msg) + return False, msg + payload = build_triage_payload(bundle_dir, kedb_dir, job) + origin = job.get("origin", "unknown") + job_id = job_path.name + return _process_triage_job_harness( + queue_root, job_path, job, bundle_dir, kedb_dir, + payload, origin, job_id, harness_model, + ) +def _write_patch_audit_harness( + bundle_dir: Path | None, + bundle_id: str | None, + result, # dportsv3.agent.attempt_loop.PatchResult + model: str, +) -> None: + """Write harness-side outputs to the bundle: patch.md, rebuild_proof.json, + changes.diff (host-side git diff in the env), and patch_audit.json.""" + text = (result.final_text or "").rstrip() + "\n" + md_bytes = text.encode("utf-8") + if bundle_id: + artifact_store_put(bundle_id, "analysis/patch.md", md_bytes, "text") + else: + analysis = bundle_dir / "analysis" + analysis.mkdir(parents=True, exist_ok=True) + (analysis / "patch.md").write_bytes(md_bytes) + + if result.proof is not None: + proof_bytes = (json.dumps(result.proof, indent=2) + "\n").encode("utf-8") + if bundle_id: + artifact_store_put(bundle_id, "analysis/rebuild_proof.json", proof_bytes, "json") + else: + (bundle_dir / "analysis" / "rebuild_proof.json").write_bytes(proof_bytes) + + audit = { + "status": result.status, + "model": model, + "tokens_used": { + "prompt": result.usage.prompt_tokens, + "completion": result.usage.completion_tokens, + "total": result.usage.total_tokens, + }, + "attempts": [ + {"attempt": a.attempt, "tokens": a.tokens, "rebuild_ok": a.rebuild_ok} + for a in result.attempts + ], + "via": "dportsv3.agent.patch", + } + audit_bytes = (json.dumps(audit, indent=2) + "\n").encode("utf-8") + if bundle_id: + artifact_store_put(bundle_id, "analysis/patch_audit.json", audit_bytes, "json") + else: + (bundle_dir / "analysis" / "patch_audit.json").write_bytes(audit_bytes) + + +def _write_changes_diff(bundle_dir: Path | None, bundle_id: str | None, env: str, origin: str) -> None: + """Capture host-side `git diff` against the env's DeltaPorts overlay HEAD + and write to analysis/changes.diff. Best-effort: failures are logged but + not fatal — the agent's reasoning is in patch.md regardless.""" + try: + from dportsv3.agent import worker # type: ignore[import-not-found] + paths = worker.env_paths(env) + delta = paths.deltaports + import subprocess + rel = f"ports/{origin}" + p = subprocess.run( + ["git", "-C", str(delta), "diff", "--", rel], + capture_output=True, text=True, check=False, + ) + diff_bytes = p.stdout.encode("utf-8") + except Exception as exc: + diff_bytes = f"# failed to capture diff: {exc}\n".encode("utf-8") + + if bundle_id: + artifact_store_put(bundle_id, "analysis/changes.diff", diff_bytes, "text") + elif bundle_dir: + out = bundle_dir / "analysis" / "changes.diff" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_bytes(diff_bytes) + + +def _process_patch_job_harness( + queue_root: Path, + job_path: Path, + job: dict, + bundle_dir: Path | None, + kedb_dir: Path | None, + payload: str, + origin: str, + job_id: str, + model: str, +) -> tuple[bool, str]: + """Patch path that uses dportsv3.agent.patch instead of opencode.""" + from dportsv3.agent import patch as harness_patch # type: ignore[import-not-found] + from dportsv3.agent import policy as harness_policy # type: ignore[import-not-found] + + api_base = os.environ.get("DP_HARNESS_PATCH_API_BASE") or None + api_key = os.environ.get("DP_HARNESS_PATCH_API_KEY") or None + custom_llm_provider = os.environ.get("DP_HARNESS_PATCH_PROVIDER") or None + timeout = int(os.environ.get("DP_HARNESS_PATCH_TIMEOUT", "600")) + + # Resolve the dev-env name. Prefer the explicit job field; fall back to + # DP_HARNESS_ENV (an operator-set default for the runner). + env = job.get("dev_env") or os.environ.get("DP_HARNESS_ENV") or "" + if not env: + msg = "patch job missing dev_env (set job 'dev_env' field or DP_HARNESS_ENV)" + write_error_note(job_path, msg) + return False, msg + + # Resolve tier. Prefer the tier the triage step already resolved + # (carried via the job's `tier=` field); fall back to re-resolving + # from triage.md if missing (legacy / hand-fired patch jobs). + bundle_id = job.get("bundle_id") + policy_path = os.environ.get( + "DP_HARNESS_POLICY", + str(Path(__file__).resolve().parent.parent / "config" / "agentic-policy.json"), + ) + try: + pol = harness_policy.load_policy(policy_path) + except Exception as exc: + msg = f"failed to load harness policy at {policy_path}: {exc}" + write_error_note(job_path, msg) + return False, msg + + tier_name = (job.get("tier") or "").strip() + if tier_name and tier_name in pol.tiers: + tier = pol.tiers[tier_name] + else: + triage_text = read_bundle_text(bundle_dir, bundle_id, "analysis/triage.md") or "" + triage = parse_triage_output(triage_text) + tier = harness_policy.tier_for(pol, triage.get("classification", ""), triage.get("confidence", "")) + + activity_log( + queue_root, "api_call_start", + f"Calling harness patch for {origin} (tier={tier.name}, env={env})", + job_id=job_id, extra={"agent": "dports-patch", "model": model, "tier": tier.name}, + ) + + start = time.time() + try: + result = harness_patch.run( + payload, + tier=tier, + env=env, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + ) + except Exception as exc: + activity_log( + queue_root, "api_error", + f"Harness patch failed for {origin}: {str(exc)[:200]}", + job_id=job_id, + ) + write_error_note(job_path, str(exc)) + return False, str(exc) + duration_ms = int((time.time() - start) * 1000) + + activity_log( + queue_root, "api_call_complete", + f"Harness patch finished for {origin} (status={result.status}, " + f"attempts={len(result.attempts)}, tokens={result.usage.total_tokens})", + job_id=job_id, duration_ms=duration_ms, + ) + + _write_patch_audit_harness(bundle_dir, bundle_id, result, model) + _write_changes_diff(bundle_dir, bundle_id, env, origin) + + activity_log( + queue_root, "write_output", + f"Wrote harness patch outputs for {origin}", + job_id=job_id, + ) + + if result.status == "success": + return True, "done" + return True, result.status # needs-help / budget-exhausted — job recorded, not retried + + +def process_patch_job( + queue_root: Path, + job_path: Path, + job: dict, + bundle_dir: Path | None, + kedb_dir: Path | None, +) -> tuple[bool, str]: + """Process a patch job via the dportsv3.agent harness.""" + harness_model = os.environ.get("DP_HARNESS_PATCH_MODEL") + if not harness_model: + msg = "DP_HARNESS_PATCH_MODEL not set; cannot run patch" + write_error_note(job_path, msg) + return False, msg + payload = build_patch_payload(bundle_dir, kedb_dir, job) + origin = job.get("origin", "unknown") + job_id = job_path.name + return _process_patch_job_harness( + queue_root, job_path, job, bundle_dir, kedb_dir, + payload, origin, job_id, harness_model, + ) +def process_job( + queue_root: Path, + job_path: Path, + dry_run: bool, + kedb_dir: Path | None, +): + """Process a single job (dispatch based on type).""" + job = parse_job_file(job_path) + job_type = job.get("type", "triage") + bundle_dir_value = job.get("bundle_dir") + bundle_dir = Path(bundle_dir_value) if bundle_dir_value else None + bundle_id = job.get("bundle_id") + origin = job.get("origin", "unknown") + job_id = job_path.name + + log(queue_root, "INFO", f"processing job {job_path.name}") + + update_runner_status("processing", job_id=job_id, stage=f"{job_type}_start", + extra={"origin": origin, "type": job_type}) + + if bundle_dir is None and not bundle_id: + log(queue_root, "ERROR", "missing bundle_id/bundle_dir in job") + write_error_note(job_path, "missing bundle_id/bundle_dir in job") + move_job(job_path, "failed") + update_runner_status("idle", job_id=None, stage=None) + return + if bundle_dir is not None and not bundle_dir.exists(): + log(queue_root, "ERROR", f"bundle_dir does not exist: {bundle_dir}") + write_error_note(job_path, f"bundle_dir does not exist: {bundle_dir}") + move_job(job_path, "failed") + update_runner_status("idle", job_id=None, stage=None) + return + + if dry_run: + if job_type == "patch": + payload = build_patch_payload(bundle_dir, kedb_dir, job) + else: + payload = build_triage_payload(bundle_dir, kedb_dir, job) + + log(queue_root, "INFO", f"[dry-run] type={job_type}, would send payload ({len(payload)} bytes)") + print("=" * 60) + print(f"JOB TYPE: {job_type}") + print("=" * 60) + print(payload) + print("=" * 60) + move_job(job_path, "pending") + update_runner_status("idle", job_id=None, stage=None) + return + + if job_type == "patch": + success, status = process_patch_job( + queue_root, job_path, job, bundle_dir, kedb_dir, + ) + elif job_type == "triage": + success, status = process_triage_job( + queue_root, job_path, job, bundle_dir, kedb_dir, + ) + else: + success, status = False, f"unknown job type: {job_type}" + + if success: + move_job(job_path, "done") + log(queue_root, "INFO", "moved job to done/") + else: + # Write error file for visibility in UI + error_file = job_path.with_suffix(".job.error") + try: + error_msg = status[:500] if status else "Unknown error" + error_file.write_text(error_msg) + log(queue_root, "DEBUG", f"wrote error file: {error_file}") + except OSError as e: + log(queue_root, "WARN", f"failed to write error file: {e}") + + move_job(job_path, "failed") + log(queue_root, "ERROR", f"moved job to failed/ ({status})") + + # Update runner status back to idle + update_runner_status("idle", job_id=None, stage=None) + + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description="Process dsynth failure jobs via the dportsv3.agent harness") + parser.add_argument("--queue-root", required=True, help="Path to queue directory") + parser.add_argument("--once", action="store_true", help="Process one job and exit") + parser.add_argument("--dry-run", action="store_true", help="Print payload without calling the LLM") + parser.add_argument("--kedb-dir", help="Path to KEDB directory (default: auto-detect)") + args = parser.parse_args() + + queue_root = Path(args.queue_root) + + for subdir in ["pending", "inflight", "done", "failed"]: + d = queue_root / subdir + if not d.exists(): + print(f"error: queue directory missing: {d}", file=sys.stderr) + sys.exit(1) + + init_state_db(queue_root) + start_heartbeat() + + if args.kedb_dir: + kedb_dir = Path(args.kedb_dir) + if not kedb_dir.exists(): + print(f"warning: KEDB directory not found: {kedb_dir}", file=sys.stderr) + kedb_dir = None + else: + kedb_dir = find_kedb_dir() + + triage_model = os.environ.get("DP_HARNESS_TRIAGE_MODEL") or "" + patch_model = os.environ.get("DP_HARNESS_PATCH_MODEL") or "" + kedb_info = str(kedb_dir) if kedb_dir else "none" + log(queue_root, "INFO", + f"starting runner (once={args.once}, dry_run={args.dry_run}, " + f"triage_model={triage_model}, patch_model={patch_model}, kedb={kedb_info})") + activity_log(queue_root, "runner_start", + f"Runner started (triage={triage_model}, patch={patch_model})") + update_runner_status("idle", job_id=None, stage=None) + + try: + if args.once: + job = claim_next_job(queue_root) + if job: + process_job(queue_root, job, args.dry_run, kedb_dir) + else: + log(queue_root, "INFO", "no jobs in queue") + else: + while True: + process_user_context_updates(queue_root) + job = claim_next_job(queue_root) + if job: + process_job(queue_root, job, args.dry_run, kedb_dir) + else: + update_runner_status("idle", job_id=None, stage="waiting") + time.sleep(5) + except KeyboardInterrupt: + log(queue_root, "INFO", "shutting down (keyboard interrupt)") + activity_log(queue_root, "runner_stop", "Runner stopped (keyboard interrupt)") + finally: + stop_heartbeat() + update_runner_status("stopped", job_id=None, stage=None) + + +if __name__ == "__main__": + main() diff --git a/scripts/artifact-store b/scripts/artifact-store new file mode 100755 index 00000000000..ba2f14ead6d --- /dev/null +++ b/scripts/artifact-store @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Standalone entry point for the artifact-store HTTP service. + +All logic lives in ``dportsv3.artifact_store``; this shim only: +- Bootstraps sys.path so the package is importable when run as a + plain script (no need to activate the generator venv). +- Hands control to ``main()``. + +Both invocations are functionally identical: + ./scripts/artifact-store --logs-root /path + python -m dportsv3.artifact_store --logs-root /path +""" +from __future__ import annotations + +import sys +from pathlib import Path + +# Same bootstrap pattern as scripts/agent-queue-runner. +_GENERATOR_DIR = Path(__file__).resolve().parent / "generator" +if _GENERATOR_DIR.is_dir() and str(_GENERATOR_DIR) not in sys.path: + sys.path.insert(0, str(_GENERATOR_DIR)) + +from dportsv3.artifact_store import main # noqa: E402 + +if __name__ == "__main__": + main() diff --git a/scripts/artifact-store-client b/scripts/artifact-store-client new file mode 100755 index 00000000000..ff83ad10673 --- /dev/null +++ b/scripts/artifact-store-client @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""artifact-store-client: tiny CLI for hooks to post artifacts.""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.request + +DEFAULT_URL = "http://127.0.0.1:8788" + + +def request_json(method: str, url: str, body: dict | None = None, headers: dict | None = None): + data = None + req_headers = {"Accept": "application/json"} + if headers: + req_headers.update(headers) + if body is not None: + payload = json.dumps(body).encode("utf-8") + req_headers["Content-Type"] = "application/json" + data = payload + req = urllib.request.Request(url, data=data, headers=req_headers, method=method) + with urllib.request.urlopen(req, timeout=10) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def request_bytes(url: str, data: bytes, headers: dict): + req = urllib.request.Request(url, data=data, headers=headers, method="POST") + with urllib.request.urlopen(req, timeout=20) as resp: + return json.loads(resp.read().decode("utf-8")) + + +def cmd_health(args): + resp = request_json("GET", f"{args.url}/health") + print(json.dumps(resp, indent=2)) + + +def cmd_bundle_upsert(args): + payload = { + "run_id": args.run_id, + "profile": args.profile, + "ts_utc": args.ts_utc, + "bundle_id": args.bundle_id, + "origin": args.origin, + "flavor": args.flavor, + "result": args.result, + "target": args.target, + } + resp = request_json("POST", f"{args.url}/v1/bundles/upsert", payload) + print(json.dumps(resp, indent=2)) + + +def cmd_put_blob(args): + if args.stdin: + data = sys.stdin.buffer.read() + else: + with open(args.file, "rb") as f: + data = f.read() + headers = { + "Accept": "application/json", + "Content-Type": "application/octet-stream", + "X-Bundle-Id": args.bundle_id, + "X-Relpath": args.relpath, + } + if args.kind: + headers["X-Kind"] = args.kind + resp = request_bytes(f"{args.url}/v1/artifacts/put", data, headers) + print(json.dumps(resp, indent=2)) + + +def cmd_put_fs(args): + payload = { + "bundle_id": args.bundle_id, + "relpath": args.relpath, + "fs_path": args.fs_path, + "kind": args.kind, + } + resp = request_json("POST", f"{args.url}/v1/artifacts/put-fs", payload) + print(json.dumps(resp, indent=2)) + + +def main(): + parser = argparse.ArgumentParser(description="artifact-store client") + parser.add_argument("--url", default=DEFAULT_URL) + sub = parser.add_subparsers(dest="cmd", required=True) + + p_health = sub.add_parser("health") + p_health.set_defaults(func=cmd_health) + + p_bundle = sub.add_parser("bundle-upsert") + p_bundle.add_argument("--run-id", required=True) + p_bundle.add_argument("--profile", default="") + p_bundle.add_argument("--ts-utc", required=True) + p_bundle.add_argument("--bundle-id", required=True) + p_bundle.add_argument("--origin", required=True) + p_bundle.add_argument("--flavor", default="") + p_bundle.add_argument("--result", default="") + p_bundle.add_argument("--target", default="") + p_bundle.set_defaults(func=cmd_bundle_upsert) + + p_blob = sub.add_parser("put-blob") + p_blob.add_argument("--bundle-id", required=True) + p_blob.add_argument("--relpath", required=True) + p_blob.add_argument("--file") + p_blob.add_argument("--stdin", action="store_true") + p_blob.add_argument("--kind", default="") + p_blob.set_defaults(func=cmd_put_blob) + + p_fs = sub.add_parser("put-fs") + p_fs.add_argument("--bundle-id", required=True) + p_fs.add_argument("--relpath", required=True) + p_fs.add_argument("--fs-path", required=True) + p_fs.add_argument("--kind", default="") + p_fs.set_defaults(func=cmd_put_fs) + + args = parser.parse_args() + try: + args.func(args) + except Exception as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/dsynth-hooks/README.md b/scripts/dsynth-hooks/README.md new file mode 100644 index 00000000000..97250d45d64 --- /dev/null +++ b/scripts/dsynth-hooks/README.md @@ -0,0 +1,83 @@ +# dsynth hooks for DeltaPorts + +A single hook set that does **both** things you want for an agentic +DeltaPorts build: + +1. **Failure evidence**: every `hook_pkg_failure` writes a full bundle + (errors, log tail, port snapshot, gzipped full log) to + `artifact-store` and enqueues a triage job for the agent harness. +2. **Build tracking**: every per-port event (`pkg_start`, `pkg_success`, + `pkg_failure`, `pkg_skipped`, `pkg_ignored`) plus `run_start` / + `run_end` reports to `dportsv3 tracker` so the cross-build dashboard + reflects the build's progress in real time. + +Tracker integration is opt-in via the config file +(`dportsv3-hooks.conf`). Without it the hooks still do the +artifact-store work; tracker calls are silent no-ops. + +## Files + +| File | Purpose | +|---|---| +| `hook_common.sh` | shared helpers (env defaults, artifact-store client wrappers, tracker integration, log distilling) | +| `dportsv3-hooks.conf.example` | config template — copy to `/etc/dsynth/dportsv3-hooks.conf` and edit | +| `hook_run_start` | initializes evidence root + starts a tracker build_run | +| `hook_run_end` | clears evidence pointer + finishes the tracker build_run | +| `hook_pkg_start` / `hook_pkg_started` | tracker mark-building (both names provided for dsynth-variant compat) | +| `hook_pkg_failure` | writes a full evidence bundle, enqueues a triage job, records `fail` in tracker | +| `hook_pkg_success` | records `pass` in tracker | +| `hook_pkg_skipped` | records `skipped` in tracker | +| `hook_pkg_ignored` | records `ignored` in tracker | + +## Install + +```sh +install -d /etc/dsynth +install -m 755 scripts/dsynth-hooks/hook_* /etc/dsynth/ +install -m 755 scripts/dsynth-hooks/hook_common.sh /etc/dsynth/ +install -m 644 scripts/dsynth-hooks/dportsv3-hooks.conf.example \ + /etc/dsynth/dportsv3-hooks.conf +``` + +Then edit `/etc/dsynth/dportsv3-hooks.conf` for at least: + +- `DPORTSV3_TRACKER_URL` (or leave commented to disable tracker integration) +- `DPORTSV3_BIN` (absolute path to your `dportsv3` wrapper) + +Defaults you usually don't need to override: + +- `DPORTSV3_TRACKER_TARGET` derives from `${PROFILE}` (one profile per target) +- `DPORTSV3_TRACKER_BUILD_TYPE` defaults to `test` — set to `release` for builds you intend to publish +- `ARTIFACT_STORE_URL` defaults to `http://127.0.0.1:8788` + +Make sure dsynth's `Hooks_Directory` points to `/etc/dsynth` (or +wherever you installed). dsynth picks up hooks by name; only one +executable per event name can exist there at a time. + +## Operational notes + +- Tracker outages don't fail dsynth. The hook logs the error to + `DPORTSV3_TRACKER_HOOK_LOG` (default: `${DIR_LOGS}/dportsv3-hooks.log`) + and exits 0. +- If `start-build` fails (typically because a prior build_run for the + same `(target, build_type)` is still active — usually because dsynth + was killed mid-run last time), tracker integration is disabled for + *this* dsynth run rather than reusing a stale run id. The state file + records `TRACKING_DISABLED=1`. Resolve by either: + 1. Manually finishing the stale run: `dportsv3 tracker finish-build --run N` + 2. Deleting the state file: `rm /path/to/state-dir/${PROFILE}.env` +- Per-profile active run state lives in + `${DPORTSV3_TRACKER_STATE_DIR}/${PROFILE}.env`. Default location is + under the evidence tree so it travels with the rest of the run data. +- Artifact-store, by contrast, is treated as required for + `hook_pkg_failure` — if its health check fails, the hook exits + non-zero (dsynth logs the failure, build continues but the failure + bundle is lost). + +## What landed where + +This is the single canonical hook set. The earlier `scripts/builderhooks/` +directory (tracker-only hooks) has been folded in and removed. The +patterns from there — config-driven, per-profile state files, +soft-fail logging, disable-on-collision — are all preserved in +`hook_common.sh`. diff --git a/scripts/dsynth-hooks/dportsv3-hooks.conf.example b/scripts/dsynth-hooks/dportsv3-hooks.conf.example new file mode 100644 index 00000000000..c4ad9c12f9f --- /dev/null +++ b/scripts/dsynth-hooks/dportsv3-hooks.conf.example @@ -0,0 +1,54 @@ +# dsynth hook configuration for DeltaPorts. +# +# Install pattern: +# install -d /etc/dsynth +# install -m 755 scripts/dsynth-hooks/hook_* /etc/dsynth/ +# install -m 755 scripts/dsynth-hooks/hook_common.sh /etc/dsynth/ +# install -m 644 scripts/dsynth-hooks/dportsv3-hooks.conf.example \ +# /etc/dsynth/dportsv3-hooks.conf +# +# Then edit /etc/dsynth/dportsv3-hooks.conf for your environment. +# +# Anything sourced from this file becomes shell variables visible to all +# hooks. Tracker integration is OPT-IN: if DPORTSV3_TRACKER_URL is +# unset (or this file is absent entirely), the tracker_* helpers in +# hook_common.sh short-circuit and hooks only do artifact-store work. + +# ----------------------------------------------------------------------------- +# Artifact-store side +# ----------------------------------------------------------------------------- + +# Override only if running artifact-store on a non-default URL/port. +# ARTIFACT_STORE_URL=http://127.0.0.1:8788 + +# Override only if the artifact-store client lives elsewhere. +# ARTIFACT_STORE_CLIENT=/build/synth/DeltaPorts/scripts/artifact-store-client + +# ----------------------------------------------------------------------------- +# Tracker side (optional) +# ----------------------------------------------------------------------------- + +# Set DPORTSV3_TRACKER_URL to enable tracker integration. Unset/empty +# disables tracker calls entirely (artifact-store work continues). +DPORTSV3_TRACKER_URL=http://127.0.0.1:8080 + +# Absolute path to the dportsv3 wrapper (used to call tracker CLI commands). +DPORTSV3_BIN=/build/synth/DeltaPorts/dportsv3 + +# Build target. Defaults to @${PROFILE} if unset (one profile per target). +# DPORTSV3_TRACKER_TARGET=@2026Q2 + +# Build type for this dsynth profile (test | release). Defaults to "test". +# DPORTSV3_TRACKER_BUILD_TYPE=test + +# Directory used to store per-profile active run state. +# Defaults to ${DIR_LOGS}/evidence/.tracker-state (colocated with the +# artifact-store evidence tree). +# DPORTSV3_TRACKER_STATE_DIR=/var/db/dsynth-tracker + +# Hook-local log file. Tracker failures log here and do not fail dsynth. +# Defaults to ${DIR_LOGS}/dportsv3-hooks.log. +# DPORTSV3_TRACKER_HOOK_LOG=/var/log/dsynth-hooks.log + +# Optional external log URL prefix. If set, record-result appends /ORIGIN. +# DPORTSV3_TRACKER_LOG_URL_BASE=https://logs.example/dsynth diff --git a/scripts/dsynth-hooks/hook_common.sh b/scripts/dsynth-hooks/hook_common.sh new file mode 100755 index 00000000000..0401744f4f1 --- /dev/null +++ b/scripts/dsynth-hooks/hook_common.sh @@ -0,0 +1,595 @@ +#!/bin/sh +# +# Common helpers for dsynth hook scripts. +# +# dsynth executes hooks with a minimal environment. Do not rely on PATH. +# + +set -eu + +PATH="/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +export PATH + +umask 022 + +: "${PROFILE:=unknown}" +: "${DIR_LOGS:=}" +: "${DIR_PORTS:=}" +: "${DIR_BUILDBASE:=}" +: "${DIR_PACKAGES:=}" +: "${DIR_REPOSITORY:=}" +: "${DIR_OPTIONS:=}" +: "${DIR_DISTFILES:=}" + +: "${ARTIFACT_STORE_URL:=http://127.0.0.1:8788}" +: "${ARTIFACT_STORE_CLIENT:=/build/synth/DeltaPorts/scripts/artifact-store-client}" + +hook_config_dir() { + # Hooks live in ConfigBase (/etc/dsynth or /usr/local/etc/dsynth). + dir=$(dirname "$0") + # best effort to canonicalize + case "$dir" in + /*) printf '%s\n' "$dir" ;; + *) printf '%s\n' "$(pwd)/$dir" ;; + esac +} + +now_utc() { + # YYYYmmdd-HHMMSSZ + date -u "+%Y%m%d-%H%M%SZ" +} + +sanitize_component() { + # Replace characters that are annoying in filenames. + # Keep it deterministic and readable. + printf '%s' "$1" | tr '/:@' '___' | tr -cd 'A-Za-z0-9._-' +} + +origin_cat() { + printf '%s' "$1" | sed 's,/.*$,,' +} + +origin_port() { + printf '%s' "$1" | sed 's,^.*/,,' +} + +logfile_for_origin() { + # Reconstruct dsynth per-port log filename. + # dsynth uses: ${DIR_LOGS}/${category}___${portname}${WorkerFlavorPrt}.log + # where WorkerFlavorPrt is "" or "@flavor". + origin_raw=$1 + flavor_raw=${2:-} + + origin_base=${origin_raw%%@*} + cat=$(origin_cat "$origin_base") + port=$(origin_port "$origin_base") + base="${cat}___${port}" + + # dsynth sets FLAVOR=$ORIGIN when no flavor; only add @ when different. + if [ -n "$flavor_raw" ] && [ "$flavor_raw" != "$origin_raw" ] && [ "$flavor_raw" != "$origin_base" ]; then + base="${base}@${flavor_raw}" + fi + + printf '%s\n' "${DIR_LOGS}/${base}.log" +} + +current_run_id() { + # hook_run_start writes a run_id into evidence root. + root=$(evidence_root) + if [ -r "${root}/.current_run" ]; then + sed -n '1p' "${root}/.current_run" || true + else + printf '%s\n' "run-${PROFILE}-unknown" + fi +} + +evidence_root() { + if [ -z "$DIR_LOGS" ]; then + # Last resort + printf '%s\n' "/tmp/dsynth-evidence" + return + fi + printf '%s\n' "${DIR_LOGS}/evidence" +} + +queue_root() { + printf '%s\n' "$(evidence_root)/queue" +} + +ensure_queue_dirs() { + qroot=$(queue_root) + mkdir -p "${qroot}/pending" + mkdir -p "${qroot}/inflight" + mkdir -p "${qroot}/done" + mkdir -p "${qroot}/failed" +} + +artifact_store() { + "${ARTIFACT_STORE_CLIENT}" --url "${ARTIFACT_STORE_URL}" "$@" +} + +require_artifact_store() { + artifact_store health >/dev/null +} + +# Check if this is a rebuild attempt (branch starts with ai-fix/) +# Returns iteration number (0 if not a rebuild attempt) +detect_rebuild_iteration() { + # If tracking context exists, use it regardless of branch + evidence=$(evidence_root) + ctx_file="${evidence}/.current_apply_context" + if [ -r "$ctx_file" ]; then + iter=$(grep '^iteration=' "$ctx_file" 2>/dev/null | head -1 | cut -d= -f2) + if [ -n "$iter" ]; then + printf '%d\n' $((iter + 1)) + return + fi + fi + + deltaports_dir="${DIR_PORTS%/DPorts*}/DeltaPorts" + + # Check if DeltaPorts directory exists + if [ ! -d "$deltaports_dir" ]; then + printf '0\n' + return + fi + + # Get current branch + current_branch=$(cd "$deltaports_dir" && git branch --show-current 2>/dev/null || true) + + # Check if it's an AI fix branch + case "$current_branch" in + ai-fix/*) + # Default to iteration 2 if we're on ai-fix branch but no context + printf '2\n' + ;; + *) + printf '0\n' + ;; + esac +} + +# Get previous bundle path from tracking context (for rebuild attempts) +get_previous_bundle() { + evidence=$(evidence_root) + ctx_file="${evidence}/.current_apply_context" + + if [ -r "$ctx_file" ]; then + grep '^previous_bundle=' "$ctx_file" 2>/dev/null | head -1 | cut -d= -f2 + fi +} + +enqueue_job() { + # Args: bundle_id origin flavor profile run_id ts + bundle_id=$1 + origin=$2 + flavor=$3 + profile=$4 + run_id=$5 + ts=$6 + + qroot=$(queue_root) + origin_s=$(sanitize_component "$origin") + + # Check if this is a rebuild attempt + iteration=$(detect_rebuild_iteration) + previous_bundle=$(get_previous_bundle) + + # Build filename, omit flavor if redundant + fname="${ts}-${profile}-${origin_s}" + if [ -n "$flavor" ] && [ "$flavor" != "$origin" ] && [ "$flavor" != "${origin%%@*}" ]; then + flavor_s=$(sanitize_component "$flavor") + fname="${fname}-@${flavor_s}" + fi + + # Add iteration suffix if this is a retry + if [ "$iteration" -gt 1 ]; then + fname="${fname}-iter${iteration}" + fi + + fname="${fname}-$$.job" + + # Write to temp file first + tmpfile="${qroot}/pending/.tmp.$$.${ts}" + + # Base job fields. target comes from DPORTSV3_TRACKER_TARGET if + # set (loaded by tracker_load_config); empty otherwise — that + # leaves jobs.target NULL, which step 5 read endpoints surface + # as "unknown" under target filters. + write_kv_file "$tmpfile" \ + "created_ts_utc=${ts}" \ + "profile=${profile}" \ + "target=${DPORTSV3_TRACKER_TARGET:-}" \ + "origin=${origin}" \ + "flavor=${flavor}" \ + "bundle_id=${bundle_id}" \ + "run_id=${run_id}" \ + "type=triage" \ + "snippet_round=0" \ + "has_snippets=false" + + # Add iteration tracking if this is a retry + if [ "$iteration" -gt 1 ]; then + printf 'iteration=%d\n' "$iteration" >>"$tmpfile" + printf 'max_iterations=3\n' >>"$tmpfile" + if [ -n "$previous_bundle" ]; then + printf 'previous_bundle=%s\n' "$previous_bundle" >>"$tmpfile" + fi + fi + + # Atomic move to final location + mv "$tmpfile" "${qroot}/pending/${fname}" +} + +write_kv_file() { + out=$1 + shift + : >"$out" + for kv in "$@"; do + printf '%s\n' "$kv" >>"$out" + done +} + +copy_if_exists() { + src=$1 + dst=$2 + if [ -e "$src" ]; then + mkdir -p "$(dirname "$dst")" + cp -p "$src" "$dst" + fi +} + +copy_glob_if_exists() { + srcdir=$1 + pattern=$2 + dstdir=$3 + if [ -d "$srcdir" ]; then + mkdir -p "$dstdir" + # shellcheck disable=SC2035 + for f in "$srcdir"/$pattern; do + [ -e "$f" ] || continue + cp -p "$f" "$dstdir/" + done + fi +} + +truncate_bytes() { + infile=$1 + max_bytes=$2 + outfile=$3 + + # Preserve whole file if already under limit. + size=$(wc -c <"$infile" | tr -d ' ') + if [ "$size" -le "$max_bytes" ]; then + cp -p "$infile" "$outfile" + return + fi + + dd if="$infile" of="$outfile" bs=1 count="$max_bytes" 2>/dev/null + printf '\n[...truncated to %s bytes...]\n' "$max_bytes" >>"$outfile" +} + +# ----------------------------------------------------------------------------- +# dportsv3 tracker integration (was scripts/builderhooks/tracker_common.sh) +# ----------------------------------------------------------------------------- +# +# Tracker integration is optional but default-on. Operator installs +# `dportsv3-hooks.conf` next to this file (or sets DPORTSV3_HOOKS_CONFIG) +# with at least DPORTSV3_TRACKER_URL + DPORTSV3_BIN, then every hook +# records per-port outcomes via `dportsv3 tracker`. Hooks soft-fail on +# tracker outages so dsynth keeps building. +# +# When DPORTSV3_TRACKER_URL is unset (no config or commented out), +# every tracker_* high-level call short-circuits with no side effects. + +DPORTSV3_HOOKS_CONFIG=${DPORTSV3_HOOKS_CONFIG:-"$(dirname "$0")/dportsv3-hooks.conf"} + +tracker_log() { + : "${DPORTSV3_TRACKER_HOOK_LOG:=${DIR_LOGS:-/tmp}/dportsv3-hooks.log}" + mkdir -p -- "$(dirname -- "$DPORTSV3_TRACKER_HOOK_LOG")" 2>/dev/null || true + printf '%s %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*" \ + >> "$DPORTSV3_TRACKER_HOOK_LOG" 2>/dev/null || true +} + +tracker_fail_soft() { + tracker_log "ERROR: $*" + exit 0 +} + +tracker_should_skip() { + # Returns 0 (true) if tracker should be skipped — config missing or + # DPORTSV3_TRACKER_URL not set. Callers can guard their work with: + # tracker_should_skip && return 0 + [ ! -f "$DPORTSV3_HOOKS_CONFIG" ] && return 0 + # shellcheck disable=SC1090 + . "$DPORTSV3_HOOKS_CONFIG" + [ -z "${DPORTSV3_TRACKER_URL:-}" ] && return 0 + return 1 +} + +tracker_load_config() { + # Idempotent: safe to call multiple times. Sets defaults for any + # unset values. Soft-fails with a clear message when required values + # can't be derived. + if [ -f "$DPORTSV3_HOOKS_CONFIG" ]; then + # shellcheck disable=SC1090 + . "$DPORTSV3_HOOKS_CONFIG" + fi + + : "${PROFILE:=unknown}" + + if [ -z "${DPORTSV3_BIN:-}" ]; then + tracker_fail_soft "DPORTSV3_BIN is not configured" + fi + if [ ! -x "$DPORTSV3_BIN" ]; then + tracker_fail_soft "DPORTSV3_BIN is not executable: $DPORTSV3_BIN" + fi + if [ -z "${DPORTSV3_TRACKER_URL:-}" ]; then + tracker_fail_soft "DPORTSV3_TRACKER_URL is not configured" + fi + + # Default target = @${PROFILE} (per the "one profile per target" policy). + # If operator already set the value, keep it. + if [ -z "${DPORTSV3_TRACKER_TARGET:-}" ]; then + case "$PROFILE" in + @*) DPORTSV3_TRACKER_TARGET=$PROFILE ;; + *) DPORTSV3_TRACKER_TARGET="@$PROFILE" ;; + esac + fi + : "${DPORTSV3_TRACKER_BUILD_TYPE:=test}" + + # Per-profile state file lives under evidence/.tracker-state by + # default so it's colocated with the artifact-store evidence tree. + : "${DPORTSV3_TRACKER_STATE_DIR:=$(evidence_root)/.tracker-state}" + mkdir -p -- "$DPORTSV3_TRACKER_STATE_DIR" 2>/dev/null || true + TRACKER_STATE_FILE="$DPORTSV3_TRACKER_STATE_DIR/${PROFILE}.env" +} + +tracker_load_state() { + if [ ! -f "$TRACKER_STATE_FILE" ]; then + tracker_fail_soft "missing tracker state file: $TRACKER_STATE_FILE" + fi + # shellcheck disable=SC1090 + . "$TRACKER_STATE_FILE" + if [ "${TRACKING_DISABLED:-0}" = "1" ]; then + exit 0 + fi + if [ -z "${RUN_ID:-}" ]; then + tracker_fail_soft "tracker state file missing RUN_ID: $TRACKER_STATE_FILE" + fi +} + +tracker_write_state() { + tmp_file="$TRACKER_STATE_FILE.tmp.$$" + umask 077 + cat > "$tmp_file" < "$tmp_file" < "$tmp_json" </dev/null; then + output=$( + "$DPORTSV3_BIN" tracker enqueue-ports \ + --run "$RUN_ID" \ + --file "$tmp_json" \ + --total "$PORTS_QUEUED" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || { + rm -f -- "$tmp_json" + tracker_fail_soft "enqueue-ports failed for $origin: $output" + } + else + output=$( + "$DPORTSV3_BIN" tracker enqueue-ports \ + --run "$RUN_ID" \ + --file "$tmp_json" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || { + rm -f -- "$tmp_json" + tracker_fail_soft "enqueue-ports failed for $origin: $output" + } + fi + + rm -f -- "$tmp_json" +} + +tracker_run_start() { + tracker_should_skip && return 0 + tracker_load_config + tracker_clear_state + + output=$( + "$DPORTSV3_BIN" tracker start-build \ + --target "$DPORTSV3_TRACKER_TARGET" \ + --type "$DPORTSV3_TRACKER_BUILD_TYPE" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || tracker_disable_state "start-build failed: $output" + + RUN_ID=$(printf '%s\n' "$output" | awk '{print $4}') + case "$RUN_ID" in + ''|*[!0-9]*) + tracker_fail_soft "unable to parse run id from start-build output: $output" + ;; + esac + + tracker_write_state + tracker_log "started tracker run $RUN_ID for profile=$PROFILE target=$DPORTSV3_TRACKER_TARGET type=$DPORTSV3_TRACKER_BUILD_TYPE" + return 0 +} + +tracker_mark_building() { + tracker_should_skip && return 0 + tracker_load_config + tracker_load_state + + if [ -z "${ORIGIN:-}" ]; then + tracker_fail_soft "missing ORIGIN for pkg start hook" + fi + if [ -z "${PKGNAME:-}" ]; then + tracker_fail_soft "missing PKGNAME for pkg start hook" + fi + + version=$(tracker_pkg_version) + tracker_enqueue_one "$ORIGIN" "$version" + + output=$( + "$DPORTSV3_BIN" tracker mark-building \ + --run "$RUN_ID" \ + --origin "$ORIGIN" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || tracker_fail_soft "mark-building failed for $ORIGIN: $output" + + tracker_log "marked building run=$RUN_ID origin=$ORIGIN version=$version" + return 0 +} + +tracker_record_result() { + # Args: result (pass | fail | skipped | ignored) + tracker_should_skip && return 0 + tracker_load_config + tracker_load_state + + result_arg=$1 + if [ -z "${ORIGIN:-}" ]; then + tracker_fail_soft "missing ORIGIN for pkg result hook" + fi + if [ -z "${PKGNAME:-}" ]; then + tracker_fail_soft "missing PKGNAME for pkg result hook" + fi + + version=$(tracker_pkg_version) + + if [ -n "${DPORTSV3_TRACKER_LOG_URL_BASE:-}" ]; then + log_url=${DPORTSV3_TRACKER_LOG_URL_BASE%/}/${ORIGIN} + output=$( + "$DPORTSV3_BIN" tracker record-result \ + --run "$RUN_ID" \ + --origin "$ORIGIN" \ + --version "$version" \ + --result "$result_arg" \ + --log-url "$log_url" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || tracker_fail_soft "record-result failed for $ORIGIN: $output" + else + output=$( + "$DPORTSV3_BIN" tracker record-result \ + --run "$RUN_ID" \ + --origin "$ORIGIN" \ + --version "$version" \ + --result "$result_arg" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || tracker_fail_soft "record-result failed for $ORIGIN: $output" + fi + + tracker_log "recorded result run=$RUN_ID origin=$ORIGIN version=$version result=$result_arg" + return 0 +} + +tracker_run_end() { + tracker_should_skip && return 0 + tracker_load_config + tracker_load_state + + output=$( + "$DPORTSV3_BIN" tracker finish-build \ + --run "$RUN_ID" \ + --server "$DPORTSV3_TRACKER_URL" 2>&1 + ) || tracker_fail_soft "finish-build failed for run $RUN_ID: $output" + + tracker_log "finished tracker run $RUN_ID for profile=$PROFILE" + tracker_clear_state + return 0 +} + +distill_log() { + logfile=$1 + outdir=$2 + + mkdir -p "$outdir" + + if [ ! -r "$logfile" ]; then + write_kv_file "${outdir}/errors.txt" "missing_log=1" "logfile=${logfile}" + return + fi + + # High-signal patterns. Keep them fairly conservative to avoid dumping + # thousands of harmless "error:" hits. + # Note: use multiple -e to avoid regex quoting issues in /bin/sh. + RG_ARGS="--no-heading --color never --line-number" + + { + echo "== Summary ==" + echo "logfile: ${logfile}" + echo + echo "== First error candidates (max 60 matches) ==" + rg ${RG_ARGS} -m 60 \ + -e 'fatal error:' \ + -e 'undefined reference' \ + -e 'ld: error:' \ + -e 'collect2: error' \ + -e 'CMake Error' \ + -e 'configure: error' \ + -e 'meson\.build:.*ERROR' \ + -e '^ninja: build stopped' \ + -e 'error: failed to run custom build command for' \ + -e 'ERROR: ' \ + -e 'No such file or directory' \ + "$logfile" || true + echo + echo "== Error blocks (context +/-2, truncated later) ==" + rg ${RG_ARGS} -C 2 \ + -e 'fatal error:' \ + -e 'undefined reference' \ + -e 'ld: error:' \ + -e 'collect2: error' \ + -e 'CMake Error' \ + -e 'configure: error' \ + -e 'meson\.build:.*ERROR' \ + -e '^ninja: build stopped' \ + -e 'error: failed to run custom build command for' \ + -e '^===>\s+Stopped\s+in\s+' \ + "$logfile" || true + echo + echo "== Tail (last 200 lines) ==" + tail -n 200 "$logfile" || true + } >"${outdir}/errors.txt.tmp" + + truncate_bytes "${outdir}/errors.txt.tmp" 200000 "${outdir}/errors.txt" + rm -f "${outdir}/errors.txt.tmp" +} diff --git a/scripts/dsynth-hooks/hook_pkg_failure b/scripts/dsynth-hooks/hook_pkg_failure new file mode 100755 index 00000000000..786d8a6817d --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_failure @@ -0,0 +1,107 @@ +#!/bin/sh + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +: "${RESULT:=failure}" +: "${ORIGIN:=}" +: "${FLAVOR:=}" +: "${PKGNAME:=}" + +require_artifact_store +ensure_queue_dirs + +run_id=$(current_run_id) +ts=$(now_utc) + +origin_base=${ORIGIN%%@*} +origin_s=$(sanitize_component "$origin_base") +flavor_s=$(sanitize_component "$FLAVOR") + +bundle_id="${origin_s}" +if [ -n "$FLAVOR" ] && [ "$FLAVOR" != "$ORIGIN" ] && [ "$FLAVOR" != "$origin_base" ]; then + bundle_id="${bundle_id}@${flavor_s}" +fi +bundle_id="${bundle_id}-${ts}" + +tracker_load_config + +artifact_store bundle-upsert \ + --run-id "${run_id}" \ + --profile "${PROFILE}" \ + --target "${DPORTSV3_TRACKER_TARGET}" \ + --ts-utc "${ts}" \ + --bundle-id "${bundle_id}" \ + --origin "${origin_base}" \ + --flavor "${FLAVOR}" \ + --result "${RESULT}" >/dev/null + +artifact_store put-blob --bundle-id "${bundle_id}" --relpath "meta.txt" --stdin </dev/null + ts_utc=${ts} + result=${RESULT} + origin=${ORIGIN} + origin_base=${origin_base} + flavor=${FLAVOR} + pkgname=${PKGNAME} + profile=${PROFILE} + dir_logs=${DIR_LOGS} + dir_ports=${DIR_PORTS} + dir_buildbase=${DIR_BUILDBASE} + dir_packages=${DIR_PACKAGES} + dir_repository=${DIR_REPOSITORY} + dir_options=${DIR_OPTIONS} + dir_distfiles=${DIR_DISTFILES} +EOF + +# Log capture +logfile=$(logfile_for_origin "$ORIGIN" "$FLAVOR") +artifact_store put-blob --bundle-id "${bundle_id}" --relpath "logfile.txt" --stdin </dev/null +logfile=${logfile} +EOF + +# Port context snapshot (minimal set) +# Prefer DeltaPorts overlay (patch base) when available, fall back to build ports tree. +overlay_portdir="/build/synth/DeltaPorts/ports/${origin_base}" +portdir="${DIR_PORTS}/${origin_base}" +if [ -d "$overlay_portdir" ]; then + portdir="$overlay_portdir" +fi + +[ -r "${portdir}/Makefile" ] && artifact_store put-blob --bundle-id "${bundle_id}" --relpath "port/Makefile" --file "${portdir}/Makefile" >/dev/null +[ -r "${portdir}/Makefile.DragonFly" ] && artifact_store put-blob --bundle-id "${bundle_id}" --relpath "port/Makefile.DragonFly" --file "${portdir}/Makefile.DragonFly" >/dev/null +[ -r "${portdir}/distinfo" ] && artifact_store put-blob --bundle-id "${bundle_id}" --relpath "port/distinfo" --file "${portdir}/distinfo" >/dev/null +[ -r "${portdir}/pkg-plist" ] && artifact_store put-blob --bundle-id "${bundle_id}" --relpath "port/pkg-plist" --file "${portdir}/pkg-plist" >/dev/null +[ -r "${portdir}/pkg-descr" ] && artifact_store put-blob --bundle-id "${bundle_id}" --relpath "port/pkg-descr" --file "${portdir}/pkg-descr" >/dev/null + +if [ -d "${portdir}/files" ]; then + for pf in "${portdir}/files"/patch-*; do + [ -r "$pf" ] || continue + rel="port/files/$(basename "$pf")" + artifact_store put-blob --bundle-id "${bundle_id}" --relpath "$rel" --file "$pf" >/dev/null + done +fi + +# Distill to a bounded error bundle. +tmpdir=$(mktemp -d) +distill_log "$logfile" "$tmpdir" +if [ -r "${tmpdir}/errors.txt" ]; then + artifact_store put-blob --bundle-id "${bundle_id}" --relpath "logs/errors.txt" --file "${tmpdir}/errors.txt" >/dev/null +fi +rm -rf "$tmpdir" + +# Keep the full log available but compressed (fs-backed) +full_log_path="${DIR_LOGS}/evidence/full-logs/${bundle_id}.full.log.gz" +if [ -r "$logfile" ]; then + mkdir -p "$(dirname "$full_log_path")" + gzip -c -9 "$logfile" > "$full_log_path" || true + artifact_store put-fs --bundle-id "${bundle_id}" --relpath "logs/full.log.gz" --fs-path "$full_log_path" --kind gzip >/dev/null +fi + +# Enqueue job for processing. +enqueue_job "$bundle_id" "$origin_base" "$FLAVOR" "$PROFILE" "$run_id" "$ts" + +# Record the failure in tracker (soft no-op if tracker isn't configured). +tracker_record_result fail + +exit 0 diff --git a/scripts/dsynth-hooks/hook_pkg_ignored b/scripts/dsynth-hooks/hook_pkg_ignored new file mode 100755 index 00000000000..b905f015def --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_ignored @@ -0,0 +1,12 @@ +#!/bin/sh +# +# hook_pkg_ignored: record the per-port outcome in tracker. +# Soft no-op if tracker isn't configured. + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +tracker_record_result ignored + +exit 0 diff --git a/scripts/dsynth-hooks/hook_pkg_skipped b/scripts/dsynth-hooks/hook_pkg_skipped new file mode 100755 index 00000000000..c8c0b16f191 --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_skipped @@ -0,0 +1,12 @@ +#!/bin/sh +# +# hook_pkg_skipped: record the per-port outcome in tracker. +# Soft no-op if tracker isn't configured. + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +tracker_record_result skipped + +exit 0 diff --git a/scripts/dsynth-hooks/hook_pkg_start b/scripts/dsynth-hooks/hook_pkg_start new file mode 100755 index 00000000000..00ffc644aea --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_start @@ -0,0 +1,14 @@ +#!/bin/sh +# +# hook_pkg_start: tracker mark-building. +# Some dsynth versions fire `hook_pkg_started` instead; both files are +# provided so either naming variant works. Soft no-op if tracker isn't +# configured. + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +tracker_mark_building + +exit 0 diff --git a/scripts/dsynth-hooks/hook_pkg_started b/scripts/dsynth-hooks/hook_pkg_started new file mode 100755 index 00000000000..71bc4a59985 --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_started @@ -0,0 +1,14 @@ +#!/bin/sh +# +# hook_pkg_started: tracker mark-building. +# Some dsynth versions fire `hook_pkg_start` instead; both files are +# provided so either naming variant works. Soft no-op if tracker isn't +# configured. + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +tracker_mark_building + +exit 0 diff --git a/scripts/dsynth-hooks/hook_pkg_success b/scripts/dsynth-hooks/hook_pkg_success new file mode 100755 index 00000000000..c650ab58c95 --- /dev/null +++ b/scripts/dsynth-hooks/hook_pkg_success @@ -0,0 +1,12 @@ +#!/bin/sh +# +# hook_pkg_success: record the per-port outcome in tracker. +# Soft no-op if tracker isn't configured. + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +tracker_record_result pass + +exit 0 diff --git a/scripts/dsynth-hooks/hook_run_end b/scripts/dsynth-hooks/hook_run_end new file mode 100755 index 00000000000..7bb5ed22e1d --- /dev/null +++ b/scripts/dsynth-hooks/hook_run_end @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +root=$(evidence_root) + +# Clear current run pointer (artifact-store side, best effort). +rm -f "${root}/.current_run" + +# Finalize the tracker build_run (soft no-op if tracker isn't configured). +tracker_run_end + +exit 0 diff --git a/scripts/dsynth-hooks/hook_run_start b/scripts/dsynth-hooks/hook_run_start new file mode 100755 index 00000000000..c7d893baa21 --- /dev/null +++ b/scripts/dsynth-hooks/hook_run_start @@ -0,0 +1,20 @@ +#!/bin/sh + +set -eu + +. "$(dirname "$0")/hook_common.sh" + +root=$(evidence_root) +ts=$(now_utc) +run_id="run-$(sanitize_component "${PROFILE}")-${ts}-$$" + +# Ensure queue directories exist. +ensure_queue_dirs + +# Record current run id for per-port hooks (artifact-store side). +printf '%s\n' "${run_id}" > "${root}/.current_run" + +# Start a tracker build_run (soft no-op if tracker isn't configured). +tracker_run_start + +exit 0 diff --git a/scripts/generator/dportsv3/agent/__init__.py b/scripts/generator/dportsv3/agent/__init__.py new file mode 100644 index 00000000000..a8cb8bf5ba2 --- /dev/null +++ b/scripts/generator/dportsv3/agent/__init__.py @@ -0,0 +1,54 @@ +"""Python harness for the agentic build-failure repair loop. + +Importing this package runs ``_ensure_tokenizers_importable()`` early +so that any subsequent ``import litellm`` (or our own ``llm`` module) +won't trip over DragonFly's broken ``py311-tokenizers`` shared object. +The stub is a no-op on platforms where the real tokenizers loads. +""" + +from __future__ import annotations + +import sys +import types + + +def _ensure_tokenizers_importable() -> None: + """Pre-empt litellm's `from tokenizers import Tokenizer` chain. + + DragonFly's py311-tokenizers tokenizers.abi3.so ships with missing + DT_NEEDED entries (libonig, esaxx); loading it fails at import + time. litellm imports tokenizers transitively for local cost + calculation, which we don't use (usage totals come from + response.usage.total_tokens). Inject a no-op stub iff the real + module can't be loaded. + """ + try: + import tokenizers # noqa: F401 + return + except ImportError: + pass + + fake = types.ModuleType("tokenizers") + + class _Encoding: + ids: list[int] = [] + tokens: list[str] = [] + + class _Tokenizer: + @classmethod + def from_pretrained(cls, *args, **kwargs) -> "_Tokenizer": + return cls() + + @classmethod + def from_file(cls, *args, **kwargs) -> "_Tokenizer": + return cls() + + def encode(self, *args, **kwargs) -> _Encoding: + return _Encoding() + + fake.Tokenizer = _Tokenizer + fake.Encoding = _Encoding + sys.modules["tokenizers"] = fake + + +_ensure_tokenizers_importable() diff --git a/scripts/generator/dportsv3/agent/_manual_test_patch_flow.py b/scripts/generator/dportsv3/agent/_manual_test_patch_flow.py new file mode 100644 index 00000000000..5fe1d026cc6 --- /dev/null +++ b/scripts/generator/dportsv3/agent/_manual_test_patch_flow.py @@ -0,0 +1,209 @@ +"""Manual smoke test for dportsv3.agent.patch against a real LLM + dev-env. + +Fixtures a minimal bundle under /tmp, calls patch.run directly (bypassing +the queue runner — orchestration is tested elsewhere). The default +target port is devel/readline, which builds cleanly, so the agent +should reach rebuild_ok=true within one or two attempts. + +Usage: + export DP_TEST_MODEL='deepseek/deepseek-v4-flash' + export DP_TEST_API_KEY='YOUR_KEY' + export DP_TEST_ENV='2026Q2' + export DP_TEST_ORIGIN='devel/readline' # default + export DP_TEST_TIER_ITERATIONS='4' # default (matches ASSIST tier) + export DP_TEST_TIER_TOKENS='120000' # default (matches ASSIST tier) + ./scripts/generator/.venv/bin/python -m dportsv3.agent._manual_test_patch_flow + +Prints per-attempt token counts, final status, and the rebuild_proof +JSON (or lack thereof). Bundle dir is preserved so you can inspect +the written artifacts (patch.md, rebuild_proof.json, patch_audit.json, +changes.diff). +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +import tempfile +from pathlib import Path + +from dportsv3.agent import llm, patch, tools, tool_loop +from dportsv3.agent.policy import Tier + + +def _install_session_dump(trace_path: Path) -> None: + """Wrap llm.complete and tools.dispatch so every turn is logged to + a JSONL file. Lets us share the full conversation post-mortem.""" + trace_path.parent.mkdir(parents=True, exist_ok=True) + trace_path.write_text("") # truncate + + real_complete = llm.complete + real_dispatch = tools.dispatch + + def _redact(messages: list[dict]) -> list[dict]: + # Truncate very long string content for the trace; keep first + # 800 chars per field. Adjust if you need more detail. + out = [] + for m in messages: + r = dict(m) + for k, v in list(r.items()): + if isinstance(v, str) and len(v) > 800: + r[k] = v[:800] + f"…[+{len(v) - 800} chars]" + out.append(r) + return out + + def traced_complete(messages, **kw): + resp = real_complete(messages, **kw) + rec = { + "kind": "llm_call", + "model": kw.get("model"), + "n_messages": len(messages), + "messages_preview": _redact(messages), + "response": { + "text": (resp.text or "")[:1200], + "tool_calls": [ + {"id": tc.id, "name": tc.name, "arguments": tc.arguments} + for tc in resp.tool_calls + ], + "usage": { + "prompt": resp.usage.prompt_tokens, + "completion": resp.usage.completion_tokens, + "total": resp.usage.total_tokens, + }, + "reasoning_content": (resp.reasoning_content or "")[:600], + }, + } + with trace_path.open("a") as f: + f.write(json.dumps(rec) + "\n") + return resp + + def traced_dispatch(name, arguments, *, env): + result = real_dispatch(name, arguments, env=env) + rec = { + "kind": "tool_dispatch", + "tool": name, + "arguments": arguments, + "result_keys": sorted(result.keys()) if isinstance(result, dict) else None, + "ok": bool(result.get("ok")) if isinstance(result, dict) else None, + # Don't include result content (file bytes etc.) — too big. + # Stdout/stderr tails get truncated. + "stdout_tail": (result.get("stdout_tail") or "")[:600] if isinstance(result, dict) else None, + "stderr_tail": (result.get("stderr_tail") or "")[:600] if isinstance(result, dict) else None, + } + with trace_path.open("a") as f: + f.write(json.dumps(rec) + "\n") + return result + + llm.complete = traced_complete + tool_loop.llm.complete = traced_complete + tools.dispatch = traced_dispatch + tool_loop.tools.dispatch = traced_dispatch + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(name)s %(message)s") + logging.getLogger("dportsv3.agent").setLevel(logging.DEBUG) + + model = os.environ.get("DP_TEST_MODEL") + if not model: + print("error: set DP_TEST_MODEL (e.g. deepseek/deepseek-v4-flash)", file=sys.stderr) + return 2 + api_base = os.environ.get("DP_TEST_API_BASE") or None + api_key = os.environ.get("DP_TEST_API_KEY") or None + provider = os.environ.get("DP_TEST_PROVIDER") or None + env = os.environ.get("DP_TEST_ENV", "2026Q2") + origin = os.environ.get("DP_TEST_ORIGIN", "devel/readline") + + tier = Tier( + name="MANUAL_TEST", + max_iterations=int(os.environ.get("DP_TEST_TIER_ITERATIONS", "4")), + max_tokens=int(os.environ.get("DP_TEST_TIER_TOKENS", "120000")), + ) + + # Fixture a minimal bundle directory. + bundle_dir = Path(tempfile.mkdtemp(prefix="dp-patch-smoke-")) + + # Install session-dump traces (per-turn LLM call + tool dispatch). + trace_path = bundle_dir / "session.jsonl" + _install_session_dump(trace_path) + (bundle_dir / "analysis").mkdir() + (bundle_dir / "meta.txt").write_text( + f"origin={origin}\nprofile=DragonFly\nbundle_id=smoke-fixture\n" + ) + # Synthetic "error": port may or may not actually fail; the agent + # decides by calling dsynth_build. Phrasing avoids over-priming. + (bundle_dir / "errors.txt").write_text( + f"# fixture bundle\n" + f"Port: {origin}\n" + f"Reported state: a recent dsynth run may have failed. Please run\n" + f"dsynth_build to verify the current state of the port and, if\n" + f"it fails, propose minimal edits to make it build.\n" + ) + (bundle_dir / "analysis" / "triage.md").write_text( + "## Classification\nunknown\n\n## Confidence\nlow\n" + ) + + payload = f"""# Patch Job (fixture) + +## Origin +{origin} + +## Bundle +{bundle_dir} + +## Errors (fixture) +{(bundle_dir / "errors.txt").read_text()} + +## Triage +Classification: unknown +Confidence: low + +Verify the port's current state in the dev-env. If it builds with +dsynth_build, emit rebuild_proof.json with rebuild_ok=true. If it +fails, propose minimal DeltaPorts edits and retry. +""" + + print("---") + print(f"bundle: {bundle_dir}") + print(f"env: {env}") + print(f"origin: {origin}") + print(f"model: {model}") + print(f"tier: iter={tier.max_iterations} tokens={tier.max_tokens}") + print("---") + + result = patch.run( + payload, + tier=tier, + env=env, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=provider, + ) + + print() + print(f"status: {result.status}") + print(f"attempts: {len(result.attempts)}") + print(f"tokens (p/c/t): {result.usage.prompt_tokens}/" + f"{result.usage.completion_tokens}/{result.usage.total_tokens}") + for a in result.attempts: + print(f" attempt {a.attempt}: {a.tokens} tokens, rebuild_ok={a.rebuild_ok}") + print() + print("rebuild_proof:") + if result.proof: + print(json.dumps(result.proof, indent=2)) + else: + print(" (none parsed)") + print() + print("final text tail (last 600 chars):") + print(result.final_text[-600:] if result.final_text else "(empty)") + print() + print(f"bundle artifacts preserved at: {bundle_dir}") + print(f"session trace (JSONL): {trace_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generator/dportsv3/agent/_manual_test_tool_loop.py b/scripts/generator/dportsv3/agent/_manual_test_tool_loop.py new file mode 100644 index 00000000000..f6c7cc5dec9 --- /dev/null +++ b/scripts/generator/dportsv3/agent/_manual_test_tool_loop.py @@ -0,0 +1,87 @@ +"""Manual smoke test for dportsv3.agent.tool_loop against a real LLM. + +Not a unit test, not committed for CI — temporary verification script. +Delete after step 4 lands and the patch flow has its own E2E smoke. + +Usage: + export DP_TEST_MODEL='openai/some-model' # litellm model string + export DP_TEST_API_BASE='https://endpoint/v1' # optional; only for custom endpoints + export DP_TEST_API_KEY='your-key' # optional; falls back to provider's standard env + export DP_TEST_PROVIDER='openai' # optional; pass through to litellm's custom_llm_provider + export DP_TEST_ENV='2026Q2' # dev-env name; default 2026Q2 + ./scripts/generator/.venv/bin/python -m dportsv3.agent._manual_test_tool_loop + +DP_TEST_PROVIDER notes: +- Leave unset for native providers (litellm routes from the model + prefix: anthropic/, deepseek/, nvidia_nim/, ...). +- Set to 'openai' when talking to an OpenAI-compatible third-party + endpoint (Groq, Together, opencode.ai/zen, ...) whose model ID + contains a substring litellm would otherwise route natively (e.g. + any 'deepseek-*' model name on opencode.ai/zen needs this). +- Set to any other litellm-supported provider name to force that path. + +What it does: +- Drives the LLM with a tiny system prompt that asks it to call + env_verify, then get_file on a known port's Makefile, then report + the PORTNAME. +- Prints per-turn debug info from tool_loop. +- Prints the final text + token totals. +""" + +from __future__ import annotations + +import logging +import os +import sys + +from dportsv3.agent import tool_loop + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(name)s %(message)s") + logging.getLogger("dportsv3.agent.tool_loop").setLevel(logging.DEBUG) + + model = os.environ.get("DP_TEST_MODEL") + if not model: + print("error: set DP_TEST_MODEL (e.g. openai/gpt-5-nano)", file=sys.stderr) + return 2 + api_base = os.environ.get("DP_TEST_API_BASE") or None + api_key = os.environ.get("DP_TEST_API_KEY") or None + provider = os.environ.get("DP_TEST_PROVIDER") or None + env = os.environ.get("DP_TEST_ENV", "2026Q2") + + messages = [ + { + "role": "system", + "content": ( + "You have tools to operate on a DragonFly ports dev-env. " + "Call env_verify first, then read " + "/work/DeltaPorts/ports/devel/readline/Makefile with get_file. " + "The Makefile is base64-encoded in the result's content field. " + "Tell me what the PORTNAME= line says." + ), + }, + {"role": "user", "content": "Go."}, + ] + + final, usage = tool_loop.run( + messages, + model=model, + env=env, + api_base=api_base, + api_key=api_key, + custom_llm_provider=provider, + timeout=120, + max_turns=8, + ) + + print("---") + print(f"final text:\n{final.text}") + print(f"tokens (prompt/completion/total): " + f"{usage.prompt_tokens}/{usage.completion_tokens}/{usage.total_tokens}") + print(f"messages in conversation history: {len(messages)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generator/dportsv3/agent/_manual_test_triage_tier.py b/scripts/generator/dportsv3/agent/_manual_test_triage_tier.py new file mode 100644 index 00000000000..9a980e9bd67 --- /dev/null +++ b/scripts/generator/dportsv3/agent/_manual_test_triage_tier.py @@ -0,0 +1,208 @@ +"""Manual smoke test for the triage flow + tier dispatch (phase 3 step 5). + +Fixtures a bundle with a synthetic but realistic error log, calls +dportsv3.agent.triage.run against a real LLM, then resolves the +trust tier via policy.tier_for and reports what the runner would +do (auto-enqueue patch vs. drop to MANUAL). + +Usage: + export DP_TEST_MODEL='deepseek/deepseek-v4-flash' + export DP_TEST_API_KEY='YOUR_KEY' + export DP_TEST_PROVIDER= # optional; for openai-compat relays + export DP_TEST_API_BASE= # optional + export DP_TEST_FIXTURE='compile-error' # or 'plist-error' | 'unknown' + ./scripts/generator/.venv/bin/python -m dportsv3.agent._manual_test_triage_tier + +Fixtures available: +- compile-error — readline-like 'lvalue required' compile error (expected + tier: ASSIST given decent confidence) +- plist-error — a missing-file/extra-file pkg-plist error (expected + tier: AUTO if model is confident) +- unknown — an opaque generic failure (expected tier: MANUAL) +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +import tempfile +from pathlib import Path + +from dportsv3.agent import llm, policy, triage + + +# ----------------------------------------------------------------------------- +# Fixture bundles — drop-in error.txt snippets representative of real +# dsynth failures. We don't fabricate the LLM's response; we feed it a +# real failure shape and let it classify. +# ----------------------------------------------------------------------------- + +FIXTURES = { + "compile-error": { + "origin": "devel/readline", + "errors": """\ +===> Building for readline-8.3 +gcc -DHAVE_CONFIG_H -DRL_LIBRARY_VERSION='"8.3"' -DSHELL -O2 -pipe -fPIC \\ + -c -o terminal.o terminal.c +./terminal.c:583:13: error: lvalue required as left operand of assignment + dumbterm = STREQ (term, "dumb") || STREQ (term, "vt52"); + ^ +1 error generated. +*** Error code 1 +Stop. +make[1]: stopped making "all" in /construction/devel/readline/work/readline-8.3 +*** Error code 1 +""", + "meta": "origin=devel/readline\nprofile=DragonFly\nbundle_id=fixture-compile\n", + }, + "plist-error": { + "origin": "lang/python311", + "errors": """\ +===> Stage Compare reports +===> Found user-supplied files in PREFIX but not in pkg-plist: + lib/python3.11/site-packages/__pycache__/test_module.cpython-311.pyc +===> Building of pkg-plist failed; some files are extra +*** Error code 1 +""", + "meta": "origin=lang/python311\nprofile=DragonFly\nbundle_id=fixture-plist\n", + }, + "unknown": { + "origin": "deskutils/somewhere", + "errors": """\ +===> Trying to stage something +[error output truncated due to log rotation] +*** Error code 137 +""", + "meta": "origin=deskutils/somewhere\nprofile=DragonFly\nbundle_id=fixture-unknown\n", + }, +} + + +def _install_session_dump(trace_path: Path) -> None: + """Wrap llm.complete so every triage turn is logged to a JSONL file.""" + trace_path.parent.mkdir(parents=True, exist_ok=True) + trace_path.write_text("") + real_complete = llm.complete + + def traced_complete(messages, **kw): + resp = real_complete(messages, **kw) + rec = { + "kind": "llm_call", + "model": kw.get("model"), + "n_messages": len(messages), + "response": { + "text": (resp.text or "")[:1500], + "reasoning_content": (resp.reasoning_content or "")[:600], + "usage": { + "prompt": resp.usage.prompt_tokens, + "completion": resp.usage.completion_tokens, + "total": resp.usage.total_tokens, + }, + }, + } + with trace_path.open("a") as f: + f.write(json.dumps(rec) + "\n") + return resp + + llm.complete = traced_complete + triage.llm.complete = traced_complete + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(name)s %(message)s") + + model = os.environ.get("DP_TEST_MODEL") + if not model: + print("error: set DP_TEST_MODEL", file=sys.stderr) + return 2 + api_base = os.environ.get("DP_TEST_API_BASE") or None + api_key = os.environ.get("DP_TEST_API_KEY") or None + provider = os.environ.get("DP_TEST_PROVIDER") or None + fixture_name = os.environ.get("DP_TEST_FIXTURE", "compile-error") + + fixture = FIXTURES.get(fixture_name) + if fixture is None: + print(f"error: unknown fixture {fixture_name!r}; choose from " + f"{list(FIXTURES)}", file=sys.stderr) + return 2 + + # Fixture a minimal bundle directory. + bundle_dir = Path(tempfile.mkdtemp(prefix=f"dp-triage-{fixture_name}-")) + (bundle_dir / "analysis").mkdir() + (bundle_dir / "meta.txt").write_text(fixture["meta"]) + (bundle_dir / "errors.txt").write_text(fixture["errors"]) + + trace_path = bundle_dir / "session.jsonl" + _install_session_dump(trace_path) + + # Build a triage payload directly — short and focused. + payload = f"""# Triage Job (fixture) + +## Origin +{fixture["origin"]} + +## Reported errors +``` +{fixture["errors"]} +``` + +Classify the failure and assign confidence according to the system prompt +output format. +""" + + print("---") + print(f"fixture: {fixture_name}") + print(f"origin: {fixture['origin']}") + print(f"model: {model}") + print(f"bundle: {bundle_dir}") + print("---") + print() + + result = triage.run( + payload, + bundle_dir=bundle_dir, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=provider, + ) + + print(f"classification: {result.classification!r}") + print(f"confidence: {result.confidence!r}") + print(f"snippet_rounds: {result.snippet_rounds}") + print(f"tokens (p/c/t): {result.usage.prompt_tokens}/" + f"{result.usage.completion_tokens}/{result.usage.total_tokens}") + print() + + # Resolve tier via the same policy the runner uses. + policy_path = os.environ.get( + "DP_HARNESS_POLICY", + str(Path(__file__).resolve().parents[4] / "config" / "agentic-policy.json"), + ) + pol = policy.load_policy(policy_path) + tier = policy.tier_for(pol, result.classification, result.confidence) + + print(f"policy: {policy_path}") + print(f"tier: {tier.name}") + print(f"max_iterations: {tier.max_iterations}") + print(f"max_tokens: {tier.max_tokens}") + print() + if tier.name == "MANUAL": + print("decision: no auto-enqueue (MANUAL tier)") + print(" operator must hand-fire a patch job if they want to proceed") + else: + print(f"decision: AUTO-ENQUEUE patch job (tier={tier.name})") + print(f" patch flow gets budget: {tier.max_iterations} attempts × " + f"{tier.max_tokens} tokens") + print() + print("final triage text (first 600 chars):") + print(result.text[:600]) + print() + print(f"trace: {trace_path}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/generator/dportsv3/agent/attempt_loop.py b/scripts/generator/dportsv3/agent/attempt_loop.py new file mode 100644 index 00000000000..b8678e00ea6 --- /dev/null +++ b/scripts/generator/dportsv3/agent/attempt_loop.py @@ -0,0 +1,195 @@ +"""Budget-bounded retry loop around tool_loop for the patch flow. + +One ``run(...)`` is one patch *job*: up to ``tier.max_iterations`` +attempts, each itself a full multi-turn tool_loop conversation. Each +attempt starts fresh from [system, user] — we do **not** extend the +prior attempt's growing history, because tool-call traces compound +fast and the budget would melt by attempt 3 otherwise. Between +attempts we append a small failure-context message describing what +went wrong, so the LLM knows it's on a retry. + +Stops when: +- the LLM emits Rebuild Proof JSON with ``rebuild_ok=true`` → success +- ``usage.total_tokens >= tier.max_tokens`` → budget-exhausted +- ``attempt == tier.max_iterations`` without success → needs-help +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, field +from pathlib import Path + +from . import llm, prompts, tool_loop +from .llm import Usage + +log = logging.getLogger(__name__) + + +@dataclass +class AttemptInfo: + attempt: int # 1-indexed + tokens: int + rebuild_ok: bool + proof: dict | None = None # parsed Rebuild Proof JSON for this attempt + + +@dataclass +class PatchResult: + status: str # "success" | "needs-help" | "budget-exhausted" + final_text: str + usage: Usage = field(default_factory=Usage) + attempts: list[AttemptInfo] = field(default_factory=list) + proof: dict | None = None # the final/winning Rebuild Proof JSON (if any) + + +_PROOF_BLOCK_RE = re.compile( + r"##\s*Rebuild Proof\s*\(JSON\)\s*\n+```(?:json)?\s*\n(.*?)\n```", + re.IGNORECASE | re.DOTALL, +) + + +def _parse_rebuild_proof(text: str) -> dict | None: + """Extract the final ``## Rebuild Proof (JSON)`` block, if present.""" + matches = _PROOF_BLOCK_RE.findall(text) + if not matches: + return None + raw = matches[-1].strip() + try: + proof = json.loads(raw) + except json.JSONDecodeError as exc: + log.warning("attempt_loop: rebuild_proof JSON parse failed: %s", exc) + return None + if not isinstance(proof, dict): + return None + return proof + + +def _failure_context_message(attempt_idx: int, prev_text: str) -> dict: + """Build the user message that nudges the LLM into a retry.""" + snippet = prev_text[-2000:] if len(prev_text) > 2000 else prev_text + return { + "role": "user", + "content": ( + f"Previous attempt #{attempt_idx} did not succeed.\n\n" + f"Tail of your prior response:\n" + f"```\n{snippet}\n```\n\n" + "Inspect what went wrong, adjust your approach, and try again. " + "If you've tried the same idea twice and it failed both times, " + "describe the obstacle in your Patch Log and stop — don't burn " + "the budget thrashing." + ), + } + + +def run( + payload: str, + *, + tier, # dportsv3.agent.policy.Tier + env: str, + model: str, + api_base: str | None = None, + api_key: str | None = None, + custom_llm_provider: str | None = None, + timeout: int = 600, + max_tool_turns: int = 12, +) -> PatchResult: + """Run the patch flow for one bundle, returning a structured PatchResult.""" + base_messages = [ + {"role": "system", "content": prompts.PATCH_SYSTEM}, + {"role": "user", "content": payload}, + ] + + total_usage = Usage() + attempts: list[AttemptInfo] = [] + prev_text = "" + final_text = "" + winning_proof: dict | None = None + + iterations = max(1, int(getattr(tier, "max_iterations", 1) or 1)) + budget = int(getattr(tier, "max_tokens", 0) or 0) + + for attempt_idx in range(1, iterations + 1): + if attempt_idx == 1: + messages = list(base_messages) + else: + messages = list(base_messages) + [_failure_context_message(attempt_idx - 1, prev_text)] + + # Remaining tokens this attempt is allowed to consume. + remaining = (budget - total_usage.total_tokens) if budget else 0 + log.info( + "attempt_loop: starting attempt %d/%d (tokens used so far: %d / %d, remaining %d)", + attempt_idx, iterations, total_usage.total_tokens, budget, remaining, + ) + + if budget and remaining <= 0: + log.warning("attempt_loop: budget already exhausted before attempt %d", attempt_idx) + return PatchResult( + status="budget-exhausted", + final_text=final_text, + usage=total_usage, + attempts=attempts, + proof=None, + ) + + response, attempt_usage = tool_loop.run( + messages, + model=model, + env=env, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + max_turns=max_tool_turns, + max_tokens=remaining, + ) + total_usage.add(attempt_usage) + prev_text = response.text or "" + final_text = prev_text + + proof = _parse_rebuild_proof(prev_text) + rebuild_ok = bool(proof and proof.get("rebuild_ok") is True) + + attempts.append( + AttemptInfo( + attempt=attempt_idx, + tokens=attempt_usage.total_tokens, + rebuild_ok=rebuild_ok, + proof=proof, + ) + ) + + if rebuild_ok: + log.info("attempt_loop: success on attempt %d", attempt_idx) + winning_proof = proof + return PatchResult( + status="success", + final_text=final_text, + usage=total_usage, + attempts=attempts, + proof=winning_proof, + ) + + if budget and total_usage.total_tokens >= budget: + log.warning( + "attempt_loop: budget exhausted after attempt %d (%d >= %d)", + attempt_idx, total_usage.total_tokens, budget, + ) + return PatchResult( + status="budget-exhausted", + final_text=final_text, + usage=total_usage, + attempts=attempts, + proof=proof, + ) + + log.info("attempt_loop: needs-help after %d attempts", iterations) + return PatchResult( + status="needs-help", + final_text=final_text, + usage=total_usage, + attempts=attempts, + proof=attempts[-1].proof if attempts else None, + ) diff --git a/scripts/generator/dportsv3/agent/llm.py b/scripts/generator/dportsv3/agent/llm.py new file mode 100644 index 00000000000..89a88b66acf --- /dev/null +++ b/scripts/generator/dportsv3/agent/llm.py @@ -0,0 +1,124 @@ +"""litellm wrapper with a normalized response shape. + +The tokenizers stub (needed on DragonFly) is in dportsv3.agent.__init__, +which runs before any module here is loaded. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass +class Usage: + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + + def add(self, other: "Usage") -> None: + self.prompt_tokens += other.prompt_tokens + self.completion_tokens += other.completion_tokens + self.total_tokens += other.total_tokens + + +@dataclass +class ToolCall: + id: str + name: str + arguments: dict + + +@dataclass +class Response: + text: str + tool_calls: list[ToolCall] = field(default_factory=list) + usage: Usage = field(default_factory=Usage) + # Thinking-mode chain-of-thought (DeepSeek's `reasoning_content`, + # OpenAI o-series reasoning summaries via the same field name on + # OpenAI-compat backends). None for non-thinking models. + reasoning_content: str | None = None + raw: object = None # opaque litellm ModelResponse for debugging + + +def complete( + messages: list[dict], + *, + model: str, + tools: list[dict] | None = None, + api_base: str | None = None, + api_key: str | None = None, + custom_llm_provider: str | None = None, + timeout: int | None = None, + temperature: float | None = None, +) -> Response: + """Call an LLM provider via litellm and return a normalized Response. + + ``messages`` is the OpenAI-style chat list. ``tools`` is the OpenAI-style + JSON schema list (litellm converts to the native shape per provider). + + ``custom_llm_provider`` forces litellm onto a specific provider's + client code path regardless of what the model name looks like. + Important when talking to OpenAI-compatible third-party endpoints + (opencode.ai/zen, Groq, Together, …) whose model IDs may contain + a native-provider substring (``deepseek-*``, ``claude-*``) that + litellm's model→provider heuristic would otherwise mis-route. Pass + ``custom_llm_provider="openai"`` together with ``api_base`` to pin + the OpenAI-compat code path. + """ + import litellm + + kwargs: dict = {"model": model, "messages": messages} + if tools: + kwargs["tools"] = tools + if api_base: + kwargs["api_base"] = api_base + if api_key: + kwargs["api_key"] = api_key + if custom_llm_provider: + kwargs["custom_llm_provider"] = custom_llm_provider + if timeout is not None: + kwargs["timeout"] = timeout + if temperature is not None: + kwargs["temperature"] = temperature + + completion = litellm.completion(**kwargs) + choice = completion.choices[0] + msg = choice.message + + text = msg.content or "" + + tool_calls: list[ToolCall] = [] + raw_calls = getattr(msg, "tool_calls", None) or [] + for call in raw_calls: + fn = call.function + arguments = fn.arguments + if isinstance(arguments, str): + import json + try: + arguments = json.loads(arguments) if arguments else {} + except json.JSONDecodeError: + arguments = {"_raw": arguments} + tool_calls.append( + ToolCall(id=call.id, name=fn.name, arguments=arguments or {}) + ) + + raw_usage = getattr(completion, "usage", None) + usage = Usage() + if raw_usage is not None: + usage.prompt_tokens = getattr(raw_usage, "prompt_tokens", 0) or 0 + usage.completion_tokens = getattr(raw_usage, "completion_tokens", 0) or 0 + usage.total_tokens = getattr(raw_usage, "total_tokens", 0) or 0 + + # Some thinking-mode providers (DeepSeek's v4-* models, certain + # OpenAI-compat relays) expose intermediate chain-of-thought as + # `reasoning_content` on the message object. The upstream API + # requires it to be echoed back on multi-turn requests. + reasoning_content = getattr(msg, "reasoning_content", None) or None + + return Response( + text=text, + tool_calls=tool_calls, + usage=usage, + reasoning_content=reasoning_content, + raw=completion, + ) diff --git a/scripts/generator/dportsv3/agent/patch.py b/scripts/generator/dportsv3/agent/patch.py new file mode 100644 index 00000000000..5a799ac75ad --- /dev/null +++ b/scripts/generator/dportsv3/agent/patch.py @@ -0,0 +1,36 @@ +"""Patch flow — thin wrapper over attempt_loop.""" + +from __future__ import annotations + +from . import attempt_loop +from .attempt_loop import PatchResult + + +def run( + payload: str, + *, + tier, + env: str, + model: str, + api_base: str | None = None, + api_key: str | None = None, + custom_llm_provider: str | None = None, + timeout: int = 600, + max_tool_turns: int = 30, +) -> PatchResult: + """Run the patch agent for one bundle. Returns the PatchResult. + + The runner is responsible for persisting the result to the bundle + (patch.md, rebuild_proof.json, changes.diff, audit JSON). + """ + return attempt_loop.run( + payload, + tier=tier, + env=env, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + max_tool_turns=max_tool_turns, + ) diff --git a/scripts/generator/dportsv3/agent/policy.py b/scripts/generator/dportsv3/agent/policy.py new file mode 100644 index 00000000000..26340f55a9b --- /dev/null +++ b/scripts/generator/dportsv3/agent/policy.py @@ -0,0 +1,87 @@ +"""Trust-tier + budget policy. + +Loads ``config/agentic-policy.json`` and maps a triage +``(classification, confidence)`` pair to a tier with budget. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path + +CONFIDENCE_ORDER = ["low", "medium", "high"] + + +@dataclass +class Tier: + name: str # "AUTO" | "ASSIST" | "MANUAL" + max_iterations: int = 0 + max_tokens: int = 0 + + +@dataclass +class Policy: + tiers: dict[str, Tier] + classification_to_tier: dict[str, str] + confidence_floor: dict[str, str] + + +def load_policy(path: Path | str) -> Policy: + raw = json.loads(Path(path).read_text()) + tiers = { + name: Tier( + name=name, + max_iterations=int(spec.get("max_iterations", 0)), + max_tokens=int(spec.get("max_tokens", 0)), + ) + for name, spec in raw.get("tiers", {}).items() + } + return Policy( + tiers=tiers, + classification_to_tier=dict(raw.get("classification_to_tier", {})), + confidence_floor=dict(raw.get("confidence_floor", {})), + ) + + +def _confidence_at_least(value: str, floor: str) -> bool: + if value not in CONFIDENCE_ORDER or floor not in CONFIDENCE_ORDER: + return False + return CONFIDENCE_ORDER.index(value) >= CONFIDENCE_ORDER.index(floor) + + +def tier_for(policy: Policy, classification: str, confidence: str) -> Tier: + """Resolve the tier for a triage outcome, cascading confidence_floor downgrades. + + Each tier carries a ``confidence_floor`` that the triage confidence + must meet. If confidence is below the floor, the tier is downgraded + one step (AUTO → ASSIST → MANUAL) and the new tier's floor is + re-evaluated. Cascades until either the floor is met or MANUAL is + reached. Unknown classifications start at MANUAL. + + Examples (with floors AUTO=high, ASSIST=medium): + plist-error + high → AUTO + plist-error + medium → ASSIST (AUTO floor not met → downgrade) + plist-error + low → MANUAL (cascades AUTO → ASSIST → MANUAL) + compile-error + low → MANUAL (ASSIST floor not met → downgrade) + """ + tier_name = policy.classification_to_tier.get(classification, "MANUAL") + # Cascade downgrades until the confidence floor is satisfied or we + # land at MANUAL (no further downgrade possible). + while True: + floor = policy.confidence_floor.get(tier_name) + if not floor or _confidence_at_least(confidence, floor): + break + next_name = _downgrade(tier_name) + if next_name == tier_name: + break + tier_name = next_name + return policy.tiers.get(tier_name) or Tier(name="MANUAL") + + +def _downgrade(tier_name: str) -> str: + if tier_name == "AUTO": + return "ASSIST" + if tier_name == "ASSIST": + return "MANUAL" + return tier_name diff --git a/scripts/generator/dportsv3/agent/prompts.py b/scripts/generator/dportsv3/agent/prompts.py new file mode 100644 index 00000000000..45a89bd74eb --- /dev/null +++ b/scripts/generator/dportsv3/agent/prompts.py @@ -0,0 +1,160 @@ +"""System prompts for the triage and patch agents. + +Bodies lifted (and adapted) from the former config/opencode/agent/*.md +files. The response-format directives below are contractual: the +runner's parsers (parse_triage_output, the rebuild_proof JSON block +extraction in attempt_loop) depend on the exact heading text. +""" + +TRIAGE_SYSTEM = """# DeltaPorts Build Failure Triage Agent + +You triage DragonFlyBSD dsynth build failures using ONLY the provided evidence. + +## Output (exact headings) + +## Classification +One of: compile-error, configure-error, patch-error, plist-error, missing-dep, fetch-error, unknown + +## Platform +One of: dragonfly-specific, freebsd-upstream, generic + +## Root Cause +1-3 sentences. + +## Evidence +- Quote exact log lines from errors.txt that support the root cause. + +## Suggested Fix +Concrete DeltaPorts-style fix plan. + +## Confidence +One of: high, medium, low + +## Notes +Optional. +""" + + +PATCH_SYSTEM = """# DeltaPorts Patch Agent + +You fix DragonFlyBSD dsynth build failures by editing files in a +disposable dev-env chroot and rebuilding with dsynth, iteratively, until +the build passes. + +## Directory layout (memorize this — it's the #1 source of wasted turns) + +The env's writable overlay has three trees under `/work/`, each with a +distinct role: + +- `/work/freebsd-ports//` + The FreeBSD ports collection — **upstream, reference only**. Never + edit. Use `get_file` here when you need to see what FreeBSD has. + +- `/work/DeltaPorts/ports//` + The DragonFly-specific overlay (patches, Makefile.DragonFly, + `dragonfly/*` files, `diffs/*.diff`). **This is the source of truth + you edit.** Always put_file here when changing a port. + +- `/work/DPorts//` + The **buildable, composed** port — what dsynth reads. It is + materialized from freebsd-ports + DeltaPorts via + `materialize_dports(origin)`. **Never edit directly; your edits + will be wiped on the next materialize.** Read-only reference. + +## The repair loop + +1. Call `env_verify` once at the start. If status != ready, stop and + report — no other tool will work. + +2. Inspect the failure. Use `get_file` and `grep` over + `/work/DeltaPorts/ports//` and `/work/DPorts//` to + understand the port's existing patches, Makefile, and what the + build is doing. + +3. Edit `/work/DeltaPorts/ports//...` files. Use `put_file` + with `expected_sha256` (from a prior `get_file`) to guard against + concurrent edits. + + For generating new patches against the extracted source: + - `extract(origin)` to fetch + extract into WRKSRC + - `dupe(/work/DPorts//work/.../file.c)` to snapshot the + original + - `put_file` to edit the source file inside WRKSRC + - `genpatch()` to produce a unified diff in + /work/genpatch-out/ + - `install_patches(origin)` to copy patches into DeltaPorts + +4. Propagate edits: `materialize_dports(origin)` rebuilds + `/work/DPorts//` from the latest DeltaPorts state. + +5. Build: `dsynth_build(origin)`. Inspect `stdout_tail` / + `stderr_tail`. If `rebuild_ok=true`, you're done; emit the final + output (below) and stop. If false, return to step 2 with the new + error info. + +6. Use `emit_diff(origin, relpath)` whenever you want to see the + net change you've made to a specific DeltaPorts file (host-side + git diff against HEAD, no commits). + +## Discipline + +- **Never** commit, push, or create branches. The env's writable + overlay holds your edits; we audit via `emit_diff`. +- **Never** edit `/work/DPorts//` directly — your changes will + be lost on the next `materialize_dports`. +- Prefer minimal, surgical edits. A 3-line patch beats a rewrite. +- Use `expected_sha256` on `put_file` when you've previously read a file. +- On `dsynth_build` failure, **call `dsynth_log(origin)` immediately**. + The real build error is in the per-port log, not in dsynth_build's + stdout_tail. Don't grep `/work/DPorts/.../*.log` — those files don't + exist; dsynth's logs live under `/work/dsynth/logs/`. +- When listing a directory, use `list_dir(path)`. `get_file` only works + on regular files (it will say "is a directory" if you pass a dir). +- **Knowing when to stop:** if you've called `dsynth_build` and it + returned `rebuild_ok=true`, you are **done** — emit the final + output immediately and stop. Don't keep exploring. +- **Knowing when to give up:** if you've tried two distinct approaches + and both failed at the same point, or you can't find the root cause + after inspecting the build log, **stop** and emit your final response + with `Rebuild Status: gave-up` and a brief explanation in Patch Log. + Don't keep burning the iteration / token budget thrashing. + +## Output (exact headings) + +When you finish (success or give-up), end your response with these +sections in this order: + +## Patch Log +Brief narrative of what you tried (one sentence per tool sequence). + +## Rebuild Status +One of: success | failed | gave-up + +## Patch Plan (JSON) +```json +{ + "origin": "category/portname", + "summary": "1-sentence what you did", + "files_touched": ["ports//...", ...], + "tools_used": ["materialize_dports", "dsynth_build", ...] +} +``` + +## Rebuild Proof (JSON) +```json +{ + "origin": "category/portname", + "rebuild_ok": true, + "dsynth_profile": "DragonFly", + "build_command": "dsynth -p DragonFly build category/portname", + "timestamp_utc": "2026-05-18T20:00:00Z" +} +``` + +The `Rebuild Proof (JSON)` block is **mandatory** in your final +response. It is parsed mechanically. `rebuild_ok` must be `true` only +if `dsynth_build` returned `rebuild_ok=true` in your most recent call. +Otherwise it must be `false`. + +No branching, no git push, no PR. Local rebuild proof only. +""" diff --git a/scripts/generator/dportsv3/agent/snippets.py b/scripts/generator/dportsv3/agent/snippets.py new file mode 100644 index 00000000000..22d4eb998bb --- /dev/null +++ b/scripts/generator/dportsv3/agent/snippets.py @@ -0,0 +1,83 @@ +"""Thin wrapper around scripts/snippet-extractor. + +The extractor reads snippet requests from ``analysis/triage.md`` (or +``analysis/patch.md``) in the bundle and writes results under +``analysis/snippets/round_N/``. We invoke it as a subprocess and return +the list of files it produced so the harness can append their content +to the next LLM call. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +DEFAULT_EXTRACTOR = Path(__file__).resolve().parents[4] / "scripts" / "snippet-extractor" + + +def extract_round( + bundle_dir: Path, + round_number: int, + *, + extractor: Path | None = None, + prefer_workdir: bool = True, +) -> tuple[int, list[Path]]: + """Run snippet-extractor for one round; return (exit_code, list of snippet files). + + Exit codes (from snippet-extractor): + 0 — success, at least some snippets extracted + 1 — no snippet requests found + 2 — all requests failed (nothing extracted) + 3 — configuration error + """ + extractor = extractor or DEFAULT_EXTRACTOR + cmd: list[str] = [ + str(extractor), + "--bundle", str(bundle_dir), + "--round", str(round_number), + ] + if prefer_workdir: + cmd.append("--prefer-workdir") + + result = subprocess.run(cmd, capture_output=True, text=True) + + round_dir = bundle_dir / "analysis" / "snippets" / f"round_{round_number}" + files: list[Path] = [] + if round_dir.is_dir(): + for sub in sorted(round_dir.rglob("*")): + if sub.is_file() and sub.suffix in (".txt", ".log"): + files.append(sub) + + return result.returncode, files + + +def format_for_prompt(bundle_dir: Path, files: list[Path]) -> str: + """Render extracted snippet files as a single string for the next user message.""" + parts: list[str] = ["## Extracted Snippets", ""] + for path in files: + try: + rel = path.relative_to(bundle_dir) + except ValueError: + rel = path + try: + content = path.read_text(errors="replace") + except OSError as exc: + content = f"" + parts.append(f"### `{rel}`") + parts.append("```") + parts.append(content) + parts.append("```") + parts.append("") + return "\n".join(parts) + + +def load_manifest(bundle_dir: Path, round_number: int) -> dict: + """Optional: read the round's manifest.json for audit/debug.""" + manifest = bundle_dir / "analysis" / "snippets" / f"round_{round_number}" / "manifest.json" + if not manifest.is_file(): + return {} + try: + return json.loads(manifest.read_text()) + except (OSError, json.JSONDecodeError): + return {} diff --git a/scripts/generator/dportsv3/agent/tool_loop.py b/scripts/generator/dportsv3/agent/tool_loop.py new file mode 100644 index 00000000000..5a3308b48ee --- /dev/null +++ b/scripts/generator/dportsv3/agent/tool_loop.py @@ -0,0 +1,135 @@ +"""Multi-turn LLM-with-tools driver. + +One call to ``run(...)`` is a single conversation with the LLM: +- Send messages + tool schemas +- If the LLM emitted ``tool_calls``, dispatch each, append the results + as ``tool`` messages, and re-call +- Stop when the LLM returns text-only (no tool calls) or when + ``max_turns`` is hit + +The caller (``patch.run`` in step 4) handles attempt-level retries +with fresh failure context; this driver is one inner attempt. +""" + +from __future__ import annotations + +import json +import logging + +from . import llm, tools +from .llm import Response, Usage + +log = logging.getLogger(__name__) + + +def _assistant_message_from(response: Response) -> dict: + """Reconstruct the assistant message dict that produced ``response``. + + Needed so the next LLM call sees the tool calls the model made on + the previous turn (otherwise it has amnesia about its own request). + Thinking-mode providers (DeepSeek v4-*, some OpenAI-compat relays) + additionally require ``reasoning_content`` to be echoed back, or + the next request fails with HTTP 400. + """ + msg: dict = {"role": "assistant", "content": response.text or ""} + if response.tool_calls: + msg["tool_calls"] = [ + { + "id": tc.id, + "type": "function", + "function": { + "name": tc.name, + "arguments": json.dumps(tc.arguments or {}), + }, + } + for tc in response.tool_calls + ] + if response.reasoning_content: + msg["reasoning_content"] = response.reasoning_content + return msg + + +def run( + messages: list[dict], + *, + model: str, + env: str, + api_base: str | None = None, + api_key: str | None = None, + custom_llm_provider: str | None = None, + timeout: int = 120, + max_turns: int = 12, + max_tokens: int = 0, +) -> tuple[Response, Usage]: + """Drive the LLM through tool calls until it returns text-only. + + ``messages`` is mutated to include each assistant + tool turn for + the duration of the loop. ``env`` is the dev-env name; every tool + call is bound to it. + + Returns the final text-only ``Response`` and the cumulative + ``Usage`` across all turns. + + Two safety caps: + - ``max_turns``: stop after this many LLM round-trips even if the + model keeps calling tools. Default 12. + - ``max_tokens``: stop when cumulative usage reaches this many + tokens. 0 (the default) disables the check — the caller is + expected to pass the remaining attempt-level budget when one + exists. + """ + total = Usage() + tool_schemas = tools.schemas() + final: Response | None = None + + for turn in range(1, max_turns + 1): + if max_tokens and total.total_tokens >= max_tokens: + log.warning( + "tool_loop: token budget exhausted on turn %d (%d >= %d)", + turn, total.total_tokens, max_tokens, + ) + return (final if final is not None else Response(text="")), total + + response = llm.complete( + messages, + model=model, + tools=tool_schemas, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + ) + total.add(response.usage) + + if not response.tool_calls: + log.debug("tool_loop: turn %d returned text-only, stopping", turn) + return response, total + + log.debug( + "tool_loop: turn %d issued %d tool call(s): %s", + turn, + len(response.tool_calls), + [tc.name for tc in response.tool_calls], + ) + + # Echo the assistant's tool-call message back into history so + # the model has continuity on the next turn. + messages.append(_assistant_message_from(response)) + + for call in response.tool_calls: + result = tools.dispatch(call.name, call.arguments, env=env) + messages.append( + { + "role": "tool", + "tool_call_id": call.id, + "name": call.name, + "content": json.dumps(result), + } + ) + final = response + + + log.warning( + "tool_loop: hit max_turns=%d without a text-only response", max_turns + ) + return (final if final is not None else Response(text="")), total diff --git a/scripts/generator/dportsv3/agent/tools.py b/scripts/generator/dportsv3/agent/tools.py new file mode 100644 index 00000000000..d4d957c70c9 --- /dev/null +++ b/scripts/generator/dportsv3/agent/tools.py @@ -0,0 +1,186 @@ +"""Tool registry for the patch agent. + +Each entry maps an OpenAI-style tool name to (a worker function, a JSON +schema). The schemas are what the LLM sees; the ``env`` argument is +bound by the caller (``patch.run``) and is not exposed to the LLM. + +``dispatch(name, arguments, env)`` is the single entry point used by +``tool_loop``: it looks up the function, calls it with ``env`` plus +the LLM-supplied arguments, and returns a result dict. Any exception +from the worker is caught and surfaced as +``{"ok": False, "error": "..."}`` so the LLM can recover on the next +turn rather than aborting the attempt. + +Schemas are hand-written (not auto-derived from signatures) because +``description`` strings materially affect tool-selection quality. +""" + +from __future__ import annotations + +import inspect +import json +import traceback +from typing import Callable + +from . import worker + + +# ----------------------------------------------------------------------------- +# JSON schemas (OpenAI tool format) +# ----------------------------------------------------------------------------- + +_STR = {"type": "string"} +_INT = {"type": "integer"} + + +def _tool(name: str, desc: str, props: dict | None = None, required: list[str] | None = None) -> dict: + return { + "type": "function", + "function": { + "name": name, + "description": desc, + "parameters": { + "type": "object", + "properties": props or {}, + "required": required or [], + }, + }, + } + + +_TOOLS: list[dict] = [ + _tool("env_verify", + "Confirm the dev-env is ready. Call first."), + _tool("list_dir", + "List a directory's entries in the writable overlay.", + {"path": _STR, "max_entries": _INT}, ["path"]), + _tool("get_file", + "Read a file. Returns encoding=text (UTF-8) or encoding=base64 (binary). " + "Use sha256 from this result in put_file's expected_sha256 to guard stale writes.", + {"path": _STR}, ["path"]), + _tool("put_file", + "Write a file. encoding='text' (UTF-8, default) or 'base64' (binary). " + "expected_sha256 is an optimistic lock — pass the sha256 from a prior get_file.", + {"path": _STR, "content": _STR, + "encoding": {"type": "string", "enum": ["text", "base64"]}, + "expected_sha256": _STR}, + ["path", "content"]), + _tool("emit_diff", + "Working-tree diff for ports// in DeltaPorts (read-only).", + {"origin": _STR, "relpath": _STR}, ["origin", "relpath"]), + _tool("grep", + "Recursive POSIX grep -rn over the writable overlay. " + "ok=True with empty matches just means 'no matches' (not an error).", + {"pattern": _STR, "path": _STR, "include": _STR, "max_bytes": _INT}, + ["pattern", "path"]), + _tool("materialize_dports", + "Propagate DeltaPorts edits into the buildable DPorts tree for one origin. " + "Call after put_file/install_patches edits and before extract/dsynth_build.", + {"origin": _STR}, ["origin"]), + _tool("extract", + "Run `make extract` for a port (after materialize_dports). Returns wrkdir + wrksrc.", + {"origin": _STR}, ["origin"]), + _tool("dupe", + "Snapshot a WRKSRC file with a .orig backup so genpatch can later diff against it.", + {"path": _STR}, ["path"]), + _tool("genpatch", + "Produce a unified diff for a duped+edited file. Output: /work/genpatch-out/patch-*.", + {"path": _STR}, ["path"]), + _tool("install_patches", + "Copy patches from /work/genpatch-out/ into DeltaPorts/ports//dragonfly/. " + "Then call materialize_dports.", + {"origin": _STR, "patches": {"type": "array", "items": {"type": "string"}}}, + ["origin"]), + _tool("dsynth_build", + "Run dsynth -S -y build . rebuild_ok=true means rc==0. " + "On failure, call dsynth_log(origin) — the actual build error is in the per-port log, " + "not in this tool's stdout_tail.", + {"origin": _STR}, ["origin"]), + _tool("dsynth_log", + "Read the tail of dsynth's per-port build log " + "(/work/dsynth/logs/.log). Call after dsynth_build failure.", + {"origin": _STR, "tail_lines": _INT}, ["origin"]), +] + + +# Map tool name → callable in worker module. +_HANDLERS: dict[str, Callable] = { + spec["function"]["name"]: getattr(worker, spec["function"]["name"]) + for spec in _TOOLS +} + + +# ----------------------------------------------------------------------------- +# Registry accessors +# ----------------------------------------------------------------------------- + + +def schemas() -> list[dict]: + """Return the OpenAI-format tool list to pass to litellm.""" + return list(_TOOLS) + + +def names() -> list[str]: + return [spec["function"]["name"] for spec in _TOOLS] + + +# ----------------------------------------------------------------------------- +# Dispatch +# ----------------------------------------------------------------------------- + + +def dispatch(name: str, arguments: dict | None, *, env: str) -> dict: + """Invoke the tool ``name`` with ``arguments`` (env bound by caller). + + Worker exceptions are caught and surfaced as + ``{"ok": False, "error": "..."}`` so the LLM can recover on its + next turn rather than aborting the attempt. Errors that signal a + bug (wrong tool name, missing required arg) are surfaced the same + way — never raised — for the same reason. + """ + handler = _HANDLERS.get(name) + if handler is None: + return {"ok": False, "error": f"unknown tool: {name!r}"} + + args = arguments or {} + if not isinstance(args, dict): + return {"ok": False, "error": f"arguments must be a JSON object, got {type(args).__name__}"} + + # Filter to declared kwargs only; reject required-but-missing. + sig = inspect.signature(handler) + params = list(sig.parameters.values()) + if not params or params[0].name != "env": + return {"ok": False, "error": f"tool {name} has no env parameter (bug)"} + + accepted = {p.name for p in params[1:]} + rejected = [k for k in args if k not in accepted] + if rejected: + return { + "ok": False, + "error": f"tool {name}: unexpected argument(s): {rejected}", + } + required = { + p.name for p in params[1:] + if p.default is inspect.Parameter.empty and p.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ) + } + missing = [k for k in required if k not in args] + if missing: + return {"ok": False, "error": f"tool {name}: missing required argument(s): {missing}"} + + try: + result = handler(env, **args) + except Exception as exc: # noqa: BLE001 — intentional broad catch; surface to LLM + return { + "ok": False, + "error": f"{type(exc).__name__}: {exc}", + "traceback": traceback.format_exc(limit=4), + } + + # Workers already return {ok: bool, ...} or raise; if a worker returned + # something else (e.g. dict without 'ok'), pass through unchanged. + if isinstance(result, dict) and "ok" not in result: + result = {"ok": True, **result} + return result diff --git a/scripts/generator/dportsv3/agent/triage.py b/scripts/generator/dportsv3/agent/triage.py new file mode 100644 index 00000000000..3e0b1852e32 --- /dev/null +++ b/scripts/generator/dportsv3/agent/triage.py @@ -0,0 +1,135 @@ +"""Triage flow — single-turn LLM call with snippet rounds folded in-process. + +The flow: + + for round in range(max_snippet_rounds): + call LLM with [system, user (+ accumulated snippets)] + write the response text to the bundle (so snippet-extractor can read it) + if response has no `## Snippet Requests` section: stop + run snippet-extractor for this round + format the extracted snippets, append to the next user turn + return classification, confidence, response_text, usage + +The bundle's ``analysis/snippets/round_N/`` files appear as the rounds +happen — no cross-job re-enqueue traffic. +""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from pathlib import Path + +from . import llm, prompts, snippets + + +@dataclass +class TriageResult: + text: str # final response text + classification: str + confidence: str + snippet_rounds: int = 0 + usage: llm.Usage = field(default_factory=llm.Usage) + + +_CLASSIFICATION_RE = re.compile(r"^##\s*Classification\s*\n+([^\n#]+)", re.MULTILINE) +_CONFIDENCE_RE = re.compile(r"^##\s*Confidence\s*\n+([^\n#]+)", re.MULTILINE) +_SNIPPET_SECTION_RE = re.compile( + r"^##\s*Snippet Requests\s*\n", re.MULTILINE +) + + +def _parse(text: str) -> tuple[str, str]: + classification = "" + confidence = "" + if m := _CLASSIFICATION_RE.search(text): + classification = m.group(1).strip().lower() + if m := _CONFIDENCE_RE.search(text): + confidence = m.group(1).strip().lower() + return classification, confidence + + +def _has_snippet_requests(text: str) -> bool: + return bool(_SNIPPET_SECTION_RE.search(text)) + + +def _write_intermediate_triage(bundle_dir: Path, text: str) -> None: + """Write ``analysis/triage.md`` so snippet-extractor can read requests from it.""" + out = bundle_dir / "analysis" / "triage.md" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text.rstrip() + "\n") + + +def run( + payload: str, + *, + bundle_dir: Path, + model: str, + api_base: str | None = None, + api_key: str | None = None, + custom_llm_provider: str | None = None, + timeout: int = 120, + max_snippet_rounds: int | None = None, +) -> TriageResult: + """Run the triage flow end-to-end for one bundle. + + ``payload`` is the markdown prompt produced by + ``agent-queue-runner.build_triage_payload``. + + ``bundle_dir`` is the live bundle directory; we write the intermediate + triage.md after each LLM round so snippet-extractor can parse requests. + """ + if max_snippet_rounds is None: + max_snippet_rounds = int(os.environ.get("DP_HARNESS_MAX_SNIPPET_ROUNDS", "5")) + + messages: list[dict] = [ + {"role": "system", "content": prompts.TRIAGE_SYSTEM}, + {"role": "user", "content": payload}, + ] + + total_usage = llm.Usage() + response_text = "" + snippet_round = 0 + + while True: + response = llm.complete( + messages, + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + ) + total_usage.add(response.usage) + response_text = response.text + + _write_intermediate_triage(bundle_dir, response_text) + + if snippet_round >= max_snippet_rounds: + break + if not _has_snippet_requests(response_text): + break + + snippet_round += 1 + rc, files = snippets.extract_round(bundle_dir, snippet_round) + if rc != 0 or not files: + # extractor failed or produced nothing usable; stop loop + break + + messages.append({"role": "assistant", "content": response_text}) + messages.append( + { + "role": "user", + "content": snippets.format_for_prompt(bundle_dir, files), + } + ) + + classification, confidence = _parse(response_text) + return TriageResult( + text=response_text, + classification=classification, + confidence=confidence, + snippet_rounds=snippet_round, + usage=total_usage, + ) diff --git a/scripts/generator/dportsv3/agent/worker.py b/scripts/generator/dportsv3/agent/worker.py new file mode 100644 index 00000000000..59bfb5f7321 --- /dev/null +++ b/scripts/generator/dportsv3/agent/worker.py @@ -0,0 +1,676 @@ +"""Worker tool functions for the agent harness, on top of dev-env primitives. + +Every function takes the dev-env name as its first argument. Filesystem +operations work host-side on the env's writable overlay +(``env_dir/writable/...``); commands that must run inside the chroot +shell out to ``dportsv3 dev-env exec``. + +The agent sees chroot-absolute paths like ``/work/DeltaPorts/ports/...``; +those translate to ``/work/DeltaPorts/ports/...`` on +the host. Paths outside ``/work/`` are rejected — the agent has no +business writing to ``/etc`` or anywhere else in the chroot. + +No git operations. The dev-env's writable overlay is the workspace; +dirty edits stay dirty. ``emit_diff`` runs ``git diff`` for audit but +never commits. + +Design note: we drive dev-env via its public CLI (``python -m dportsv3 +dev-env ...``) rather than importing ``dports_dev_env`` directly. The +CLI is the contract that's stable across dev-env refactors; the +``EnvironmentStore`` internals are not. Subprocess overhead is bounded +(``env_paths`` is cached per-process; ``env_verify`` runs once per +attempt; chroot-bound ops are unavoidably subprocess anyway). +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path + +# How to invoke `dportsv3`. The wrapper script at the repo root is the +# only entry point that knows how to route the `dev-env` subcommand — +# it dispatches to a separate venv. Using `python -m dportsv3` would +# bypass that routing and fail. +# +# Resolution order: +# 1. DPORTSV3_CMD env var (whitespace-split) — for tests / overrides +# 2. /dportsv3 sibling lookup relative to this file +# 3. `dportsv3` on PATH (via shutil.which) +# Otherwise, raise at first use. +def _resolve_dportsv3_cmd() -> list[str]: + override = os.environ.get("DPORTSV3_CMD") + if override: + return override.split() + sibling = Path(__file__).resolve().parents[4] / "dportsv3" + if sibling.is_file(): + return [str(sibling)] + found = shutil.which("dportsv3") + if found: + return [found] + raise RuntimeError( + "could not locate dportsv3 wrapper " + "(set DPORTSV3_CMD, put dportsv3 on PATH, or run from the repo)" + ) + + +_DPORTSV3_CMD: list[str] | None = None + + +def _dportsv3_cmd() -> list[str]: + global _DPORTSV3_CMD + if _DPORTSV3_CMD is None: + _DPORTSV3_CMD = _resolve_dportsv3_cmd() + return _DPORTSV3_CMD + + +# ----------------------------------------------------------------------------- +# Path resolution + dev-env state +# ----------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class EnvPaths: + """Host-side paths for a dev-env.""" + env_dir: Path + writable: Path + + @property + def deltaports(self) -> Path: + return self.writable / "work" / "DeltaPorts" + + +def _run_dportsv3(*args: str) -> subprocess.CompletedProcess: + return subprocess.run( + [*_dportsv3_cmd(), *args], + capture_output=True, text=True, check=False, + ) + + +# Cap per-stream output the LLM sees. Build errors live at the tail of +# the log; we preserve the LAST bytes when truncating, not the first. +_MAX_STREAM_BYTES = 32_768 + + +def _tail(s: str, max_bytes: int = _MAX_STREAM_BYTES) -> tuple[str, bool]: + if len(s) <= max_bytes: + return s, False + return "…[truncated]…\n" + s[-max_bytes:], True + + +def _exec_result(rc: int, stdout: str, stderr: str, **extra) -> dict: + """Build a uniform tool-result dict from a subprocess outcome. + + LLM-facing tools return this shape so the harness/LLM can inspect + the failure rather than recovering from an opaque exception. + """ + out, out_trunc = _tail(stdout) + err, err_trunc = _tail(stderr) + return { + "ok": rc == 0, + "rc": rc, + "stdout_tail": out, + "stderr_tail": err, + "stdout_truncated": out_trunc, + "stderr_truncated": err_trunc, + **extra, + } + + +@lru_cache(maxsize=32) +def env_paths(env: str) -> EnvPaths: + """Resolve host-side paths for ``env``. Cached per-process. + + Each ``dportsv3 dev-env path`` invocation costs a few hundred ms; + caching means repeated tool calls during one patch attempt pay it + only once. The env_dir for a given name is immutable for the + lifetime of that env, so caching is safe. + """ + p1 = _run_dportsv3("dev-env", "path", env) + if p1.returncode != 0: + raise RuntimeError(f"dev-env path failed: {(p1.stderr or p1.stdout).strip()}") + p2 = _run_dportsv3("dev-env", "path", env, "--writable") + if p2.returncode != 0: + raise RuntimeError(f"dev-env path --writable failed: {(p2.stderr or p2.stdout).strip()}") + return EnvPaths(env_dir=Path(p1.stdout.strip()), writable=Path(p2.stdout.strip())) + + +def env_verify(env: str) -> dict: + """Return the env's state dict; raise if not usable. + + Wraps ``dportsv3 dev-env status NAME``. Fails only when the env is + missing or in a non-``ready`` state (``creating``, ``destroying``, + ``failed``). A ``root_mounted: false`` env is still usable: host-side + tool ops operate on the writable overlay directly, and + ``dportsv3 dev-env exec`` auto-mounts on demand for chroot ops. + """ + p = _run_dportsv3("dev-env", "status", env) + if p.returncode != 0: + raise RuntimeError(f"dev-env status failed: {(p.stderr or p.stdout).strip()}") + info = json.loads(p.stdout.strip()) + if info.get("status") != "ready": + raise RuntimeError(f"env {env} not ready: status={info.get('status')}") + return info + + +def _resolve_chroot_path(paths: EnvPaths, chroot_path: str) -> Path: + """Translate an in-chroot absolute path to its host-side location. + + Only paths under ``/work/`` are allowed; ``..`` traversal escaping + the writable overlay is rejected via ``Path.relative_to`` after + realpath resolution. + """ + if not (chroot_path == "/work" or chroot_path.startswith("/work/")): + raise ValueError(f"path must be under /work (got {chroot_path!r})") + rel = chroot_path.lstrip("/") # "work/..." + resolved = (paths.writable / rel).resolve() + writable_root = paths.writable.resolve() + try: + resolved.relative_to(writable_root) + except ValueError as exc: + raise ValueError(f"path escapes writable overlay: {chroot_path!r}") from exc + return resolved + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +# ----------------------------------------------------------------------------- +# Tool functions (host-side, simple set — chroot-bound ones land in step 2c) +# ----------------------------------------------------------------------------- + + +def list_dir(env: str, path: str, *, max_entries: int = 200) -> dict: + """List the contents of a directory in the env's writable overlay. + + Returns up to ``max_entries`` entries, each with ``name``, ``kind`` + (file/dir/symlink/other), and ``size`` (for files). Useful when + you don't know what's inside a directory or you need to find a + config.log / patch / dragonfly-overlay file without a recursive + grep. + """ + paths = env_paths(env) + host = _resolve_chroot_path(paths, path) + if not host.exists(): + return {"ok": False, "error": f"no such path: {path}", "kind": "missing", "path": path} + if not host.is_dir(): + return {"ok": False, "error": f"not a directory: {path}", "kind": "not_a_directory", "path": path} + entries: list[dict] = [] + truncated = False + for i, child in enumerate(sorted(host.iterdir())): + if i >= max_entries: + truncated = True + break + if child.is_symlink(): + kind = "symlink" + size = 0 + elif child.is_dir(): + kind = "dir" + size = 0 + elif child.is_file(): + kind = "file" + try: + size = child.stat().st_size + except OSError: + size = 0 + else: + kind = "other" + size = 0 + entries.append({"name": child.name, "kind": kind, "size": size}) + return { + "ok": True, + "path": path, + "entries": entries, + "truncated": truncated, + "total_returned": len(entries), + } + + +def get_file(env: str, path: str) -> dict: + """Read ``path`` from the env's writable overlay. + + Returns ``encoding='text'`` with raw UTF-8 ``content`` when the file + decodes cleanly and contains no NULs (the common case for source, + Makefiles, patches, docs). Falls back to ``encoding='base64'`` for + binary content. sha256 is computed over the **bytes**, so the round + trip via ``put_file(expected_sha256=...)`` works regardless of + encoding. + + Distinct error envelopes for "doesn't exist" vs "is a directory" + vs "read failed" — the agent can react usefully rather than guessing. + """ + paths = env_paths(env) + host = _resolve_chroot_path(paths, path) + if not host.exists(): + return { + "ok": False, + "error": f"no such file: {path}", + "kind": "missing", + "path": path, + } + if host.is_dir(): + return { + "ok": False, + "error": f"path is a directory; use list_dir or grep instead: {path}", + "kind": "is_directory", + "path": path, + } + if not host.is_file(): + return { + "ok": False, + "error": f"not a regular file: {path}", + "kind": "not_a_regular_file", + "path": path, + } + data = host.read_bytes() + + text: str | None = None + if b"\x00" not in data: + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + text = None + + if text is not None: + return { + "path": path, + "encoding": "text", + "content": text, + "sha256": _sha256(data), + "size": len(data), + } + return { + "path": path, + "encoding": "base64", + "content": base64.b64encode(data).decode("ascii"), + "sha256": _sha256(data), + "size": len(data), + } + + +def put_file( + env: str, + path: str, + content: str, + *, + encoding: str = "text", + expected_sha256: str | None = None, +) -> dict: + """Write ``content`` to ``path`` in the env's writable overlay. + + ``encoding`` is ``"text"`` (UTF-8) or ``"base64"``. If + ``expected_sha256`` is given, the existing file's sha256 must match + or the write fails (optimistic lock against the LLM editing stale + content). File mode is preserved for existing files. + """ + paths = env_paths(env) + host = _resolve_chroot_path(paths, path) + + if encoding == "base64": + data = base64.b64decode(content.encode("ascii")) + elif encoding == "text": + data = content.encode("utf-8") + else: + raise ValueError(f"unknown encoding: {encoding}") + + if expected_sha256 is not None: + if not host.is_file(): + raise RuntimeError( + f"put_file: expected_sha256 given but file does not exist: {path}" + ) + current = _sha256(host.read_bytes()) + if current != expected_sha256: + raise RuntimeError( + f"put_file: sha256 mismatch on {path} " + f"(expected {expected_sha256[:12]}…, got {current[:12]}…)" + ) + + mode = host.stat().st_mode & 0o777 if host.is_file() else None + host.parent.mkdir(parents=True, exist_ok=True) + host.write_bytes(data) + if mode is not None: + host.chmod(mode) + return {"path": path, "sha256": _sha256(data), "size": len(data)} + + +def emit_diff(env: str, origin: str, relpath: str) -> dict: + """Return the working-tree diff for ``ports//``. + + Pure read — never commits, never stages. ``diff`` is empty when + the file hasn't been modified vs HEAD. ``ok`` is False only on + git invocation errors (rc >= 128), not on "no changes" (rc=0). + """ + paths = env_paths(env) + rel = f"ports/{origin}/{relpath}" + p = subprocess.run( + ["git", "-C", str(paths.deltaports), "diff", "--", rel], + capture_output=True, text=True, check=False, + ) + diff_text, diff_trunc = _tail(p.stdout, max_bytes=_MAX_STREAM_BYTES * 2) + return _exec_result( + p.returncode, "", p.stderr, + origin=origin, + relpath=relpath, + diff=diff_text, + diff_truncated=diff_trunc, + ) + + +def grep( + env: str, + pattern: str, + path: str, + *, + include: str | None = None, + max_bytes: int = 8192, +) -> dict: + """Recursive grep over the env's writable overlay. + + Uses POSIX ``grep -rn`` (always present on dfly) — not ripgrep, + which isn't packaged for DragonFly. ``ok=True`` whenever grep + ran without error, even when there were zero matches (rc=1 from + grep is "no matches", not a failure). ``ok=False`` only when + grep itself crashed (rc>=2) or the path is invalid. + """ + paths = env_paths(env) + host = _resolve_chroot_path(paths, path) + if not host.exists(): + return { + "ok": False, + "error": f"no such path: {path}", + "pattern": pattern, + "matches": "", + "match_count": 0, + } + # -E extended regex (closest to rg's default), -r recursive, -n line numbers, + # -I skip binary files, --include= glob (gnu/bsd grep both support it). + cmd = ["grep", "-rnIE"] + if include: + cmd.append(f"--include={include}") + cmd.extend(["--", pattern, str(host)]) + try: + p = subprocess.run(cmd, capture_output=True, text=True, check=False) + except OSError as exc: + return { + "ok": False, + "error": f"grep invocation failed: {exc}", + "pattern": pattern, + "matches": "", + "match_count": 0, + } + # grep exit codes: 0 = matches, 1 = no matches, ≥2 = error + if p.returncode >= 2: + return { + "ok": False, + "error": (p.stderr.strip() or f"grep exited with rc={p.returncode}"), + "pattern": pattern, + "matches": p.stdout, + "match_count": 0, + } + output = p.stdout + match_count = output.count("\n") if output else 0 + truncated = False + if len(output) > max_bytes: + output = output[:max_bytes] + truncated = True + return { + "ok": True, + "root": str(host), + "pattern": pattern, + "matches": output, + "match_count": match_count, + "truncated": truncated, + "max_bytes": max_bytes, + } + + +# ----------------------------------------------------------------------------- +# Chroot-bound tool functions — all shell out via `dportsv3 dev-env exec` +# ----------------------------------------------------------------------------- + + +def _exec( + env: str, + *argv: str, + cwd: str = "/work/DeltaPorts", + input_text: str | None = None, + timeout: int | None = None, +) -> subprocess.CompletedProcess: + """Run ``argv`` inside the dev-env chroot, return CompletedProcess. + + ``dev-env exec`` auto-mounts the env root on demand. Stdout/stderr + are captured; callers decide how to surface them to the LLM. + + ``input_text`` is fed to the subprocess's stdin (useful for tools + that prompt — e.g. dsynth's "rebuild local repository? [Y/n]"). If + None, stdin is /dev/null (so an unexpected prompt fails fast rather + than hanging). + """ + return subprocess.run( + [*_dportsv3_cmd(), "dev-env", "exec", "--quiet", "--cwd", cwd, env, "--", *argv], + capture_output=True, + text=True, + check=False, + input=input_text if input_text is not None else "", + timeout=timeout, + ) + + +def materialize_dports(env: str, origin: str) -> dict: + """Regenerate the DPorts tree for ``origin`` using ``reapply`` inside the env. + + Returns a result dict (``ok``, ``rc``, ``stdout_tail``, + ``stderr_tail``, etc.). The LLM inspects ``ok`` and the tails + to decide what to do — no exceptions bubble up for build failures. + """ + p = _exec(env, "reapply", origin) + return _exec_result(p.returncode, p.stdout, p.stderr, origin=origin) + + +PORTSDIR = "/work/DPorts" +WRKDIRPREFIX = "/work/obj" + + +def _make_vars() -> list[str]: + """Common make variable overrides for ports operations inside the env. + + - ``PORTSDIR``: dports tree lives at /work/DPorts, not /usr/dports. + - ``WRKDIRPREFIX``: bsd.port.mk defaults to /usr/obj/dports which is + read-only in the chroot; point it at writable /work/obj. + - ``BATCH=yes``: skip config dialogs (the agent has no terminal). + """ + return [ + f"PORTSDIR={PORTSDIR}", + f"WRKDIRPREFIX={WRKDIRPREFIX}", + "BATCH=yes", + ] + + +def extract(env: str, origin: str) -> dict: + """Run ``make extract`` for ``origin`` in DPorts; return WRKDIR + WRKSRC on success. + + Sets ``PORTSDIR`` and ``WRKDIRPREFIX`` because the dev-env layout + doesn't match the conventional ``/usr/dports`` + ``/usr/obj/dports`` + paths bsd.port.mk expects. On failure (``ok`` False), ``wrkdir`` + and ``wrksrc`` are empty. + """ + port_dir = f"{PORTSDIR}/{origin}" + make_vars = _make_vars() + p = _exec(env, "make", "-C", port_dir, *make_vars, "extract", cwd=port_dir) + if p.returncode != 0: + return _exec_result(p.returncode, p.stdout, p.stderr, + origin=origin, wrkdir="", wrksrc="") + q = _exec(env, "make", "-C", port_dir, *make_vars, + "-V", "WRKDIR", "-V", "WRKSRC", cwd=port_dir) + if q.returncode != 0: + return _exec_result(q.returncode, q.stdout, q.stderr, + origin=origin, wrkdir="", wrksrc="", + extract_step="query-wrkdir") + lines = [line.strip() for line in q.stdout.splitlines() if line.strip()] + wrkdir = lines[0] if len(lines) > 0 else "" + wrksrc = lines[1] if len(lines) > 1 else "" + return _exec_result(0, p.stdout, p.stderr, + origin=origin, wrkdir=wrkdir, wrksrc=wrksrc) + + +def dupe(env: str, path: str) -> dict: + """Run ``dupe PATH`` inside the chroot (clones source file with .orig backup). + + Returns a result dict; LLM inspects ``ok``. + """ + p = _exec(env, "dupe", path) + return _exec_result(p.returncode, p.stdout, p.stderr, path=path) + + +def genpatch(env: str, path: str) -> dict: + """Run ``genpatch PATH`` inside the chroot; list generated patch files. + + Always lists ``patch-*`` files from ``/work/genpatch-out/`` regardless + of rc (genpatch may produce partial output on failure). LLM inspects + ``ok`` to decide whether the patches are trustworthy. + """ + p = _exec(env, "genpatch", path) + paths = env_paths(env) + genpatch_out = paths.writable / "work" / "genpatch-out" + patches: list[str] = [] + if genpatch_out.is_dir(): + for f in sorted(genpatch_out.iterdir()): + if f.is_file() and f.name.startswith("patch-"): + patches.append(f.name) + return _exec_result( + p.returncode, p.stdout, p.stderr, + path=path, + output_dir=str(genpatch_out), + patches=patches, + ) + + +def install_patches(env: str, origin: str, patches: list[str] | None = None) -> dict: + """Copy patches from ``/work/genpatch-out/`` into DeltaPorts overlay. + + Destination is + ``/work/DeltaPorts/ports//dragonfly/``. + Host-side file copy; no chroot exec needed since both source and + destination are in the writable overlay. If ``patches`` is None, + every ``patch-*`` file in ``genpatch-out/`` is installed. + """ + paths = env_paths(env) + src = paths.writable / "work" / "genpatch-out" + dst = paths.deltaports / "ports" / origin / "dragonfly" + if not src.is_dir(): + raise FileNotFoundError(f"genpatch output dir does not exist: {src}") + if patches is None: + candidates = [f for f in sorted(src.iterdir()) if f.is_file() and f.name.startswith("patch-")] + else: + candidates = [src / name for name in patches] + missing = [str(p) for p in candidates if not p.is_file()] + if missing: + raise FileNotFoundError(f"missing patches: {missing}") + dst.mkdir(parents=True, exist_ok=True) + installed: list[str] = [] + for f in candidates: + target = dst / f.name + shutil.copy2(f, target) + installed.append(str(target.relative_to(paths.deltaports))) + return {"origin": origin, "destination": str(dst), "installed": installed} + + +DSYNTH_LOGS_DIR = "/work/dsynth/logs" + + +def _dsynth_log_path(origin: str) -> str: + """Where dsynth writes the per-port build log inside the chroot. + + dsynth replaces '/' in the origin with '___' for its log filenames + (per the dsynth source convention). + """ + return f"{DSYNTH_LOGS_DIR}/{origin.replace('/', '___')}.log" + + +def dsynth_build(env: str, origin: str) -> dict: + """Run ``dsynth build `` inside the chroot. + + Invokes dsynth directly (not via the ``dbuild`` helper) with: + - ``-S`` to disable the ncurses TUI (otherwise stdout is curses + escape codes the LLM can't parse) + - ``-y`` to assume-yes on all prompts (no stdin gymnastics) + + The result dict carries ``log_hint``: the in-chroot path to the + per-port build log. On failure, the agent should call + ``dsynth_log(origin)`` to read it — stdout/stderr_tail here only + capture the wrapper output, not the actual build error. + + The ``dbuild`` helper at + ``scripts/tools/dev-env/dports_dev_env/helpers.py:62-90`` is + intentionally not used: it doesn't pass -S/-y because humans use + it interactively. + """ + # Read DPORTS_DSYNTH_PROFILE inside the chroot (set by dev-env's + # build_env_dict at helpers.py:113) and invoke dsynth directly. + # Using sh -c so we can reference the env var on the chroot side. + cmd = ( + 'dsynth -S -y -p "$DPORTS_DSYNTH_PROFILE" build "$1"' + ) + p = _exec(env, "/bin/sh", "-c", cmd, "_", origin) + log_hint = _dsynth_log_path(origin) + return _exec_result( + p.returncode, p.stdout, p.stderr, + origin=origin, + rebuild_ok=p.returncode == 0, + log_hint=log_hint, + ) + + +def dsynth_log(env: str, origin: str, tail_lines: int = 200) -> dict: + """Read the tail of dsynth's per-port build log. + + Call this when ``dsynth_build`` returned ``rebuild_ok=false``; + the actual error message lives here, not in dsynth_build's + stdout (which is just wrapper output). + + Returns the last ``tail_lines`` lines from + ``/work/dsynth/logs/.log``. + """ + paths = env_paths(env) + log_path = paths.writable / "work" / "dsynth" / "logs" / f"{origin.replace('/', '___')}.log" + if not log_path.is_file(): + return { + "ok": False, + "error": f"no log at {log_path}", + "origin": origin, + "log_path": str(log_path), + "tail": "", + } + try: + text = log_path.read_text(errors="replace") + except OSError as exc: + return { + "ok": False, + "error": f"read failed: {exc}", + "origin": origin, + "log_path": str(log_path), + "tail": "", + } + lines = text.splitlines() + if tail_lines > 0 and len(lines) > tail_lines: + truncated = True + kept = lines[-tail_lines:] + else: + truncated = False + kept = lines + return { + "ok": True, + "origin": origin, + "log_path": str(log_path), + "tail": "\n".join(kept), + "truncated": truncated, + "total_lines": len(lines), + } diff --git a/scripts/generator/dportsv3/artifact_store.py b/scripts/generator/dportsv3/artifact_store.py new file mode 100644 index 00000000000..0b0e6e0602b --- /dev/null +++ b/scripts/generator/dportsv3/artifact_store.py @@ -0,0 +1,396 @@ +"""HTTP service that writes state.db, blobs, and full logs. + +The single writer for state.db (the central evidence + agentic metadata +store). Tracker reads it; runner writes to it via this HTTP layer. +Schema lives in ``dportsv3.db.schema`` and is shared with tracker. + +Entry points: +- ``scripts/artifact-store`` — standalone shim (no venv required) +- ``python -m dportsv3.artifact_store`` (or the venv's bin/artifact-store + console script once pyproject.toml is updated) +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import sqlite3 +import threading +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +from .db.schema import init_db as _init_state_db + +DEFAULT_BIND = "127.0.0.1" +DEFAULT_PORT = 8788 +DEFAULT_LOGS_ROOT = "/build/synth/logs" + + +def log(level: str, message: str) -> None: + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + print(f"{ts} {level:5} {message}") + + +def emit_event(conn: sqlite3.Connection, event_type: str, data: dict[str, Any]) -> None: + ts = datetime.now(timezone.utc).isoformat() + conn.execute( + "INSERT INTO events (ts, type, data_json) VALUES (?, ?, ?)", + (ts, event_type, json.dumps(data)), + ) + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def blob_path(root: Path, sha: str) -> Path: + return root / "objects" / "sha256" / sha[0:2] / sha[2:4] / sha + + +class ArtifactStore: + def __init__(self, logs_root: Path) -> None: + self.logs_root = logs_root + self.evidence_root = logs_root / "evidence" + self.blob_root = self.evidence_root / "blobstore" + self.full_logs_root = self.evidence_root / "full-logs" + self.db_path = self.evidence_root / "state.db" + self._lock = threading.Lock() + + self._ensure_dirs() + self.conn = sqlite3.connect(str(self.db_path), check_same_thread=False) + self.conn.row_factory = sqlite3.Row + _init_state_db(self.conn) + + def _ensure_dirs(self) -> None: + self.evidence_root.mkdir(parents=True, exist_ok=True) + (self.blob_root / "objects" / "sha256").mkdir(parents=True, exist_ok=True) + self.full_logs_root.mkdir(parents=True, exist_ok=True) + + def upsert_run_bundle(self, payload: dict[str, Any]) -> None: + run_id = payload.get("run_id") + profile = payload.get("profile") + bundle_id = payload.get("bundle_id") + origin = payload.get("origin") + flavor = payload.get("flavor") + ts_utc = payload.get("ts_utc") + result = payload.get("result") + target = payload.get("target") + now = datetime.now(timezone.utc).isoformat() + + with self._lock: + if run_id: + self.conn.execute( + """INSERT INTO runs (run_id, profile, target, path, ts_start, ts_end, last_seen_at) + VALUES (?, ?, ?, NULL, ?, NULL, ?) + ON CONFLICT(run_id) DO UPDATE SET + profile=excluded.profile, + target=COALESCE(excluded.target, runs.target), + last_seen_at=excluded.last_seen_at""", + (run_id, profile, target, ts_utc, now), + ) + + self.conn.execute( + """INSERT INTO bundles (bundle_id, run_id, origin, flavor, ts_utc, result, target, path, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?, NULL, ?) + ON CONFLICT(bundle_id) DO UPDATE SET + run_id=excluded.run_id, + origin=excluded.origin, + flavor=excluded.flavor, + ts_utc=excluded.ts_utc, + result=excluded.result, + target=COALESCE(excluded.target, bundles.target), + last_seen_at=excluded.last_seen_at""", + (bundle_id, run_id, origin, flavor, ts_utc, result, target, now), + ) + emit_event(self.conn, "bundle_upserted", { + "bundle_id": bundle_id, + "run_id": run_id, + "origin": origin, + "result": result, + "target": target, + }) + self.conn.commit() + + def put_blob(self, bundle_id: str, relpath: str, data: bytes, kind: str | None) -> dict[str, Any]: + sha = sha256_bytes(data) + obj_path = blob_path(self.blob_root, sha) + if not obj_path.exists(): + obj_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = obj_path.with_suffix(".tmp") + tmp_path.write_bytes(data) + tmp_path.rename(obj_path) + + now = datetime.now(timezone.utc).isoformat() + with self._lock: + self.conn.execute( + """INSERT INTO blob_objects (sha256, size, created_at) + VALUES (?, ?, ?) + ON CONFLICT(sha256) DO NOTHING""", + (sha, len(data), now), + ) + self.conn.execute( + """INSERT INTO artifact_refs (bundle_id, relpath, backend, sha256, fs_path, kind, size, created_at) + VALUES (?, ?, 'blob', ?, NULL, ?, ?, ?) + ON CONFLICT(bundle_id, relpath) DO UPDATE SET + backend='blob', sha256=excluded.sha256, fs_path=NULL, + kind=excluded.kind, size=excluded.size, created_at=excluded.created_at""", + (bundle_id, relpath, sha, kind, len(data), now), + ) + emit_event(self.conn, "artifact_put", { + "bundle_id": bundle_id, + "artifact": relpath, + "backend": "blob", + }) + self.conn.commit() + + return {"sha256": sha, "size": len(data)} + + def put_fs_ref(self, bundle_id: str, relpath: str, fs_path: str, kind: str | None) -> dict[str, Any]: + path = Path(fs_path) + size = path.stat().st_size if path.exists() else None + now = datetime.now(timezone.utc).isoformat() + with self._lock: + self.conn.execute( + """INSERT INTO artifact_refs (bundle_id, relpath, backend, sha256, fs_path, kind, size, created_at) + VALUES (?, ?, 'fs', NULL, ?, ?, ?, ?) + ON CONFLICT(bundle_id, relpath) DO UPDATE SET + backend='fs', sha256=NULL, fs_path=excluded.fs_path, + kind=excluded.kind, size=excluded.size, created_at=excluded.created_at""", + (bundle_id, relpath, fs_path, kind, size, now), + ) + emit_event(self.conn, "artifact_put", { + "bundle_id": bundle_id, + "artifact": relpath, + "backend": "fs", + }) + self.conn.commit() + + return {"size": size} + + def get_artifact(self, bundle_id: str, relpath: str) -> tuple[str, Path] | None: + row = self.conn.execute( + """SELECT backend, sha256, fs_path FROM artifact_refs + WHERE bundle_id = ? AND relpath = ?""", + (bundle_id, relpath), + ).fetchone() + if not row: + return None + if row["backend"] == "blob": + obj_path = blob_path(self.blob_root, row["sha256"]) + return "blob", obj_path + return "fs", Path(row["fs_path"]) + + def upsert_user_context(self, run_id: str, origin: str, context_text: str) -> int: + """Set or update the operator's hint text for one (run_id, origin). + + Bumps ``context_rev`` by 1 on every write so the runner's + ``process_user_context_updates`` loop can detect new input and + re-enqueue a triage retry. + + Returns the new ``context_rev``. + """ + now = datetime.now(timezone.utc).isoformat() + with self._lock: + row = self.conn.execute( + "SELECT context_rev FROM user_context WHERE run_id = ? AND origin = ?", + (run_id, origin), + ).fetchone() + if row: + new_rev = int(row["context_rev"]) + 1 + self.conn.execute( + """UPDATE user_context + SET context_text = ?, updated_at = ?, context_rev = ? + WHERE run_id = ? AND origin = ?""", + (context_text, now, new_rev, run_id, origin), + ) + else: + new_rev = 1 + self.conn.execute( + """INSERT INTO user_context + (run_id, origin, context_text, updated_at, context_rev) + VALUES (?, ?, ?, ?, ?)""", + (run_id, origin, context_text, now, new_rev), + ) + emit_event(self.conn, "user_context_updated", { + "run_id": run_id, + "origin": origin, + "context_rev": new_rev, + "updated_at": now, + }) + self.conn.commit() + return new_rev + + +class Handler(BaseHTTPRequestHandler): + server: "ArtifactStoreServer" + + def log_message(self, format: str, *args: Any) -> None: + log("HTTP", f"{self.address_string()} - {format % args}") + + def _send_json(self, data: Any, status: int = 200) -> None: + body = json.dumps(data, indent=2).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def _send_error_json(self, status: int, message: str) -> None: + self._send_json({"error": message}, status) + + def _read_json_body(self) -> dict[str, Any] | None: + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + if length <= 0: + return None + raw = self.rfile.read(length) + if not raw: + return None + try: + return json.loads(raw.decode("utf-8")) + except json.JSONDecodeError: + return None + + def do_GET(self) -> None: + parsed = urlparse(self.path) + path = parsed.path + query = parse_qs(parsed.query) + store = self.server.store + + if path == "/health": + self._send_json({ + "ok": True, + "db_path": str(store.db_path), + "blobstore_root": str(store.blob_root), + "full_logs_root": str(store.full_logs_root), + }) + return + + if path == "/v1/artifacts/get": + bundle_id = query.get("bundle_id", [None])[0] + relpath = query.get("relpath", [None])[0] + if not bundle_id or not relpath: + self._send_error_json(400, "bundle_id and relpath required") + return + result = store.get_artifact(bundle_id, relpath) + if not result: + self._send_error_json(404, "artifact not found") + return + backend, file_path = result + if not file_path.exists(): + self._send_error_json(404, "artifact file missing") + return + data = file_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/octet-stream") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + return + + self._send_error_json(404, "Not found") + + def do_POST(self) -> None: + store = self.server.store + parsed = urlparse(self.path) + path = parsed.path + + if path == "/v1/bundles/upsert": + body = self._read_json_body() + if not body: + self._send_error_json(400, "invalid JSON body") + return + bundle_id = body.get("bundle_id") + if not bundle_id: + self._send_error_json(400, "bundle_id required") + return + store.upsert_run_bundle(body) + self._send_json({"ok": True}) + return + + if path == "/v1/artifacts/put": + bundle_id = self.headers.get("X-Bundle-Id") + relpath = self.headers.get("X-Relpath") + kind = self.headers.get("X-Kind") + if not bundle_id or not relpath: + self._send_error_json(400, "X-Bundle-Id and X-Relpath required") + return + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + length = 0 + data = self.rfile.read(length) if length > 0 else b"" + if data is None: + data = b"" + result = store.put_blob(bundle_id, relpath, data, kind) + self._send_json({"ok": True, **result}) + return + + if path == "/v1/artifacts/put-fs": + body = self._read_json_body() + if not body: + self._send_error_json(400, "invalid JSON body") + return + bundle_id = body.get("bundle_id") + relpath = body.get("relpath") + fs_path = body.get("fs_path") + kind = body.get("kind") + if not bundle_id or not relpath or not fs_path: + self._send_error_json(400, "bundle_id, relpath, fs_path required") + return + result = store.put_fs_ref(bundle_id, relpath, fs_path, kind) + self._send_json({"ok": True, **result}) + return + + if path == "/v1/user-context": + body = self._read_json_body() + if not body: + self._send_error_json(400, "invalid JSON body") + return + run_id = body.get("run_id") + origin = body.get("origin") + context_text = body.get("context_text") + if not run_id or not origin or context_text is None: + self._send_error_json(400, "run_id, origin, context_text required") + return + context_text = str(context_text).strip() + if not context_text: + self._send_error_json(400, "context_text cannot be empty") + return + if len(context_text) > 8000: + self._send_error_json(400, "context_text too long (max 8000 chars)") + return + new_rev = store.upsert_user_context(run_id, origin, context_text) + self._send_json({"ok": True, "context_rev": new_rev}) + return + + self._send_error_json(404, "Not found") + + +class ArtifactStoreServer(HTTPServer): + def __init__(self, server_address: tuple[str, int], RequestHandlerClass: type, store: ArtifactStore) -> None: + super().__init__(server_address, RequestHandlerClass) + self.store = store + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Local artifact store (blobstore + metadata)") + parser.add_argument("--bind", default=DEFAULT_BIND) + parser.add_argument("--port", type=int, default=DEFAULT_PORT) + parser.add_argument("--logs-root", default=DEFAULT_LOGS_ROOT) + args = parser.parse_args(argv) + + store = ArtifactStore(Path(args.logs_root)) + server = ArtifactStoreServer((args.bind, args.port), Handler, store) + log("INFO", f"artifact-store listening on http://{args.bind}:{args.port}") + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/scripts/generator/dportsv3/cli.py b/scripts/generator/dportsv3/cli.py index 2b1465a718d..f90820ef658 100644 --- a/scripts/generator/dportsv3/cli.py +++ b/scripts/generator/dportsv3/cli.py @@ -32,6 +32,8 @@ def create_parser() -> argparse.ArgumentParser: _register_dsl_parser(subparsers) _register_migrate_parser(subparsers) _register_tracker_parser(subparsers) + _register_artifact_store_parser(subparsers) + _register_agent_queue_runner_parser(subparsers) return parser @@ -448,8 +450,8 @@ def _register_tracker_parser(subparsers: argparse._SubParsersAction) -> None: serve.add_argument( "--db", type=Path, - default=Path("tracker.db"), - help="SQLite database path", + default=None, + help="SQLite database path. If unset, uses DPORTSV3_STATE_DB env var, else $PWD/state.db", ) start = tracker_sub.add_parser("start-build", help="Create a build run") @@ -524,10 +526,68 @@ def _register_tracker_parser(subparsers: argparse._SubParsersAction) -> None: compare.add_argument("--json", action="store_true", help="Pretty JSON output") +def _register_artifact_store_parser(subparsers: argparse._SubParsersAction) -> None: + """Register artifact-store command (serves bundles into state.db). + + The subparser is a marker only; argv past this subcommand is + forwarded verbatim to ``dportsv3.artifact_store.main`` by ``main`` + below (REMAINDER nargs doesn't reliably absorb ``--flag``-style + args). + """ + subparsers.add_parser( + "artifact-store", + help="Run the artifact-store HTTP service (forwards --bind/--port/--logs-root)", + add_help=False, + ) + + +def _register_agent_queue_runner_parser( + subparsers: argparse._SubParsersAction, +) -> None: + """Register agent-queue-runner forwarder. + + The runner lives at ``scripts/agent-queue-runner`` (still a + standalone script). Forwarder ``execv``s it so operator + invocations match the rest of the dportsv3 surface. + """ + subparsers.add_parser( + "agent-queue-runner", + help="Run the agent queue runner (forwards to scripts/agent-queue-runner)", + add_help=False, + ) + + def main(argv: list[str] | None = None) -> int: """CLI entrypoint.""" + raw = list(argv) if argv is not None else sys.argv[1:] + + # artifact-store is a thin forwarder — let its own argparse handle + # --bind/--port/--logs-root rather than splitting flag parsing + # across two layers. ``argparse.REMAINDER`` is unreliable for this. + if raw and raw[0] == "artifact-store": + from dportsv3.artifact_store import main as artifact_store_main + + artifact_store_main(raw[1:]) + return 0 + + # agent-queue-runner lives at scripts/agent-queue-runner; execv to + # it so its argparse handles --queue-root / --once / etc directly. + if raw and raw[0] == "agent-queue-runner": + import os as _os + from pathlib import Path as _Path + + # Resolve scripts/agent-queue-runner relative to this package: + # parents[0]=dportsv3, parents[1]=generator, parents[2]=scripts. + runner = _Path(__file__).resolve().parents[2] / "agent-queue-runner" + if not runner.is_file(): + print(f"agent-queue-runner not found at {runner}", file=sys.stderr) + return 1 + _os.execv(str(runner), [str(runner), *raw[1:]]) + # execv never returns on success + return 0 + parser = create_parser() - args = parser.parse_args(argv) + args = parser.parse_args(raw) if not args.command: parser.print_help() diff --git a/scripts/generator/dportsv3/commands/tracker.py b/scripts/generator/dportsv3/commands/tracker.py index 1a0fd313306..3ed4a2d4c24 100644 --- a/scripts/generator/dportsv3/commands/tracker.py +++ b/scripts/generator/dportsv3/commands/tracker.py @@ -56,6 +56,25 @@ def cmd_tracker(args: Namespace) -> int: return 1 +def _resolve_state_db_path(args: Namespace) -> Path: + """Resolve the state.db path with the precedence: + 1. --db PATH (explicit operator override) + 2. DPORTSV3_STATE_DB env var + 3. $PWD/state.db (fall-back default) + + Tracker reads + writes the same file artifact-store writes. The + operator is responsible for ensuring the path matches whatever + artifact-store was started with (typically --logs-root + /build/synth/logs → /build/synth/logs/evidence/state.db). + """ + if args.db is not None: + return Path(args.db) + env_db = os.environ.get("DPORTSV3_STATE_DB") + if env_db: + return Path(env_db) + return Path.cwd() / "state.db" + + def _cmd_serve(args: Namespace) -> int: uvicorn_spec = importlib_util.find_spec("uvicorn") if uvicorn_spec is None: @@ -68,7 +87,8 @@ def _cmd_serve(args: Namespace) -> int: from dportsv3.tracker.server import create_app - app = create_app(Path(args.db)) + db_path = _resolve_state_db_path(args) + app = create_app(db_path) uvicorn.run(app, host="0.0.0.0", port=int(args.port)) return 0 diff --git a/scripts/generator/dportsv3/db/__init__.py b/scripts/generator/dportsv3/db/__init__.py new file mode 100644 index 00000000000..17cbd66eacd --- /dev/null +++ b/scripts/generator/dportsv3/db/__init__.py @@ -0,0 +1 @@ +"""Shared database concerns for dportsv3 (schema, init, future migrations).""" diff --git a/scripts/generator/dportsv3/db/schema.py b/scripts/generator/dportsv3/db/schema.py new file mode 100644 index 00000000000..a7176e66b0e --- /dev/null +++ b/scripts/generator/dportsv3/db/schema.py @@ -0,0 +1,235 @@ +"""Shared SQLite schema for state.db. + +state.db is owned by ``artifact-store`` (the single writer). Tracker +becomes a read-only consumer in Phase 4 step 4. Defining the schema +here keeps both consumers in sync and makes integration tests easy +(import + spin up against an in-memory DB). + +The tracker tables (``build_types``, ``build_runs``, ``build_results``, +``port_status``) were folded in by Phase 4 step 1; until tracker.db +retires (step 9) the equivalent definitions in +``scripts/generator/dportsv3/tracker/db.py`` must stay identical. +""" + +from __future__ import annotations + +import sqlite3 + +# Default build types seeded into the build_types table on first start. +# Matches DEFAULT_BUILD_TYPES in dportsv3.tracker.db. +DEFAULT_BUILD_TYPES: tuple[str, ...] = ("test", "release") + +# All CREATE TABLE / CREATE INDEX statements for state.db. Idempotent — +# safe to re-run on an existing DB. +SCHEMA = """ +CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, + profile TEXT, + path TEXT, + ts_start TEXT, + ts_end TEXT, + last_seen_at TEXT +); + +CREATE TABLE IF NOT EXISTS bundles ( + bundle_id TEXT PRIMARY KEY, + run_id TEXT, + origin TEXT, + flavor TEXT, + ts_utc TEXT, + result TEXT, + path TEXT, + last_seen_at TEXT +); + +CREATE TABLE IF NOT EXISTS jobs ( + job_id TEXT PRIMARY KEY, + state TEXT, + type TEXT, + origin TEXT, + flavor TEXT, + bundle_dir TEXT, + created_ts_utc TEXT, + path TEXT, + last_error TEXT, + last_seen_at TEXT +); + +CREATE TABLE IF NOT EXISTS artifacts ( + bundle_id TEXT, + relpath TEXT, + kind TEXT, + mtime REAL, + size INTEGER, + PRIMARY KEY (bundle_id, relpath) +); + +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + type TEXT NOT NULL, + data_json TEXT +); + +CREATE TABLE IF NOT EXISTS activity_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts TEXT NOT NULL, + job_id TEXT, + stage TEXT, + message TEXT, + duration_ms INTEGER, + extra_json TEXT +); + +CREATE TABLE IF NOT EXISTS runner_status ( + id INTEGER PRIMARY KEY DEFAULT 1 CHECK (id = 1), + status TEXT NOT NULL DEFAULT 'unknown', + job_id TEXT, + current_stage TEXT, + started_at TEXT, + updated_at TEXT, + extra_json TEXT +); + +CREATE TABLE IF NOT EXISTS user_context ( + run_id TEXT NOT NULL, + origin TEXT NOT NULL, + context_text TEXT NOT NULL, + updated_at TEXT NOT NULL, + context_rev INTEGER NOT NULL DEFAULT 1, + PRIMARY KEY (run_id, origin) +); + +CREATE TABLE IF NOT EXISTS user_context_requests ( + run_id TEXT NOT NULL, + origin TEXT NOT NULL, + bundle_id TEXT NOT NULL, + confidence TEXT, + classification TEXT, + iteration INTEGER, + max_iterations INTEGER, + requested_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + last_context_rev_handled INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (run_id, origin, bundle_id) +); + +CREATE TABLE IF NOT EXISTS blob_objects ( + sha256 TEXT PRIMARY KEY, + size INTEGER NOT NULL, + created_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS artifact_refs ( + bundle_id TEXT NOT NULL, + relpath TEXT NOT NULL, + backend TEXT NOT NULL, + sha256 TEXT, + fs_path TEXT, + kind TEXT, + size INTEGER, + created_at TEXT NOT NULL, + PRIMARY KEY (bundle_id, relpath) +); + +CREATE INDEX IF NOT EXISTS idx_events_id ON events(id); +CREATE INDEX IF NOT EXISTS idx_activity_log_ts ON activity_log(ts); +CREATE INDEX IF NOT EXISTS idx_user_context_updated ON user_context(updated_at); +CREATE INDEX IF NOT EXISTS idx_user_context_requests_pending ON user_context_requests(status, requested_at); +CREATE INDEX IF NOT EXISTS idx_artifact_refs_bundle ON artifact_refs(bundle_id); +CREATE INDEX IF NOT EXISTS idx_artifact_refs_sha ON artifact_refs(sha256); + +-- Phase 4 step 1: tracker schema folded into state.db. +CREATE TABLE IF NOT EXISTS build_types ( + name TEXT PRIMARY KEY +); + +CREATE TABLE IF NOT EXISTS build_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + build_type TEXT NOT NULL REFERENCES build_types(name), + started_at TEXT NOT NULL, + finished_at TEXT, + commit_sha TEXT, + commit_branch TEXT, + commit_pushed_at TEXT +); + +CREATE TABLE IF NOT EXISTS build_results ( + build_run_id INTEGER NOT NULL REFERENCES build_runs(id), + origin TEXT NOT NULL, + version TEXT NOT NULL, + result TEXT NOT NULL, + log_url TEXT, + recorded_at TEXT NOT NULL, + PRIMARY KEY (build_run_id, origin) +); + +CREATE TABLE IF NOT EXISTS port_status ( + target TEXT NOT NULL, + origin TEXT NOT NULL, + last_attempt_version TEXT, + last_attempt_result TEXT, + last_attempt_at TEXT, + last_attempt_run_id INTEGER REFERENCES build_runs(id), + last_success_version TEXT, + last_success_at TEXT, + last_success_run_id INTEGER REFERENCES build_runs(id), + PRIMARY KEY (target, origin) +); + +CREATE INDEX IF NOT EXISTS idx_build_runs_target ON build_runs(target); +CREATE INDEX IF NOT EXISTS idx_build_results_origin ON build_results(origin); +CREATE INDEX IF NOT EXISTS idx_port_status_target ON port_status(target); +CREATE INDEX IF NOT EXISTS idx_port_status_failures + ON port_status(target, last_attempt_result); +CREATE INDEX IF NOT EXISTS idx_build_runs_target_type_started + ON build_runs(target, build_type, started_at DESC, id DESC); +CREATE UNIQUE INDEX IF NOT EXISTS uq_build_runs_active + ON build_runs(target, build_type) + WHERE finished_at IS NULL; +""" + +# Idempotent ADD COLUMN migrations. Wrapped at call time because SQLite +# raises OperationalError when the column already exists. Order matters +# only insofar as they target their own tables — no cross-deps. +MIGRATIONS: tuple[str, ...] = ( + "ALTER TABLE build_results ADD COLUMN status TEXT NOT NULL DEFAULT 'recorded'", + "ALTER TABLE build_runs ADD COLUMN total_expected INTEGER", + "ALTER TABLE runs ADD COLUMN build_run_id INTEGER", + # Phase 4 step 5: target awareness on the agentic side. Nullable + # because pre-step-5 rows exist; new writes carry target. + "ALTER TABLE bundles ADD COLUMN target TEXT", + "ALTER TABLE jobs ADD COLUMN target TEXT", + "ALTER TABLE runs ADD COLUMN target TEXT", + "CREATE INDEX IF NOT EXISTS idx_bundles_target ON bundles(target)", + "CREATE INDEX IF NOT EXISTS idx_jobs_target ON jobs(target)", + "CREATE INDEX IF NOT EXISTS idx_runs_target ON runs(target)", +) + + +def init_db(conn: sqlite3.Connection) -> None: + """Run schema + seeds + migrations on an open connection. + + Called by artifact-store at startup. Tracker (read-only) doesn't + need this — it just opens the DB and queries. Sets PRAGMAs first + so the rest of the call inherits them. + """ + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=5000") + # Enforce FK constraints introduced by the tracker tables + # (build_results.build_run_id -> build_runs.id, etc.). The original + # artifact-store tables have no FKs, so the only impact is on + # writes to the folded-in tracker tables. + conn.execute("PRAGMA foreign_keys=ON") + conn.executescript(SCHEMA) + conn.executemany( + "INSERT OR IGNORE INTO build_types(name) VALUES (?)", + [(name,) for name in DEFAULT_BUILD_TYPES], + ) + for stmt in MIGRATIONS: + try: + conn.execute(stmt) + except sqlite3.OperationalError: + pass # column already exists + conn.commit() diff --git a/scripts/generator/dportsv3/tracker/agentic_queries.py b/scripts/generator/dportsv3/tracker/agentic_queries.py new file mode 100644 index 00000000000..ba6f50db07d --- /dev/null +++ b/scripts/generator/dportsv3/tracker/agentic_queries.py @@ -0,0 +1,284 @@ +"""SQL helpers for the tracker's agentic-read endpoints. + +These queries read the state.db tables originally owned by the +(now-retired) state-server: ``runs``, ``bundles``, ``jobs``, ``events``, +``activity_log``, ``runner_status``, ``artifact_refs``. + +Target filtering: ``bundles``, ``jobs``, ``runs`` carry a nullable +``target`` column added in step 5. Filter is applied as an equality +match when supplied. ``NULL``-target rows surface only when no filter +is set — they're legacy or filed by a writer that didn't know its +target. +""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + + +def _row_dict(row: sqlite3.Row) -> dict[str, Any]: + return {str(key): row[key] for key in row.keys()} + + +def _maybe(row: sqlite3.Row | None) -> dict[str, Any] | None: + return _row_dict(row) if row is not None else None + + +def agentic_status(conn: sqlite3.Connection) -> dict[str, Any]: + """Global aggregate counts for /api/agentic-status.""" + bundles = conn.execute("SELECT count(*) FROM bundles").fetchone()[0] + jobs_pending = conn.execute( + "SELECT count(*) FROM jobs WHERE state = 'pending'" + ).fetchone()[0] + jobs_inflight = conn.execute( + "SELECT count(*) FROM jobs WHERE state = 'inflight'" + ).fetchone()[0] + jobs_done = conn.execute( + "SELECT count(*) FROM jobs WHERE state = 'done'" + ).fetchone()[0] + jobs_failed = conn.execute( + "SELECT count(*) FROM jobs WHERE state = 'failed'" + ).fetchone()[0] + runs = conn.execute("SELECT count(*) FROM runs").fetchone()[0] + return { + "bundles": bundles, + "runs": runs, + "jobs": { + "pending": jobs_pending, + "inflight": jobs_inflight, + "done": jobs_done, + "failed": jobs_failed, + }, + } + + +def recent_activity( + conn: sqlite3.Connection, + limit: int = 10, + target: str | None = None, +) -> list[dict[str, Any]]: + """Most recent activity_log rows, newest first. + + activity_log itself has no target column — filter is applied via + a join to the originating job's target when target is supplied. + Rows whose job_id doesn't resolve are dropped under filter. + """ + if target is None: + rows = conn.execute( + "SELECT * FROM activity_log ORDER BY id DESC LIMIT ?", + (max(1, int(limit)),), + ).fetchall() + else: + rows = conn.execute( + """SELECT activity_log.* + FROM activity_log + JOIN jobs ON jobs.job_id = activity_log.job_id + WHERE jobs.target = ? + ORDER BY activity_log.id DESC + LIMIT ?""", + (target, max(1, int(limit))), + ).fetchall() + return [_row_dict(row) for row in rows] + + +def runner_status(conn: sqlite3.Connection) -> dict[str, Any]: + """Singleton runner_status row, or a defaulted shape if unset.""" + row = conn.execute( + "SELECT * FROM runner_status WHERE id = 1" + ).fetchone() + if row is None: + return {"status": "unknown"} + return _row_dict(row) + + +def list_runs( + conn: sqlite3.Connection, + target: str | None = None, + limit: int = 50, +) -> list[dict[str, Any]]: + sql = "SELECT * FROM runs" + params: list[Any] = [] + if target is not None: + sql += " WHERE target = ?" + params.append(target) + sql += " ORDER BY ts_start DESC, run_id DESC LIMIT ?" + params.append(max(1, int(limit))) + return [_row_dict(row) for row in conn.execute(sql, params).fetchall()] + + +def get_run(conn: sqlite3.Connection, run_id: str) -> dict[str, Any] | None: + return _maybe( + conn.execute("SELECT * FROM runs WHERE run_id = ?", (run_id,)).fetchone() + ) + + +def list_jobs( + conn: sqlite3.Connection, + state: str | None = None, + target: str | None = None, + limit: int = 100, +) -> list[dict[str, Any]]: + sql = "SELECT * FROM jobs" + clauses: list[str] = [] + params: list[Any] = [] + if state is not None: + clauses.append("state = ?") + params.append(state) + if target is not None: + clauses.append("target = ?") + params.append(target) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY created_ts_utc DESC, job_id DESC LIMIT ?" + params.append(max(1, int(limit))) + return [_row_dict(row) for row in conn.execute(sql, params).fetchall()] + + +def get_job(conn: sqlite3.Connection, job_id: str) -> dict[str, Any] | None: + return _maybe( + conn.execute("SELECT * FROM jobs WHERE job_id = ?", (job_id,)).fetchone() + ) + + +def list_bundles( + conn: sqlite3.Connection, + target: str | None = None, + origin: str | None = None, + limit: int = 100, +) -> list[dict[str, Any]]: + sql = "SELECT * FROM bundles" + clauses: list[str] = [] + params: list[Any] = [] + if target is not None: + clauses.append("target = ?") + params.append(target) + if origin is not None: + clauses.append("origin = ?") + params.append(origin) + if clauses: + sql += " WHERE " + " AND ".join(clauses) + sql += " ORDER BY ts_utc DESC, bundle_id DESC LIMIT ?" + params.append(max(1, int(limit))) + return [_row_dict(row) for row in conn.execute(sql, params).fetchall()] + + +def get_bundle(conn: sqlite3.Connection, bundle_id: str) -> dict[str, Any] | None: + row = conn.execute( + "SELECT * FROM bundles WHERE bundle_id = ?", (bundle_id,) + ).fetchone() + if row is None: + return None + bundle = _row_dict(row) + artifacts = conn.execute( + """SELECT relpath, backend, sha256, fs_path, kind, size, created_at + FROM artifact_refs + WHERE bundle_id = ? + ORDER BY relpath ASC""", + (bundle_id,), + ).fetchall() + bundle["artifacts"] = [_row_dict(r) for r in artifacts] + return bundle + + +def get_artifact_ref( + conn: sqlite3.Connection, bundle_id: str, relpath: str +) -> dict[str, Any] | None: + return _maybe( + conn.execute( + """SELECT backend, sha256, fs_path, kind, size + FROM artifact_refs + WHERE bundle_id = ? AND relpath = ?""", + (bundle_id, relpath), + ).fetchone() + ) + + +def list_port_bundles( + conn: sqlite3.Connection, + origin: str, + target: str | None = None, + limit: int = 50, +) -> list[dict[str, Any]]: + sql = "SELECT * FROM bundles WHERE origin = ?" + params: list[Any] = [origin] + if target is not None: + sql += " AND target = ?" + params.append(target) + sql += " ORDER BY ts_utc DESC LIMIT ?" + params.append(max(1, int(limit))) + return [_row_dict(row) for row in conn.execute(sql, params).fetchall()] + + +def distinct_targets(conn: sqlite3.Connection) -> list[str]: + """Sorted list of non-NULL targets seen across bundles/jobs/runs. + + Used to populate target-selector dropdowns on the HTML views. + """ + rows = conn.execute( + """SELECT DISTINCT target FROM ( + SELECT target FROM bundles + UNION SELECT target FROM jobs + UNION SELECT target FROM runs + ) + WHERE target IS NOT NULL AND target <> '' + ORDER BY target ASC""" + ).fetchall() + return [str(row[0]) for row in rows] + + +def activity_for_job( + conn: sqlite3.Connection, job_id: str, limit: int = 50 +) -> list[dict[str, Any]]: + """Activity-log rows for one job_id, newest first.""" + rows = conn.execute( + "SELECT * FROM activity_log WHERE job_id = ? ORDER BY id DESC LIMIT ?", + (job_id, max(1, int(limit))), + ).fetchall() + return [_row_dict(row) for row in rows] + + +def bundles_for_run( + conn: sqlite3.Connection, run_id: str, limit: int = 200 +) -> list[dict[str, Any]]: + rows = conn.execute( + "SELECT * FROM bundles WHERE run_id = ? ORDER BY ts_utc DESC LIMIT ?", + (run_id, max(1, int(limit))), + ).fetchall() + return [_row_dict(row) for row in rows] + + +def events_since( + conn: sqlite3.Connection, + last_id: int = 0, + target: str | None = None, + limit: int = 100, +) -> list[dict[str, Any]]: + """Return events with ``id > last_id``, oldest first. + + Used by the SSE endpoint to tail events. Target filter is best-effort: + an event's ``data_json`` carries ``target`` when the originating + write knew it (post-step-5). Pre-step-5 events have no target and + surface only when no filter is set. + """ + rows = conn.execute( + "SELECT id, ts, type, data_json FROM events WHERE id > ? ORDER BY id ASC LIMIT ?", + (int(last_id), max(1, int(limit))), + ).fetchall() + items = [_row_dict(row) for row in rows] + if target is None: + return items + out: list[dict[str, Any]] = [] + import json + + for item in items: + raw = item.get("data_json") + if not raw: + continue + try: + payload = json.loads(raw) + except (ValueError, TypeError): + continue + if payload.get("target") == target: + out.append(item) + return out diff --git a/scripts/generator/dportsv3/tracker/db.py b/scripts/generator/dportsv3/tracker/db.py index 68934cdcee0..7e70de22406 100644 --- a/scripts/generator/dportsv3/tracker/db.py +++ b/scripts/generator/dportsv3/tracker/db.py @@ -1,4 +1,15 @@ -"""SQLite-backed build tracker database helpers.""" +"""SQLite-backed build tracker database helpers. + +As of Phase 4 step 4 the tracker is a read+write consumer of ``state.db`` +(the same file artifact-store writes). The schema is defined once in +``dportsv3.db.schema``; this module imports it and provides the +tracker-specific query helpers on top. + +Two writers (artifact-store + this module) share state.db under SQLite +WAL — one writer at a time at the SQLite layer, readers proceed in +parallel. Each connection opens with the same PRAGMA set (WAL, +busy_timeout=5000, foreign_keys=ON) via ``open_db`` / ``init_db``. +""" from __future__ import annotations @@ -8,9 +19,28 @@ from typing import Any, cast from dportsv3.common.validation import is_compose_target +from dportsv3.db.schema import DEFAULT_BUILD_TYPES, init_db as _init_state_db VALID_BUILD_RESULTS = frozenset({"success", "failure", "skipped", "ignored"}) -DEFAULT_BUILD_TYPES = ("test", "release") + + +def open_db(db_path: str | Path) -> sqlite3.Connection: + """Open one configured SQLite connection for tracker operations. + + PRAGMAs match what ``dportsv3.db.schema.init_db`` sets so that + artifact-store and tracker write under identical conditions. Pragmas + are per-connection in SQLite — applying them here on every + connection (the tracker server opens fresh ones per request after + commit a14fe9c4dab) is the only safe pattern. + """ + path_text = str(db_path) + conn = sqlite3.connect(path_text, check_same_thread=False) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + if path_text != ":memory:": + conn.execute("PRAGMA journal_mode = WAL") + conn.execute("PRAGMA busy_timeout = 5000") + return conn class ActiveBuildError(RuntimeError): @@ -29,82 +59,14 @@ def __init__(self, active_run: dict[str, Any]) -> None: def init_db(db_path: str | Path) -> sqlite3.Connection: - """Initialize the tracker schema and return one configured connection.""" - path_text = str(db_path) - conn = sqlite3.connect(path_text, check_same_thread=False) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA foreign_keys = ON") - if path_text != ":memory:": - conn.execute("PRAGMA journal_mode = WAL") - - with conn: - conn.executescript( - """ - CREATE TABLE IF NOT EXISTS build_types ( - name TEXT PRIMARY KEY - ); - - CREATE TABLE IF NOT EXISTS build_runs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - target TEXT NOT NULL, - build_type TEXT NOT NULL REFERENCES build_types(name), - started_at TEXT NOT NULL, - finished_at TEXT, - commit_sha TEXT, - commit_branch TEXT, - commit_pushed_at TEXT - ); - - CREATE TABLE IF NOT EXISTS build_results ( - build_run_id INTEGER NOT NULL REFERENCES build_runs(id), - origin TEXT NOT NULL, - version TEXT NOT NULL, - result TEXT NOT NULL, - log_url TEXT, - recorded_at TEXT NOT NULL, - PRIMARY KEY (build_run_id, origin) - ); - - CREATE TABLE IF NOT EXISTS port_status ( - target TEXT NOT NULL, - origin TEXT NOT NULL, - last_attempt_version TEXT, - last_attempt_result TEXT, - last_attempt_at TEXT, - last_attempt_run_id INTEGER REFERENCES build_runs(id), - last_success_version TEXT, - last_success_at TEXT, - last_success_run_id INTEGER REFERENCES build_runs(id), - PRIMARY KEY (target, origin) - ); - - CREATE INDEX IF NOT EXISTS idx_build_runs_target ON build_runs(target); - CREATE INDEX IF NOT EXISTS idx_build_results_origin ON build_results(origin); - CREATE INDEX IF NOT EXISTS idx_port_status_target ON port_status(target); - CREATE INDEX IF NOT EXISTS idx_port_status_failures - ON port_status(target, last_attempt_result); - CREATE INDEX IF NOT EXISTS idx_build_runs_target_type_started - ON build_runs(target, build_type, started_at DESC, id DESC); - CREATE UNIQUE INDEX IF NOT EXISTS uq_build_runs_active - ON build_runs(target, build_type) - WHERE finished_at IS NULL; - """ - ) - conn.executemany( - "INSERT OR IGNORE INTO build_types(name) VALUES (?)", - [(name,) for name in DEFAULT_BUILD_TYPES], - ) - - # Idempotent schema migrations for queue tracking - for stmt in ( - "ALTER TABLE build_results ADD COLUMN status TEXT NOT NULL DEFAULT 'recorded'", - "ALTER TABLE build_runs ADD COLUMN total_expected INTEGER", - ): - try: - conn.execute(stmt) - except sqlite3.OperationalError: - pass # column already exists - + """Open state.db and ensure the schema + seed + migrations are present. + + Delegates to ``dportsv3.db.schema.init_db`` so the schema definition + lives in one place. Idempotent on existing files (artifact-store may + already have initialized the same DB). + """ + conn = open_db(db_path) + _init_state_db(conn) return conn diff --git a/scripts/generator/dportsv3/tracker/progress_adapter.py b/scripts/generator/dportsv3/tracker/progress_adapter.py new file mode 100644 index 00000000000..39d912d0e95 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/progress_adapter.py @@ -0,0 +1,274 @@ +"""Adapter that exposes tracker data in dsynth-progress' JSON shape. + +The dsynth-progress UI (``www/example/progress.{html,js,css}``) consumes +two endpoints: + +- ``summary.json`` — profile + kickoff + stats + active builders +- ``_history.json`` — array of build entries, paginated into chunks + +This module maps the tracker's ``state.db`` rows (``build_runs``, +``build_results``, ``port_status``) into that shape so the lifted UI +runs against tracker data without modification. + +Result vocabulary mapping: +- tracker ``success`` → dsynth ``built`` +- tracker ``failure`` → dsynth ``failed`` +- tracker ``skipped`` → dsynth ``skipped`` +- tracker ``ignored`` → dsynth ``ignored`` +- (no tracker analog) → dsynth ``meta`` — left at 0 + +Chunk size is fixed at 1000 entries per ``_history.json``, matching +dsynth-progress' own chunking. ``kfiles`` in summary.json is the count +of chunks the UI should fetch. +""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + +CHUNK_SIZE = 1000 + +_RESULT_TO_DSYNTH = { + "success": "built", + "failure": "failed", + "skipped": "skipped", + "ignored": "ignored", +} + + +def _latest_run_id(conn: sqlite3.Connection, target: str) -> int | None: + row = conn.execute( + """SELECT id FROM build_runs + WHERE target = ? + ORDER BY started_at DESC, id DESC LIMIT 1""", + (target,), + ).fetchone() + return int(row[0]) if row else None + + +def target_summary(conn: sqlite3.Connection, target: str) -> dict[str, Any]: + """Return the summary.json shape for the latest run on ``target``.""" + run_id = _latest_run_id(conn, target) + if run_id is None: + return _empty_summary(target) + summary = _run_summary_by_id(conn, run_id) + return summary if summary is not None else _empty_summary(target) + + +def run_summary(conn: sqlite3.Connection, run_id: int) -> dict[str, Any] | None: + """Return the summary.json shape for a specific build_run, or None.""" + return _run_summary_by_id(conn, run_id) + + +def _run_summary_by_id( + conn: sqlite3.Connection, run_id: int +) -> dict[str, Any] | None: + run = conn.execute( + """SELECT id, target, build_type, started_at, finished_at, total_expected + FROM build_runs WHERE id = ?""", + (run_id,), + ).fetchone() + if run is None: + return None + + counts = conn.execute( + """SELECT + COUNT(*) AS total, + COALESCE(SUM(CASE WHEN result = 'success' THEN 1 ELSE 0 END), 0) AS built, + COALESCE(SUM(CASE WHEN result = 'failure' THEN 1 ELSE 0 END), 0) AS failed, + COALESCE(SUM(CASE WHEN result = 'skipped' THEN 1 ELSE 0 END), 0) AS skipped, + COALESCE(SUM(CASE WHEN result = 'ignored' THEN 1 ELSE 0 END), 0) AS ignored, + COALESCE(SUM(CASE WHEN status = 'building' THEN 1 ELSE 0 END), 0) AS building, + COALESCE(SUM(CASE WHEN status = 'queued' THEN 1 ELSE 0 END), 0) AS queued + FROM build_results WHERE build_run_id = ?""", + (run_id,), + ).fetchone() + + total_recorded = int(counts["total"]) + total_expected = int(run["total_expected"] or total_recorded) + remains = max(0, total_expected - total_recorded) + + elapsed = _elapsed_str(str(run["started_at"]), run["finished_at"]) + + builders = _active_builders(conn, run_id) + + # kfiles counts chunks of *historical* rows the UI will fetch — + # building/queued rows live in `builders`, not in NN_history.json, + # so exclude them from the chunk math. + historical = total_recorded - int(counts["building"]) - int(counts["queued"]) + kfiles = max(1, (historical + CHUNK_SIZE - 1) // CHUNK_SIZE) if historical else 0 + + return { + "profile": str(run["target"]), + "kickoff": _format_kickoff(str(run["started_at"])), + "kfiles": kfiles, + "active": 1 if run["finished_at"] is None else 0, + "stats": { + "queued": total_expected, + "built": int(counts["built"]), + "failed": int(counts["failed"]), + "ignored": int(counts["ignored"]), + "skipped": int(counts["skipped"]), + "remains": remains, + "meta": 0, + "elapsed": elapsed, + "pkghour": 0, + "impulse": 0, + "swapinfo": " -", + "load": " -", + }, + "builders": builders, + } + + +def target_history_chunk( + conn: sqlite3.Connection, + target: str, + chunk_index: int, +) -> list[dict[str, Any]]: + """Return one chunk of build entries for the latest run on ``target``.""" + run_id = _latest_run_id(conn, target) + if run_id is None: + return [] + return run_history_chunk(conn, run_id, chunk_index) + + +def run_history_chunk( + conn: sqlite3.Connection, + run_id: int, + chunk_index: int, +) -> list[dict[str, Any]]: + """Return one chunk of build entries for a specific build_run. + + Chunks are 1-indexed to match dsynth-progress (``01_history.json`` = + chunk 1). Returns an empty list past the last chunk. + """ + if chunk_index < 1: + return [] + offset = (chunk_index - 1) * CHUNK_SIZE + # 'building' and 'queued' rows are in-flight — they belong in + # summary.builders, not in the historical record. + rows = conn.execute( + """SELECT origin, version, result, recorded_at, status + FROM build_results + WHERE build_run_id = ? + AND status NOT IN ('building', 'queued') + ORDER BY recorded_at ASC, origin ASC + LIMIT ? OFFSET ?""", + (run_id, CHUNK_SIZE, offset), + ).fetchall() + + entries: list[dict[str, Any]] = [] + for i, row in enumerate(rows): + entries.append( + { + "entry": offset + i + 1, + "elapsed": "", + "ID": "00", + "result": _RESULT_TO_DSYNTH.get( + str(row["result"] or ""), str(row["result"] or "") + ), + "origin": str(row["origin"]), + "info": str(row["version"] or ""), + "duration": "", + } + ) + return entries + + +def _empty_summary(target: str) -> dict[str, Any]: + return { + "profile": target, + "kickoff": "", + "kfiles": 0, + "active": 0, + "stats": { + "queued": 0, + "built": 0, + "failed": 0, + "ignored": 0, + "skipped": 0, + "remains": 0, + "meta": 0, + "elapsed": "", + "pkghour": 0, + "impulse": 0, + "swapinfo": " -", + "load": " -", + }, + "builders": [], + } + + +def _active_builders( + conn: sqlite3.Connection, run_id: int +) -> list[dict[str, Any]]: + """One row per port currently in 'building' state. + + Tracker has no per-builder-slot model — every in-progress port maps + to one virtual slot ID (zero-padded index). Matches dsynth-progress' + table shape without claiming we have N physical builder slots. + """ + rows = conn.execute( + """SELECT origin, recorded_at + FROM build_results + WHERE build_run_id = ? AND status = 'building' + ORDER BY recorded_at ASC""", + (run_id,), + ).fetchall() + out: list[dict[str, Any]] = [] + for i, row in enumerate(rows): + out.append( + { + "ID": _two_digit(i), + "elapsed": " --:--:--", + "phase": "build", + "origin": str(row["origin"]), + "lines": "", + } + ) + return out + + +def _two_digit(n: int) -> str: + return f"{n:02d}" if n < 100 else str(n) + + +def _elapsed_str(started_at: str, finished_at: str | None) -> str: + """HH:MM:SS between two ISO timestamps (best-effort). + + dsynth-progress' format is space-padded HH:MM:SS. If timestamps + don't parse, returns empty. + """ + from datetime import datetime + + try: + start = datetime.fromisoformat(started_at) + except (ValueError, TypeError): + return "" + if finished_at: + try: + end = datetime.fromisoformat(finished_at) + except (ValueError, TypeError): + return "" + else: + end = datetime.now(start.tzinfo) if start.tzinfo else datetime.now() + delta = end - start + secs = max(0, int(delta.total_seconds())) + h, rem = divmod(secs, 3600) + m, s = divmod(rem, 60) + return f"{h:02d}:{m:02d}:{s:02d}" + + +def _format_kickoff(started_at: str) -> str: + """Best-effort match to dsynth's ' DD-Mon-YYYY HH:MM:SS UTC' format.""" + from datetime import datetime, timezone + + try: + dt = datetime.fromisoformat(started_at) + except (ValueError, TypeError): + return started_at + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).strftime(" %d-%b-%Y %H:%M:%S UTC") diff --git a/scripts/generator/dportsv3/tracker/server.py b/scripts/generator/dportsv3/tracker/server.py index 82c6c2873e7..9655ad8c9c8 100644 --- a/scripts/generator/dportsv3/tracker/server.py +++ b/scripts/generator/dportsv3/tracker/server.py @@ -3,10 +3,35 @@ from __future__ import annotations import importlib +import os +from contextlib import contextmanager from importlib import util as importlib_util from pathlib import Path from typing import Any, cast +from dportsv3.tracker.progress_adapter import ( + run_history_chunk, + run_summary, + target_history_chunk, + target_summary, +) +from dportsv3.tracker.agentic_queries import ( + activity_for_job, + agentic_status, + bundles_for_run, + distinct_targets, + events_since, + get_artifact_ref, + get_bundle, + get_job, + get_run, + list_bundles, + list_jobs, + list_port_bundles, + list_runs, + recent_activity, + runner_status, +) from dportsv3.tracker.db import ( ActiveBuildError, compare_builds, @@ -23,6 +48,7 @@ get_target_summary, init_db, list_build_runs, + open_db, record_results, update_port_status, ) @@ -67,6 +93,8 @@ HTMLResponseType = _responses.HTMLResponse StaticFilesType = _staticfiles.StaticFiles Jinja2TemplatesType = _templating.Jinja2Templates + FileResponseType = _responses.FileResponse + StreamingResponseType = _responses.StreamingResponse else: class _MissingFastAPI: @@ -100,6 +128,39 @@ def _missing_query(*_args: Any, **_kwargs: Any) -> None: HTMLResponseType = _MissingHTMLResponse StaticFilesType = _MissingStaticFiles Jinja2TemplatesType = _MissingTemplates + FileResponseType = _MissingHTMLResponse + StreamingResponseType = _MissingHTMLResponse + + +def _resolve_artifact_path( + artifact_root: Path, ref: dict[str, Any] +) -> Path | None: + """Locate the on-disk file for an artifact_refs row. + + Two backends: + - 'blob': content-addressed under ``/objects/sha256/aa/bb/`` + - 'fs': absolute ``fs_path`` recorded at upsert time + """ + backend = ref.get("backend") + if backend == "blob": + sha = ref.get("sha256") + if not sha or len(sha) < 4: + return None + return ( + artifact_root + / "blobstore" + / "objects" + / "sha256" + / sha[0:2] + / sha[2:4] + / sha + ) + if backend == "fs": + fs_path = ref.get("fs_path") + if not fs_path: + return None + return Path(fs_path) + return None def create_app(db_path: str | Path) -> Any: @@ -120,10 +181,16 @@ def create_app(db_path: str | Path) -> Any: HTMLResponse = cast(Any, HTMLResponseType) StaticFiles = cast(Any, StaticFilesType) Jinja2Templates = cast(Any, Jinja2TemplatesType) + FileResponse = cast(Any, FileResponseType) + StreamingResponse = cast(Any, StreamingResponseType) app: Any = FastAPI(title="DeltaPorts Build Tracker") app.state.db_path = str(db_path) - app.state.db_conn = None + # Resolves /api/bundles//artifacts/ for the 'blob' + # backend. Defaults match artifact-store's --logs-root default. + app.state.artifact_root = Path( + os.environ.get("DPORTSV3_ARTIFACT_ROOT", "/build/synth/logs/evidence") + ) templates_dir = Path(__file__).with_name("templates") static_dir = Path(__file__).with_name("static") templates: Any = Jinja2Templates(directory=str(templates_dir)) @@ -132,19 +199,20 @@ def create_app(db_path: str | Path) -> Any: @app.on_event("startup") def _startup() -> None: - app.state.db_conn = init_db(app.state.db_path) + conn = init_db(app.state.db_path) + conn.close() @app.on_event("shutdown") def _shutdown() -> None: - if app.state.db_conn is not None: - app.state.db_conn.close() - app.state.db_conn = None + return None + @contextmanager def _conn() -> Any: - conn = app.state.db_conn - if conn is None: - raise RuntimeError("Tracker DB connection is not initialized") - return conn + conn = open_db(app.state.db_path) + try: + yield conn + finally: + conn.close() def _raise_http_error(exc: Exception) -> None: if isinstance(exc, ActiveBuildError): @@ -165,12 +233,13 @@ def _raise_http_error(exc: Exception) -> None: def start_build(payload: StartBuildRequest) -> dict[str, int]: run_id = 0 try: - run_id = create_build_run( - _conn(), - target=payload.target, - build_type=payload.build_type, - started_at=payload.started_at, - ) + with _conn() as conn: + run_id = create_build_run( + conn, + target=payload.target, + build_type=payload.build_type, + started_at=payload.started_at, + ) except Exception as exc: _raise_http_error(exc) raise AssertionError("unreachable") @@ -179,14 +248,15 @@ def start_build(payload: StartBuildRequest) -> dict[str, int]: @app.patch("/api/builds/{run_id}") def finish_build(run_id: int, payload: FinishBuildRequest) -> dict[str, bool]: try: - finish_build_run( - _conn(), - run_id=run_id, - finished_at=payload.finished_at, - commit_sha=payload.commit_sha, - commit_branch=payload.commit_branch, - commit_pushed_at=payload.commit_pushed_at, - ) + with _conn() as conn: + finish_build_run( + conn, + run_id=run_id, + finished_at=payload.finished_at, + commit_sha=payload.commit_sha, + commit_branch=payload.commit_branch, + commit_pushed_at=payload.commit_pushed_at, + ) except Exception as exc: _raise_http_error(exc) return {"ok": True} @@ -198,13 +268,14 @@ def add_results( ) -> dict[str, int]: recorded = 0 try: - run = get_build_run(_conn(), run_id) - recorded = record_results( - _conn(), - run_id=run_id, - target=str(run["target"]), - results=[item.model_dump() for item in payload.results], - ) + with _conn() as conn: + run = get_build_run(conn, run_id) + recorded = record_results( + conn, + run_id=run_id, + target=str(run["target"]), + results=[item.model_dump() for item in payload.results], + ) except Exception as exc: _raise_http_error(exc) raise AssertionError("unreachable") @@ -213,12 +284,13 @@ def add_results( @app.post("/api/builds/{run_id}/queue", response_model=EnqueueResponse) def enqueue(run_id: int, payload: EnqueueRequest) -> dict[str, int]: try: - count = enqueue_ports( - _conn(), - run_id, - [item.model_dump() for item in payload.ports], - total_expected=payload.total_expected, - ) + with _conn() as conn: + count = enqueue_ports( + conn, + run_id, + [item.model_dump() for item in payload.ports], + total_expected=payload.total_expected, + ) except Exception as exc: _raise_http_error(exc) raise AssertionError("unreachable") @@ -231,7 +303,8 @@ def patch_port_status( payload: UpdatePortStatusRequest, ) -> dict[str, bool]: try: - update_port_status(_conn(), run_id, origin, payload.status) + with _conn() as conn: + update_port_status(conn, run_id, origin, payload.status) except Exception as exc: _raise_http_error(exc) return {"ok": True} @@ -242,14 +315,16 @@ def api_list_builds( build_type: str | None = None, limit: int = Query(default=20, ge=1, le=1000), ) -> list[dict[str, Any]]: - return list_build_runs( - _conn(), target=target, build_type=build_type, limit=limit - ) + with _conn() as conn: + return list_build_runs( + conn, target=target, build_type=build_type, limit=limit + ) @app.get("/api/builds/compare", response_model=BuildCompareOut) def api_compare_builds(a: int, b: int) -> dict[str, Any]: try: - return compare_builds(_conn(), a, b) + with _conn() as conn: + return compare_builds(conn, a, b) except Exception as exc: _raise_http_error(exc) raise AssertionError("unreachable") @@ -257,10 +332,11 @@ def api_compare_builds(a: int, b: int) -> dict[str, Any]: @app.get("/api/builds/{run_id}") def api_get_build(run_id: int) -> dict[str, Any]: try: - return { - "build_run": get_build_run(_conn(), run_id), - "results": get_build_results(_conn(), run_id), - } + with _conn() as conn: + return { + "build_run": get_build_run(conn, run_id), + "results": get_build_results(conn, run_id), + } except Exception as exc: _raise_http_error(exc) raise AssertionError("unreachable") @@ -270,63 +346,346 @@ def api_status( target: str | None = None, origin: str | None = None, ) -> list[dict[str, Any]]: - return get_port_status(_conn(), target=target, origin=origin) + with _conn() as conn: + return get_port_status(conn, target=target, origin=origin) @app.get("/api/failures", response_model=list[PortStatusOut]) def api_failures(target: str) -> list[dict[str, Any]]: - return get_failures(_conn(), target) + with _conn() as conn: + return get_failures(conn, target) @app.get("/api/diff", response_model=DiffOut) def api_diff(a: str, b: str) -> dict[str, Any]: - return get_diff(_conn(), a, b) + with _conn() as conn: + return get_diff(conn, a, b) + + # ------------------------------------------------------------------ + # Agentic-read endpoints (absorbed from the retired state-server). + # ------------------------------------------------------------------ + + @app.get("/api/health") + def api_health() -> dict[str, str]: + return {"status": "ok"} + + @app.get("/api/agentic-status") + def api_agentic_status() -> dict[str, Any]: + with _conn() as conn: + return agentic_status(conn) + + @app.get("/api/activity") + def api_activity( + limit: int = Query(default=10, ge=1, le=500), + target: str | None = None, + ) -> list[dict[str, Any]]: + with _conn() as conn: + return recent_activity(conn, limit=limit, target=target) + + @app.get("/api/runner-status") + def api_runner_status() -> dict[str, Any]: + with _conn() as conn: + return runner_status(conn) + + @app.get("/api/runs") + def api_runs( + target: str | None = None, + limit: int = Query(default=50, ge=1, le=500), + ) -> list[dict[str, Any]]: + with _conn() as conn: + return list_runs(conn, target=target, limit=limit) + + @app.get("/api/runs/{run_id}") + def api_run_detail(run_id: str) -> dict[str, Any]: + with _conn() as conn: + row = get_run(conn, run_id) + if row is None: + raise HTTPException(status_code=404, detail=f"Unknown run: {run_id}") + return row + + @app.get("/api/jobs") + def api_jobs( + state: str | None = None, + target: str | None = None, + limit: int = Query(default=100, ge=1, le=500), + ) -> list[dict[str, Any]]: + with _conn() as conn: + return list_jobs(conn, state=state, target=target, limit=limit) + + @app.get("/api/jobs/{job_id}") + def api_job_detail(job_id: str) -> dict[str, Any]: + with _conn() as conn: + row = get_job(conn, job_id) + if row is None: + raise HTTPException(status_code=404, detail=f"Unknown job: {job_id}") + return row + + @app.get("/api/bundles") + def api_bundles( + target: str | None = None, + origin: str | None = None, + limit: int = Query(default=100, ge=1, le=500), + ) -> list[dict[str, Any]]: + with _conn() as conn: + return list_bundles(conn, target=target, origin=origin, limit=limit) + + @app.get("/api/bundles/{bundle_id}") + def api_bundle_detail(bundle_id: str) -> dict[str, Any]: + with _conn() as conn: + row = get_bundle(conn, bundle_id) + if row is None: + raise HTTPException(status_code=404, detail=f"Unknown bundle: {bundle_id}") + return row + + @app.get("/api/ports/{origin:path}") + def api_port_bundles( + origin: str, + target: str | None = None, + limit: int = Query(default=50, ge=1, le=500), + ) -> list[dict[str, Any]]: + with _conn() as conn: + return list_port_bundles(conn, origin=origin, target=target, limit=limit) + + @app.get("/api/bundles/{bundle_id}/artifacts/{relpath:path}") + def api_bundle_artifact(bundle_id: str, relpath: str) -> Any: + # Resolve via artifact_refs, then stream from disk. Two backends: + # 'blob' (content-addressed under blob_root/objects/sha256) or + # 'fs' (absolute path in fs_path). + with _conn() as conn: + ref = get_artifact_ref(conn, bundle_id, relpath) + if ref is None: + raise HTTPException(status_code=404, detail="Unknown artifact") + path = _resolve_artifact_path(app.state.artifact_root, ref) + if path is None or not path.exists(): + raise HTTPException(status_code=404, detail="Artifact file missing") + media_type = "application/gzip" if (ref.get("kind") == "gzip") else "application/octet-stream" + return FileResponse(str(path), media_type=media_type) + + @app.get("/api/events") + def api_events( + target: str | None = None, + last_id: int = 0, + ) -> Any: + # Server-sent events: poll the events table on a 1s tick, emit + # rows with id > last_id, filter by target (best-effort — see + # events_since docstring). + import asyncio + import json as _json + + async def _gen() -> Any: + cursor = int(last_id) + try: + while True: + with _conn() as conn: + rows = events_since(conn, last_id=cursor, target=target) + for row in rows: + cursor = max(cursor, int(row["id"])) + payload = _json.dumps(row, default=str) + yield f"event: {row['type']}\ndata: {payload}\n\n" + await asyncio.sleep(1.0) + except asyncio.CancelledError: + return + + return StreamingResponse(_gen(), media_type="text/event-stream") @app.get("/", response_class=HTMLResponse) def dashboard_index(request: RequestType) -> Any: - active_builds = get_active_builds_summary(_conn()) + with _conn() as conn: + active_builds = get_active_builds_summary(conn) + return templates.TemplateResponse( + request, + "index.html", + { + "title": "Targets", + "targets": get_target_summary(conn), + "active_builds": active_builds, + "refresh_seconds": 30 if active_builds else None, + }, + ) + + # ------------------------------------------------------------------ + # Phase 4 step 6: agentic HTML views. + # ------------------------------------------------------------------ + + @app.get("/agentic", response_class=HTMLResponse) + def agentic_index(request: RequestType) -> Any: + with _conn() as conn: + return templates.TemplateResponse( + request, + "agentic_index.html", + { + "title": "Agentic", + "status": agentic_status(conn), + "recent_bundles": list_bundles(conn, limit=10), + "recent_jobs": list_jobs(conn, limit=10), + }, + ) + + @app.get("/agentic/bundles", response_class=HTMLResponse) + def agentic_bundles( + request: RequestType, + target: str | None = None, + origin: str | None = None, + ) -> Any: + target_value = target or None + origin_value = (origin or "").strip() or None + with _conn() as conn: + return templates.TemplateResponse( + request, + "agentic_bundles.html", + { + "title": "Bundles", + "bundles": list_bundles( + conn, target=target_value, origin=origin_value, limit=200 + ), + "target_options": distinct_targets(conn), + "selected_target": target_value, + "selected_origin": origin_value, + }, + ) + + @app.get("/agentic/bundles/{bundle_id}", response_class=HTMLResponse) + def agentic_bundle_detail(request: RequestType, bundle_id: str) -> Any: + with _conn() as conn: + bundle = get_bundle(conn, bundle_id) + if bundle is None: + raise HTTPException(status_code=404, detail=f"Unknown bundle: {bundle_id}") return templates.TemplateResponse( request, - "index.html", - { - "title": "Targets", - "targets": get_target_summary(_conn()), - "active_builds": active_builds, - "refresh_seconds": 30 if active_builds else None, - }, + "agentic_bundle.html", + {"title": bundle_id, "bundle": bundle}, ) - @app.get("/target/{target}", response_class=HTMLResponse) - def dashboard_target( + @app.get("/agentic/jobs", response_class=HTMLResponse) + def agentic_jobs( request: RequestType, - target: str, - status_filter: str = Query(default="all", alias="filter"), - q: str = "", - page: int = Query(default=1, ge=1), + target: str | None = None, + state: str | None = None, ) -> Any: - rows = get_port_status(_conn(), target=target) - query = q.strip().lower() - if status_filter == "failures": - rows = [row for row in rows if row.get("last_attempt_result") == "failure"] - elif status_filter == "successes": - rows = [row for row in rows if row.get("last_attempt_result") == "success"] - if query: - rows = [row for row in rows if query in str(row.get("origin", "")).lower()] - page_size = 100 - start = (page - 1) * page_size - page_rows = rows[start : start + page_size] - page_count = max(1, (len(rows) + page_size - 1) // page_size) + target_value = target or None + state_value = state or None + with _conn() as conn: + return templates.TemplateResponse( + request, + "agentic_jobs.html", + { + "title": "Jobs", + "jobs": list_jobs( + conn, state=state_value, target=target_value, limit=200 + ), + "target_options": distinct_targets(conn), + "selected_target": target_value, + "selected_state": state_value, + }, + ) + + @app.get("/agentic/jobs/{job_id}", response_class=HTMLResponse) + def agentic_job_detail(request: RequestType, job_id: str) -> Any: + with _conn() as conn: + job = get_job(conn, job_id) + activity = activity_for_job(conn, job_id) if job is not None else [] + if job is None: + raise HTTPException(status_code=404, detail=f"Unknown job: {job_id}") + return templates.TemplateResponse( + request, + "agentic_job.html", + {"title": job_id, "job": job, "activity": activity}, + ) + + @app.get("/agentic/runs/{run_id}", response_class=HTMLResponse) + def agentic_run_detail(request: RequestType, run_id: str) -> Any: + with _conn() as conn: + run = get_run(conn, run_id) + bundles = bundles_for_run(conn, run_id) if run is not None else [] + if run is None: + raise HTTPException(status_code=404, detail=f"Unknown run: {run_id}") return templates.TemplateResponse( request, - "target.html", + "agentic_run.html", + {"title": run_id, "run": run, "bundles": bundles}, + ) + + @app.get("/agentic/runner", response_class=HTMLResponse) + def agentic_runner(request: RequestType) -> Any: + with _conn() as conn: + return templates.TemplateResponse( + request, + "agentic_runner.html", + {"title": "Runner", "runner": runner_status(conn)}, + ) + + @app.get("/agentic/activity", response_class=HTMLResponse) + def agentic_activity( + request: RequestType, + target: str | None = None, + ) -> Any: + target_value = target or None + with _conn() as conn: + return templates.TemplateResponse( + request, + "agentic_activity.html", + { + "title": "Activity", + "activity": recent_activity(conn, limit=200, target=target_value), + "target_options": distinct_targets(conn), + "selected_target": target_value, + }, + ) + + # ------------------------------------------------------------------ + # Phase 5 step 1: dsynth-progress UI adapter. Lifts the + # www/example/progress.{html,js,css} UI and feeds it from tracker + # data via two JSON endpoints (summary + chunked history). No + # change to the existing /target/{target} dashboard yet. + # ------------------------------------------------------------------ + + # JSON endpoints live under /api/progress/{target}/ to stay clear + # of the legacy /target/{target}/{cat}/{port} catch-all. The HTML + # page is served at the canonical /target/{target} below. + + @app.get("/api/progress/{target}/summary.json") + def progress_summary(target: str) -> dict[str, Any]: + with _conn() as conn: + return target_summary(conn, target) + + @app.get("/api/progress/{target}/{chunk}_history.json") + def progress_history(target: str, chunk: str) -> Any: + try: + chunk_index = int(chunk) + except ValueError: + raise HTTPException(status_code=404, detail="Bad chunk index") + with _conn() as conn: + # Returns [] past the last chunk — kfiles in summary.json + # bounds the UI's fetch range so this is rarely hit. + return target_history_chunk(conn, target, chunk_index) + + @app.get("/api/progress/build/{run_id}/summary.json") + def progress_build_summary(run_id: int) -> dict[str, Any]: + with _conn() as conn: + summary = run_summary(conn, run_id) + if summary is None: + raise HTTPException(status_code=404, detail=f"Unknown build run: {run_id}") + return summary + + @app.get("/api/progress/build/{run_id}/{chunk}_history.json") + def progress_build_history(run_id: int, chunk: str) -> Any: + try: + chunk_index = int(chunk) + except ValueError: + raise HTTPException(status_code=404, detail="Bad chunk index") + with _conn() as conn: + return run_history_chunk(conn, run_id, chunk_index) + + @app.get("/target/{target}", response_class=HTMLResponse) + def dashboard_target(request: RequestType, target: str) -> Any: + # The page uses progress.{css,js} (lifted from dsynth-progress) + # and fetches data from /api/progress/{target}/. The tag + # pins those relative URLs to the canonical API root. + return templates.TemplateResponse( + request, + "progress.html", { "title": target, "target": target, - "rows": page_rows, - "status_filter": status_filter, - "query": q, - "page": page, - "page_count": page_count, - "page_size": page_size, - "total_rows": len(rows), + "progress_base": f"/api/progress/{target}/", }, ) @@ -335,22 +694,23 @@ def dashboard_port_detail( request: RequestType, target: str, cat: str, port: str ) -> Any: origin = f"{cat}/{port}" - rows = get_port_status(_conn(), target=target, origin=origin) - if not rows: - raise HTTPException( - status_code=404, detail=f"Unknown port status: {target} {origin}" + with _conn() as conn: + rows = get_port_status(conn, target=target, origin=origin) + if not rows: + raise HTTPException( + status_code=404, detail=f"Unknown port status: {target} {origin}" + ) + return templates.TemplateResponse( + request, + "port_detail.html", + { + "title": f"{origin} {target}", + "target": target, + "origin": origin, + "status": rows[0], + "history": get_port_history(conn, target, origin, limit=20), + }, ) - return templates.TemplateResponse( - request, - "port_detail.html", - { - "title": f"{origin} {target}", - "target": target, - "origin": origin, - "status": rows[0], - "history": get_port_history(_conn(), target, origin, limit=20), - }, - ) @app.get("/builds", response_class=HTMLResponse) def dashboard_builds( @@ -359,62 +719,52 @@ def dashboard_builds( build_type: str | None = None, limit: int = Query(default=50, ge=1, le=500), ) -> Any: - runs = list_build_runs( - _conn(), target=target, build_type=build_type, limit=limit - ) - compare_links = _resolve_compare_links(runs) - return templates.TemplateResponse( - request, - "builds.html", - { - "title": "Builds", - "runs": runs, - "compare_links": compare_links, - "target": target, - "build_type": build_type, - }, - ) + with _conn() as conn: + runs = list_build_runs( + conn, target=target, build_type=build_type, limit=limit + ) + compare_links = _resolve_compare_links(runs) + return templates.TemplateResponse( + request, + "builds.html", + { + "title": "Builds", + "runs": runs, + "compare_links": compare_links, + "target": target, + "build_type": build_type, + }, + ) @app.get("/builds/compare", response_class=HTMLResponse) def dashboard_build_compare(request: RequestType, a: int, b: int) -> Any: - return templates.TemplateResponse( - request, - "build_compare.html", - { - "title": "Build Compare", - "compare": compare_builds(_conn(), a, b), - }, - ) + with _conn() as conn: + return templates.TemplateResponse( + request, + "build_compare.html", + { + "title": "Build Compare", + "compare": compare_builds(conn, a, b), + }, + ) @app.get("/builds/{run_id}", response_class=HTMLResponse) - def dashboard_build_detail( - request: RequestType, - run_id: int, - status_filter: str = Query(default="all", alias="filter"), - ) -> Any: - payload = { - "build_run": get_build_run(_conn(), run_id), - "results": get_build_results(_conn(), run_id), - } - build = payload["build_run"] - results = payload["results"] - if status_filter == "failures": - results = [row for row in results if row.get("result") == "failure"] - elif status_filter == "successes": - results = [row for row in results if row.get("result") == "success"] - elif status_filter == "building": - results = [row for row in results if row.get("status") == "building"] - elif status_filter == "queued": - results = [row for row in results if row.get("status") == "queued"] + def dashboard_build_detail(request: RequestType, run_id: int) -> Any: + # Build detail uses the same dsynth-progress UI as /target/{target}, + # just scoped to one run_id. Verify the run exists so unknown + # IDs 404 here rather than at the JSON fetch. + try: + with _conn() as conn: + build = get_build_run(conn, run_id) + except ValueError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc return templates.TemplateResponse( request, - "build_detail.html", + "progress.html", { "title": f"Build {run_id}", - "build": build, - "results": results, - "status_filter": status_filter, - "refresh_seconds": 10 if build.get("finished_at") is None else None, + "target": f"{build['target']} (run {run_id})", + "progress_base": f"/api/progress/build/{run_id}/", }, ) @@ -424,19 +774,20 @@ def dashboard_diff( a: str | None = None, b: str | None = None, ) -> Any: - targets = get_target_summary(_conn()) - diff_payload = get_diff(_conn(), a, b) if a and b else None - return templates.TemplateResponse( - request, - "diff.html", - { - "title": "Target Diff", - "targets": targets, - "target_a": a, - "target_b": b, - "diff": diff_payload, - }, - ) + with _conn() as conn: + targets = get_target_summary(conn) + diff_payload = get_diff(conn, a, b) if a and b else None + return templates.TemplateResponse( + request, + "diff.html", + { + "title": "Target Diff", + "targets": targets, + "target_a": a, + "target_b": b, + "diff": diff_payload, + }, + ) return app diff --git a/scripts/generator/dportsv3/tracker/static/custom.css b/scripts/generator/dportsv3/tracker/static/custom.css deleted file mode 100644 index 45942fa960d..00000000000 --- a/scripts/generator/dportsv3/tracker/static/custom.css +++ /dev/null @@ -1,16 +0,0 @@ -:root { - --tracker-failure: #9b2226; - --tracker-success: #386641; -} - -.progress { - height: 8px; -} - -.status-row-failure td:first-child a { - color: var(--tracker-failure); -} - -.status-row-success td:first-child a { - color: var(--tracker-success); -} diff --git a/scripts/generator/dportsv3/tracker/static/dsynth.png b/scripts/generator/dportsv3/tracker/static/dsynth.png new file mode 100644 index 00000000000..f305ac5db5a Binary files /dev/null and b/scripts/generator/dportsv3/tracker/static/dsynth.png differ diff --git a/scripts/generator/dportsv3/tracker/static/favicon.png b/scripts/generator/dportsv3/tracker/static/favicon.png new file mode 100644 index 00000000000..a7c23c1ad4a Binary files /dev/null and b/scripts/generator/dportsv3/tracker/static/favicon.png differ diff --git a/scripts/generator/dportsv3/tracker/static/progress.css b/scripts/generator/dportsv3/tracker/static/progress.css new file mode 100644 index 00000000000..818b5896cd9 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/static/progress.css @@ -0,0 +1,736 @@ +*, *::before, *::after { + box-sizing: border-box; + margin: 0; + padding: 0; +} + +:root { + --amber: #c47000; + --amber-dim: #a05800; + --amber-border: rgba(180, 100, 0, 0.22); + --bg: #f7f5f0; + --bg-card: #ffffff; + --text: #1c1c1c; + --text-dim: #888070; + + --c-built: #2a7a2a; + --c-built-bg: #ecf7ec; + --c-failed: #c03030; + --c-failed-bg: #fdeaea; + --c-skipped: #b36800; + --c-skipped-bg: #fff5e6; + --c-ignored: #7a6000; + --c-ignored-bg: #fffbe6; + --c-meta: #6040b0; + --c-meta-bg: #f2eeff; + --c-neutral-bg: #eeebe4; +} + +body { + font-family: Menlo, Consolas, "Courier New", monospace; + font-size: 12px; + background: var(--bg); + color: var(--text); + margin: 0; + padding: 0; +} + +a { text-decoration: none; color: var(--amber); } +a:hover { color: var(--amber-dim); } + +/* ── Header ── */ + +#header { + position: sticky; + top: 0; + z-index: 20; + background: var(--bg-card); + border-bottom: 1px solid var(--amber-border); + box-shadow: 0 2px 16px rgba(0, 0, 0, 0.06); +} + +/* Row 1: logo + build info + stress */ + +#header-top { + border-bottom: 1px solid var(--amber-border); +} + +.header-top-inner { + display: flex; + align-items: center; + gap: 20px; + padding: 8px 24px; + flex-wrap: wrap; +} + +.header-spacer { flex: 1; } + +#logo { + flex-shrink: 0; + padding: 2px 0; +} + +#logo img { + display: block; + height: 40px; + width: auto; +} + +#build_info table { + border: 0; + border-collapse: collapse; + margin: 0; + font-size: 11px; +} + +#build_info th, +#build_info td { + border: 0; + padding: 1px 4px 1px 0; +} + +#build_info th { + text-align: right; + color: var(--text-dim); + font-weight: 500; + white-space: nowrap; +} + +#build_info td { + color: var(--amber); + min-width: 140px; +} + +/* Stress metrics */ + +#stress table { + border: 0; + border-collapse: separate; + border-spacing: 3px; + margin: 0; +} + +#stress table th, +#stress table td { + border: 0; + padding: 3px 10px; + border-radius: 3px; + text-align: center; +} + +#stress table thead th { + background: transparent; + color: var(--text-dim); + font-size: 10px; + letter-spacing: 0.07em; + text-transform: uppercase; + font-weight: 500; + padding-bottom: 2px; +} + +#stress table tbody td { + background: var(--c-neutral-bg); + color: var(--text); + font-size: 12px; + min-width: 60px; +} + +/* Load coloring */ +#stats_load.load-ok { background: var(--c-built-bg); color: var(--c-built); } +#stats_load.load-warn { background: var(--c-skipped-bg); color: var(--c-skipped); } +#stats_load.load-crit { background: var(--c-failed-bg); color: var(--c-failed); } + +/* Row 2: stat cards */ + +#header-stats { + display: flex; + align-items: stretch; + gap: 0; + padding: 0 16px; + overflow-x: auto; +} + +.stat-card { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 8px 18px 7px; + gap: 1px; + border-right: 1px solid var(--amber-border); + min-width: 68px; + transition: background 0.12s; +} + +.stat-card:first-child { border-left: 1px solid var(--amber-border); } + +.stat-card[onclick] { cursor: pointer; } +.stat-card[onclick]:hover { background: rgba(196, 112, 0, 0.06); } + +.stat-icon { + font-size: 14px; + line-height: 1; +} + +.stat-number { + font-size: 16px; + font-weight: 700; + line-height: 1.1; +} + +.stat-label { + font-size: 9px; + letter-spacing: 0.08em; + text-transform: uppercase; + color: var(--text-dim); + line-height: 1; +} + +/* Per-result card coloring */ +.stat-card[data-result="queued"] .stat-icon, +.stat-card[data-result="queued"] .stat-number { color: var(--text-dim); } + +.stat-card[data-result="built"] .stat-icon, +.stat-card[data-result="built"] .stat-number { color: var(--c-built); } + +.stat-card[data-result="meta"] .stat-icon, +.stat-card[data-result="meta"] .stat-number { color: var(--c-meta); } + +.stat-card[data-result="failed"] .stat-icon, +.stat-card[data-result="failed"] .stat-number { color: var(--c-failed); } + +.stat-card[data-result="ignored"] .stat-icon, +.stat-card[data-result="ignored"] .stat-number { color: var(--c-ignored); } + +.stat-card[data-result="skipped"] .stat-icon, +.stat-card[data-result="skipped"] .stat-number { color: var(--c-skipped); } + +.stat-card[data-result="remains"] .stat-icon, +.stat-card[data-result="remains"] .stat-number { color: var(--text-dim); } + +/* ── Main ── */ + +#main { + padding: 16px 24px 56px; +} + +/* ── Builders table ── */ + +#builders_zone_2 { + background: #111; + border-radius: 4px; + overflow: hidden; + border: 1px solid #252525; + margin-bottom: 20px; +} + +.builders_table { + font-family: Menlo, Consolas, "Courier New", monospace; + font-size: 12px; + border-collapse: collapse; + border-spacing: 0; + width: 100%; + margin: 0; + border: 0; +} + +.builders_table td { + border: 0; + padding: 4px 8px; +} + +.builders_table thead td { + background: #1a1a2e; + color: #8888aa; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.1em; + text-transform: uppercase; + padding: 6px 8px; +} + +.builders_table tbody td { + border-bottom: 1px solid #1c1c1c; +} + +.builders_table td:nth-child(1) { width: 60px; text-align: center; cursor: pointer; } +.builders_table td:nth-child(2) { width: 90px; color: #888; } +.builders_table td:nth-child(3) { width: 160px; color: #ffd060; } +.builders_table td:nth-child(4) { color: #66ddff; } +.builders_table td:nth-child(5) { width: 80px; text-align: right; padding-right: 14px; color: #777; } + +/* Builder status dot */ +.b-dot { + display: inline-block; + width: 7px; + height: 7px; + border-radius: 50%; + margin-right: 5px; + vertical-align: middle; + flex-shrink: 0; +} +.b-dot-idle { background: #444; } +.b-dot-active { background: #2a9a2a; box-shadow: 0 0 4px #2a9a2a; } + +/* ANSI palette */ +.b01,.b33{color:#FFF} +.b02,.b34{color:#9F9} +.b03,.b35{color:#FF9} +.b04,.b36{color:#F9F} +.b05,.b37{color:#F99} +.b06,.b38{color:#06F} +.b07,.b39{color:#0FF} +.b08,.b40{color:#808080} +.b09,.b41{color:#CCC} +.b10,.b42{color:#0C6} +.b11,.b43{color:#C96} +.b12,.b44{color:#C0C} +.b13,.b45{color:#C00} +.b14,.b46{color:#06F} +.b15,.b47{color:#099} +.b16,.b48{color:#adad85} +.b17,.b49{color:#FFF;text-decoration:underline} +.b18,.b50{color:#9F9;text-decoration:underline} +.b19,.b51{color:#FF9;text-decoration:underline} +.b20,.b52{color:#F9F;text-decoration:underline} +.b21,.b53{color:#F99;text-decoration:underline} +.b22,.b54{color:#06F;text-decoration:underline} +.b23,.b55{color:#0FF;text-decoration:underline} +.b24,.b56{color:#808080;text-decoration:underline} +.b25,.b57{color:#CCC;text-decoration:underline} +.b26,.b58{color:#0C6;text-decoration:underline} +.b27,.b59{color:#C96;text-decoration:underline} +.b28,.b60{color:#C0C;text-decoration:underline} +.b29,.b61{color:#C00;text-decoration:underline} +.b30,.b62{color:#06F;text-decoration:underline} +.b31,.b63{color:#099;text-decoration:underline} +.b32,.b64{color:#adad85;text-decoration:underline} + +/* ── Loading state ── */ + +#loading-state { + display: flex; + align-items: center; + gap: 8px; + font-size: 11px; + color: var(--text-dim); + padding: 4px 0 8px; + min-height: 24px; + letter-spacing: 0.04em; +} + +.spinner { + display: inline-block; + width: 11px; + height: 11px; + border: 2px solid var(--amber-border); + border-top-color: var(--amber); + border-radius: 50%; + animation: spin 0.7s linear infinite; + flex-shrink: 0; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +/* ── Preset tags ── */ + +#presets { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 10px; +} + +.preset-tag { + display: inline-block; + padding: 3px 10px; + border: 1px solid var(--amber-border); + border-radius: 20px; + font-size: 11px; + color: var(--text-dim); + cursor: pointer; + user-select: none; + transition: border-color 0.15s, color 0.15s, background 0.15s; +} + +.preset-tag:hover { + border-color: var(--amber); + color: var(--amber); +} + +.preset-tag.active { + background: var(--amber); + border-color: var(--amber); + color: #fff; +} + +/* ── Report toolbar ── */ + +#report-controls { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 10px; + gap: 12px; +} + +#search-input { + border: 1px solid var(--amber-border); + border-radius: 3px; + padding: 5px 10px; + font-family: inherit; + font-size: 12px; + background: var(--bg-card); + color: var(--text); + outline: none; + width: 340px; +} + +#search-input:focus { + border-color: var(--amber); + box-shadow: 0 0 0 2px rgba(196, 112, 0, 0.1); +} + +#pagination { + display: flex; + align-items: center; + gap: 10px; +} + +#page-info { + font-size: 11px; + color: var(--text-dim); + min-width: 100px; + text-align: center; +} + +#per-page { + border: 1px solid var(--amber-border); + border-radius: 3px; + background: var(--bg-card); + color: var(--text-dim); + padding: 3px 6px; + font-family: inherit; + font-size: 12px; + cursor: pointer; + margin-right: 6px; +} + +#per-page:focus { outline: none; border-color: var(--amber); } + +#page-prev, +#page-next { + border: 1px solid var(--amber-border); + border-radius: 3px; + background: var(--bg-card); + color: var(--text-dim); + padding: 3px 10px; + font-size: 13px; + cursor: pointer; + font-family: inherit; + transition: border-color 0.15s, color 0.15s; +} + +#page-prev:hover:not(:disabled), +#page-next:hover:not(:disabled) { + border-color: var(--amber); + color: var(--amber); +} + +#page-prev:disabled, +#page-next:disabled { + opacity: 0.35; + cursor: default; +} + +/* ── Report table ── */ + +#report { + margin: 0; +} + +#report_table { + border-collapse: collapse; + border-spacing: 0; + width: 100%; + border: 1px solid var(--amber-border); + border-radius: 4px; + overflow: hidden; + margin: 0; +} + +#report_table th { + background: var(--bg-card); + color: var(--text-dim); + font-size: 10px; + letter-spacing: 0.07em; + text-transform: uppercase; + padding: 8px 10px; + border: 0; + border-bottom: 1px solid var(--amber-border); + text-align: center; + white-space: nowrap; + user-select: none; +} + +#report_table th[data-col] { + cursor: pointer; +} + +#report_table th[data-col]:hover { + color: var(--amber); +} + +#report_table th.sort-asc::after { content: ' ↑'; color: var(--amber); } +#report_table th.sort-desc::after { content: ' ↓'; color: var(--amber); } + +#report_table td { + padding: 5px 8px; + border: 0; + border-bottom: 1px solid rgba(0, 0, 0, 0.04); + font-size: 12px; + text-align: center; + color: var(--text); +} + +#report_table .odd td { + background: rgba(180, 100, 0, 0.025); +} + +#report_table th:nth-child(1) { width: 5%; } +#report_table th:nth-child(2) { width: 7%; } +#report_table th:nth-child(3) { width: 4%; } +#report_table th:nth-child(4) { width: 7%; } +#report_table th:nth-child(5) { width: 30%; text-align: left; } +#report_table th:nth-child(6) { width: 30%; text-align: left; } +#report_table th:nth-child(7) { width: 5%; } +#report_table th:nth-child(8) { width: 7%; } + +#report_table td:nth-child(5), +#report_table td:nth-child(6) { + text-align: left; + padding-left: 8px; +} + +/* Result badges inside the report table */ +.result { + display: inline-block; + padding: 2px 8px; + border-radius: 3px; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.05em; + text-transform: uppercase; +} + +/* Result color classes — badges + stat card accent */ +.built { background: var(--c-built-bg); color: var(--c-built); } +.failed { background: var(--c-failed-bg); color: var(--c-failed); } +.skipped { background: var(--c-skipped-bg); color: var(--c-skipped); } +.ignored { background: var(--c-ignored-bg); color: var(--c-ignored); } +.meta { background: var(--c-meta-bg); color: var(--c-meta); } + +.entry { cursor: pointer; } +.timehack { color: var(--c-failed); } + +.dataTables_empty { + background: #fffbf0 !important; + color: var(--text-dim); + border: 0 !important; +} + +/* ── Footer ── */ + +#footer { + position: fixed; + bottom: 0; + left: 0; + right: 0; + height: 30px; + background: var(--bg-card); + border-top: 1px solid var(--amber-border); + z-index: 20; + display: flex; + align-items: center; + padding: 0 24px; + gap: 16px; +} + +#nav { + font-size: 11px; + color: var(--text-dim); + letter-spacing: 0.09em; + text-transform: uppercase; + flex-shrink: 0; +} + +#progress { + flex: 1; + height: 10px; + display: flex; + align-items: stretch; + border-radius: 3px; + overflow: hidden; + background: var(--c-neutral-bg); +} + +#progressbar-css { + display: flex; + width: 100%; + height: 100%; +} + +.pb-seg { + height: 100%; + min-width: 0; + transition: flex-grow 0.4s ease; +} + +.pb-built { background: var(--c-built); } +.pb-meta { background: var(--c-meta); } +.pb-failed { background: var(--c-failed); } +.pb-ignored { background: var(--c-ignored); } +.pb-skipped { background: var(--c-skipped); } + +@media (max-width: 880px) { + #stress { display: none; } + .header-top-inner { gap: 12px; } +} + +/* ── Tracker pages (index, agentic) lifted from dsynth-progress ── */ + +#tracker-nav { + background: var(--bg-card); + border-bottom: 1px solid var(--amber-border); + padding: 6px 24px; + font-size: 12px; + color: var(--text-dim); + display: flex; + align-items: center; + gap: 12px; +} +#tracker-nav a { color: var(--amber); } +#tracker-nav .nav-sep { color: var(--amber-border); } +#tracker-nav .nav-spacer { flex: 1; } + +.tracker-page { + padding: 16px 24px 40px; + max-width: 1400px; + margin: 0 auto; +} +.tracker-page h1 { + font-size: 18px; + font-weight: 600; + margin-bottom: 4px; +} +.tracker-page .lede { + color: var(--text-dim); + font-size: 12px; + margin-bottom: 20px; +} +.tracker-page h2 { + font-size: 14px; + font-weight: 600; + margin: 24px 0 8px; + color: var(--text-dim); +} + +.tracker-card { + background: var(--bg-card); + border: 1px solid var(--amber-border); + border-radius: 4px; + padding: 12px 16px; + margin-bottom: 16px; +} + +.target-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); + gap: 12px; +} +.target-card { + background: var(--bg-card); + border: 1px solid var(--amber-border); + border-radius: 4px; + padding: 12px 14px; + text-decoration: none; + color: inherit; + display: block; + transition: box-shadow .12s; +} +.target-card:hover { + box-shadow: 0 2px 12px rgba(0,0,0,0.08); + color: inherit; +} +.target-card .target-name { + font-size: 14px; + font-weight: 600; + color: var(--amber); + margin-bottom: 6px; +} +.target-card .target-meta { + color: var(--text-dim); + font-size: 11px; + margin-bottom: 8px; +} +.target-card .target-stats { + display: flex; + flex-wrap: wrap; + gap: 6px; +} +.target-card .stat-pill { + padding: 2px 6px; + border-radius: 3px; + font-size: 11px; + font-variant-numeric: tabular-nums; +} +.stat-pill.built { background: var(--c-built-bg); color: var(--c-built); } +.stat-pill.failed { background: var(--c-failed-bg); color: var(--c-failed); } +.stat-pill.skipped { background: var(--c-skipped-bg); color: var(--c-skipped); } +.stat-pill.ignored { background: var(--c-ignored-bg); color: var(--c-ignored); } +.stat-pill.total { background: var(--c-neutral-bg); color: var(--text); } + +table.tracker-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + background: var(--bg-card); +} +table.tracker-table th, +table.tracker-table td { + text-align: left; + padding: 6px 10px; + border-bottom: 1px solid var(--amber-border); + white-space: nowrap; +} +table.tracker-table th { + color: var(--text-dim); + font-weight: 600; + background: var(--bg); +} +table.tracker-table tbody tr:hover { background: var(--c-neutral-bg); } + +.progress-mini { + position: relative; + background: var(--c-neutral-bg); + border-radius: 2px; + height: 6px; + min-width: 80px; + overflow: hidden; +} +.progress-mini .pm-fill { + position: absolute; + top: 0; left: 0; bottom: 0; + background: var(--c-built); +} + +.empty-state { + color: var(--text-dim); + padding: 24px; + text-align: center; + background: var(--bg-card); + border: 1px dashed var(--amber-border); + border-radius: 4px; +} diff --git a/scripts/generator/dportsv3/tracker/static/progress.js b/scripts/generator/dportsv3/tracker/static/progress.js new file mode 100644 index 00000000000..01219b0502a --- /dev/null +++ b/scripts/generator/dportsv3/tracker/static/progress.js @@ -0,0 +1,459 @@ +var SbInterval = 10; // seconds between polls when active +var PER_PAGE = 50; + +var allRows = []; // raw entry objects accumulated from history files +var filteredRows = []; // subset after filter + sort +var currentPage = 0; +var filterText = ''; +var sortCol = 'entry'; +var sortDir = -1; // -1 desc, 1 asc (newest first by default) +var kfiles = 0; +var lastKfile = 0; +var run_active = false; + +/* ── Utilities ── */ + +function $(id) { return document.getElementById(id); } + +function setText(id, val) { + var el = $(id); + if (el) el.textContent = (val !== undefined && val !== null) ? String(val).trim() : ''; +} + +function esc(s) { + if (!s) return ''; + return String(s) + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function digit2(n) { return n > 9 ? '' + n : '0' + n; } + +function fetchJSON(url) { + return fetch(url).then(function(r) { + if (!r.ok) throw new Error(r.status); + return r.json(); + }); +} + +/* ── Loading state ── */ + +function setLoadingState(text) { + var el = $('loading-state'); + if (!el) return; + if (!text) { + el.innerHTML = ''; + } else { + el.innerHTML = '' + text; + } +} + +/* ── Progress bar (CSS segments) ── */ + +function updateProgress(stats) { + var q = stats.queued || 1; + + function setSeg(id, n) { + var el = $(id); + if (!el) return; + if (!n) { + el.style.flexGrow = 0; + el.style.display = 'none'; + } else { + el.style.display = ''; + el.style.flexGrow = n / q; + } + } + + setSeg('seg_built', stats.built); + setSeg('seg_meta', stats.meta); + setSeg('seg_failed', stats.failed); + setSeg('seg_ignored', stats.ignored); + setSeg('seg_skipped', stats.skipped); +} + +/* ── Header ── */ + +function updateSummary(data) { + kfiles = parseInt(data.kfiles) || 0; + run_active = parseInt(data.active) !== 0; + + setText('profile', data.profile); + setText('kickoff', data.kickoff); + setText('polling', run_active ? 'Active' : 'Complete'); + + if (data.stats) { + var s = data.stats; + setText('stats_queued', s.queued); + setText('stats_built', s.built); + setText('stats_meta', s.meta); + setText('stats_failed', s.failed); + setText('stats_ignored', s.ignored); + setText('stats_skipped', s.skipped); + setText('stats_remains', s.remains); + setText('stats_elapsed', s.elapsed); + setText('stats_pkghour', s.pkghour); + setText('stats_impulse', s.impulse); + setText('stats_swapinfo', s.swapinfo); + + updateLoad(s.load); + updateProgress(s); + } + + updateBuilders(data.builders || []); +} + +function updateLoad(loadStr) { + var el = $('stats_load'); + if (!el) return; + el.textContent = loadStr ? String(loadStr).trim() : ''; + var v = parseFloat(loadStr) || 0; + el.className = v >= 8 ? 'load-crit' : v >= 4 ? 'load-warn' : 'load-ok'; +} + +function updateBuilders(builders) { + var tbody = document.querySelector('#builders_body tbody'); + if (!tbody) return; + var html = ''; + for (var i = 0; i < builders.length; i++) { + var b = builders[i]; + var idle = (b.phase === 'Idle' || !b.phase); + var dot = ''; + html += + '' + + '' + dot + esc(b.ID) + '' + + '' + esc(b.elapsed) + '' + + '' + esc(b.phase) + '' + + '' + esc(b.origin) + '' + + '' + esc(b.lines) + '' + + ''; + } + tbody.innerHTML = html; +} + +/* ── Search: advanced query parser ── */ +/* + * Supported syntax (space-separated, all terms ANDed): + * graphics plain term — matches origin or result + * result:failed field prefix — matches specific field + * origin:lang field prefix + * id:03 field prefix + */ + +function parseQuery(query) { + var terms = []; + var parts = query.trim().split(/\s+/); + for (var i = 0; i < parts.length; i++) { + var p = parts[i]; + if (!p) continue; + var colon = p.indexOf(':'); + if (colon > 0) { + terms.push({ field: p.slice(0, colon), value: p.slice(colon + 1).toLowerCase() }); + } else { + terms.push({ field: null, value: p.toLowerCase() }); + } + } + return terms; +} + +function matchRow(d, terms) { + for (var i = 0; i < terms.length; i++) { + var t = terms[i]; + var v = t.value; + if (t.field === 'result') { + if (d.result.toLowerCase().indexOf(v) < 0) return false; + } else if (t.field === 'origin') { + if (d.origin.toLowerCase().indexOf(v) < 0) return false; + } else if (t.field === 'id') { + if (String(d.ID).toLowerCase().indexOf(v) < 0) return false; + } else if (t.field === 'phase') { + var phase = (d.info || '').split(':')[0].toLowerCase(); + if (phase.indexOf(v) < 0) return false; + } else { + var idStr = '[' + d.ID + ']'; + if (d.origin.toLowerCase().indexOf(v) < 0 && + d.result.toLowerCase().indexOf(v) < 0 && + idStr.toLowerCase().indexOf(v) < 0) return false; + } + } + return true; +} + +/* ── Sort ── */ + +function sortBy(col) { + if (sortCol === col) { + sortDir = -sortDir; + } else { + sortCol = col; + sortDir = (col === 'entry' || col === 'skip') ? -1 : 1; + } + currentPage = 0; + sortRows(); + renderTable(); + renderPagination(); + updateSortHeaders(); +} + +function sortRows() { + filteredRows.sort(function(a, b) { + if (sortCol === 'entry' || sortCol === 'skip') { + return sortDir * ((+a[sortCol] || 0) - (+b[sortCol] || 0)); + } + var av = a[sortCol] || ''; + var bv = b[sortCol] || ''; + return sortDir * String(av).localeCompare(String(bv)); + }); +} + +function updateSortHeaders() { + var ths = document.querySelectorAll('#report_table thead th[data-col]'); + for (var i = 0; i < ths.length; i++) { + var th = ths[i]; + th.classList.remove('sort-asc', 'sort-desc'); + if (th.getAttribute('data-col') === sortCol) { + th.classList.add(sortDir === 1 ? 'sort-asc' : 'sort-desc'); + } + } +} + +/* ── Filter + render ── */ + +var activePreset = null; + +function preset(txt, col) { + activePreset = txt + '|' + (col || ''); + if (col) { + sortCol = col; + sortDir = -1; + } + filter(txt); +} + +function filter(txt) { + var input = $('search-input'); + if (input) input.value = txt; + document.querySelectorAll('.preset-tag').forEach(function(el) { + var key = el.dataset.filter + '|' + (el.dataset.sort || ''); + el.classList.toggle('active', key === activePreset); + }); + activePreset = null; + filterText = txt.toLowerCase(); + currentPage = 0; + applyFilter(); + renderTable(); + renderPagination(); +} + +function applyFilter() { + var source; + if (!filterText) { + source = allRows; + } else { + var terms = parseQuery(filterText); + source = allRows.filter(function(d) { return matchRow(d, terms); }); + } + filteredRows = source.slice(); + sortRows(); +} + +var renderTimer = null; +var lastRender = 0; +var RENDER_INTERVAL = 500; // ms — max render frequency during loading + +function doRender() { + clearTimeout(renderTimer); + renderTimer = null; + lastRender = Date.now(); + applyFilter(); + renderTable(); + renderPagination(); +} + +function computeSkip(d) { + if (d.result === 'failed') { + var p = (d.info || '').split(':'); + return parseInt(p[1]) || 0; + } + if (d.result === 'ignored') { + var p = (d.info || '').split(':|:'); + return parseInt(p[1]) || 0; + } + return 0; +} + +function addHistoryData(data) { + for (var i = 0; i < data.length; i++) { + var d = data[i]; + d.skip = computeSkip(d); + allRows.push(d); + } + var due = RENDER_INTERVAL - (Date.now() - lastRender); + if (due <= 0) { + doRender(); + } else { + clearTimeout(renderTimer); + renderTimer = setTimeout(doRender, due); + } +} + +/* ── Table render ── */ + +var RESULT_GLYPHS = { + built: '✓', + failed: '✗', + meta: '◈', + ignored: '⊘', + skipped: '⇢' +}; + +function logfile(origin) { + var p = origin.split('/'); + return '../' + p[0] + '___' + (p[1] || '') + '.log'; +} + +function infoHTML(result, origin, info) { + if (result === 'meta') return 'meta-node complete.'; + if (result === 'built') return 'logfile'; + if (result === 'failed') return 'Failed ' + esc((info || '').split(':')[0]) + ' phase (logfile)'; + if (result === 'skipped') return 'Issue with ' + esc(info); + if (result === 'ignored') return esc((info || '').split(':|:')[0]); + return ''; +} + +function skipHTML(d) { + if (d.result === 'failed' || d.result === 'ignored') return String(d.skip); + return ''; +} + +function originLink(origin) { + var p = origin.split('/'); + var name = p[1] ? p[1].split('@')[0] : origin; + return '' + esc(origin) + ''; +} + +function renderTable() { + var tbody = $('report_body'); + var start = currentPage * PER_PAGE; + var rows = filteredRows.slice(start, start + PER_PAGE); + var html = ''; + + for (var i = 0; i < rows.length; i++) { + var d = rows[i]; + var cls = (i % 2 === 1) ? ' class="odd"' : ''; + var glyph = RESULT_GLYPHS[d.result] || ''; + html += + '' + + '' + d.entry + '' + + '' + esc(d.elapsed) + '' + + '[' + esc(d.ID) + ']' + + '
' + glyph + ' ' + d.result + '
' + + '' + originLink(d.origin) + '' + + '' + infoHTML(d.result, d.origin, d.info) + '' + + '' + skipHTML(d) + '' + + '' + esc(d.duration) + '' + + ''; + } + tbody.innerHTML = html; + updateSortHeaders(); +} + +function renderPagination() { + var total = filteredRows.length; + var pages = Math.ceil(total / PER_PAGE); + var info = $('page-info'); + var prev = $('page-prev'); + var next = $('page-next'); + if (!info) return; + + if (total === 0) { + info.textContent = 'No results'; + } else { + var s = currentPage * PER_PAGE + 1; + var e = Math.min((currentPage + 1) * PER_PAGE, total); + info.textContent = s + '–' + e + ' of ' + total; + } + prev.disabled = currentPage === 0; + next.disabled = currentPage >= pages - 1; +} + +function setPerPage(n) { + PER_PAGE = parseInt(n); + currentPage = 0; + renderTable(); + renderPagination(); +} + +function prevPage() { + if (currentPage > 0) { + currentPage--; + renderTable(); + renderPagination(); + } +} + +function nextPage() { + if (currentPage < Math.ceil(filteredRows.length / PER_PAGE) - 1) { + currentPage++; + renderTable(); + renderPagination(); + } +} + +/* ── Fetch loop ── */ + +var historyRunning = false; + +async function loadNewHistory() { + if (historyRunning) return; + historyRunning = true; + + var total = kfiles; + var start = lastKfile + 1; + + if (start > total) { historyRunning = false; return; } + + var count = total - start + 1; + setLoadingState('Loading build logs (' + count + ' file' + (count > 1 ? 's' : '') + ')'); + + var keys = []; + var fetches = []; + for (var k = start; k <= total; k++) { + keys.push(k); + fetches.push(fetchJSON(digit2(k) + '_history.json').catch(function() { return null; })); + } + + var results = await Promise.all(fetches); + + for (var i = 0; i < results.length; i++) { + if (results[i] === null) break; // gap or not-yet-written file — stop, retry next poll + addHistoryData(results[i]); + lastKfile = keys[i]; + } + + setLoadingState(''); + historyRunning = false; +} + +async function pollSummary() { + try { + var data = await fetchJSON('summary.json'); + updateSummary(data); + loadNewHistory(); // fire-and-forget: runs concurrently with next summary poll + if (run_active) { + setTimeout(pollSummary, SbInterval * 1000); + } else { + var zone = $('builders_zone_2'); + if (zone) zone.style.display = 'none'; + } + } catch (e) { + setTimeout(pollSummary, SbInterval * 500); + } +} + +document.addEventListener('DOMContentLoaded', pollSummary); + +document.addEventListener('keydown', function(e) { + if (e.key === 'F5') e.preventDefault(); +}); diff --git a/scripts/generator/dportsv3/tracker/templates/_target_filter.html b/scripts/generator/dportsv3/tracker/templates/_target_filter.html new file mode 100644 index 00000000000..2cdd2707503 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/_target_filter.html @@ -0,0 +1,25 @@ +{# Inline target+origin filter form used by agentic list pages. #} +
+ + + {% if show_state %} + + + {% endif %} + {% if show_origin %} + + + + {% endif %} +
diff --git a/scripts/generator/dportsv3/tracker/templates/_tracker_nav.html b/scripts/generator/dportsv3/tracker/templates/_tracker_nav.html new file mode 100644 index 00000000000..eb81de92691 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/_tracker_nav.html @@ -0,0 +1,20 @@ +{# Top nav bar shared by tracker pages. Expects `request` in context. #} +
+ Tracker + {% if nav_crumbs %} + {% for crumb in nav_crumbs %} + / + {% if crumb.url %} + {{ crumb.label }} + {% else %} + {{ crumb.label }} + {% endif %} + {% endfor %} + {% endif %} + + Builds + · + Diff + · + Agentic +
diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_activity.html b/scripts/generator/dportsv3/tracker/templates/agentic_activity.html new file mode 100644 index 00000000000..54b4cf910fc --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_activity.html @@ -0,0 +1,39 @@ + + + + + + Activity — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Activity"}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

Activity log

+

Recent agentic stage transitions.

+ + {% include "_target_filter.html" %} + + + + + {% for a in activity %} + + + + + + + + {% else %} + + {% endfor %} + +
WhenJobStageDurationMessage
{{ a.ts }}{% if a.job_id %}{{ a.job_id }}{% else %}—{% endif %}{{ a.stage or '—' }}{{ a.duration_ms if a.duration_ms is not none else '—' }}{{ a.message or '—' }}
No activity yet.
+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_bundle.html b/scripts/generator/dportsv3/tracker/templates/agentic_bundle.html new file mode 100644 index 00000000000..9f95e3fb1e2 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_bundle.html @@ -0,0 +1,48 @@ + + + + + + {{ bundle.bundle_id }} — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Bundles", "url": request.url_for('agentic_bundles')}, {"label": bundle.bundle_id}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

{{ bundle.bundle_id }}

+ +
+ + + + + + +
Origin{{ bundle.origin }}{% if bundle.flavor %} @ {{ bundle.flavor }}{% endif %}
Target{{ bundle.target or '—' }}
Result{{ bundle.result }}
Timestamp{{ bundle.ts_utc }}
Run{% if bundle.run_id %}{{ bundle.run_id }}{% else %}—{% endif %}
+
+ +

Artifacts

+ {% if bundle.artifacts %} + + + + {% for a in bundle.artifacts %} + + + + + + + {% endfor %} + +
PathBackendKindSize
{{ a.relpath }}{{ a.backend }}{{ a.kind or '—' }}{{ a.size if a.size is not none else '—' }}
+ {% else %} +
No artifacts recorded for this bundle.
+ {% endif %} +
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_bundles.html b/scripts/generator/dportsv3/tracker/templates/agentic_bundles.html new file mode 100644 index 00000000000..4fc0115e52a --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_bundles.html @@ -0,0 +1,43 @@ + + + + + + Bundles — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Bundles"}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

Bundles

+

Failure-evidence bundles uploaded by dsynth hooks.

+ + {% with show_origin = True %}{% include "_target_filter.html" %}{% endwith %} + + + + + + + {% for b in bundles %} + + + + + + + + + + {% else %} + + {% endfor %} + +
BundleOriginFlavorTargetResultWhenRun
{{ b.bundle_id }}{{ b.origin }}{{ b.flavor or '—' }}{{ b.target or '—' }}{{ b.result }}{{ b.ts_utc }}{% if b.run_id %}{{ b.run_id }}{% else %}—{% endif %}
No bundles match this filter.
+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_index.html b/scripts/generator/dportsv3/tracker/templates/agentic_index.html new file mode 100644 index 00000000000..863115f4a4d --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_index.html @@ -0,0 +1,82 @@ + + + + + + Agentic — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic"}] %}{% include "_tracker_nav.html" %}{% endwith %} + +
+

Agentic

+

Failure bundles, jobs, and runner state across targets.

+ +
+
+
Bundles
+
{{ status.bundles }}
+
+
+
Runs
+
{{ status.runs }}
+
+
+
Jobs pending / inflight
+
{{ status.jobs.pending }} / {{ status.jobs.inflight }}
+
+
+
Jobs done / failed
+
{{ status.jobs.done }} / {{ status.jobs.failed }}
+
+
+ +

Recent bundles

+ + + + {% for b in recent_bundles %} + + + + + + + {% else %} + + {% endfor %} + +
BundleOriginTargetResult
{{ b.bundle_id }}{{ b.origin }}{{ b.target or '—' }}{{ b.result }}
No bundles yet.
+

All bundles →

+ +

Recent jobs

+ + + + {% for j in recent_jobs %} + + + + + + + {% else %} + + {% endfor %} + +
JobStateOriginTarget
{{ j.job_id }} + {% set cls = {'pending':'ignored','inflight':'skipped','done':'built','failed':'failed'}.get(j.state,'total') %} + {{ j.state }} + {{ j.origin or '—' }}{{ j.target or '—' }}
No jobs yet.
+

+ All jobs → +  ·  + Runner status +  ·  + Activity log +

+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_job.html b/scripts/generator/dportsv3/tracker/templates/agentic_job.html new file mode 100644 index 00000000000..b8874624eb5 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_job.html @@ -0,0 +1,51 @@ + + + + + + {{ job.job_id }} — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Jobs", "url": request.url_for('agentic_jobs')}, {"label": job.job_id}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

{{ job.job_id }}

+ +
+ + + + + + + + + {% if job.last_error %} + + {% endif %} +
State{{ job.state }}
Type{{ job.type or '—' }}
Origin{{ job.origin or '—' }}{% if job.flavor %} @ {{ job.flavor }}{% endif %}
Target{{ job.target or '—' }}
Created{{ job.created_ts_utc or '—' }}
Bundle dir{{ job.bundle_dir or '—' }}
Path{{ job.path or '—' }}
Last error
{{ job.last_error }}
+
+ + {% if activity %} +

Activity

+ + + + {% for a in activity %} + + + + + + + {% endfor %} + +
WhenStageDurationMessage
{{ a.ts }}{{ a.stage or '—' }}{{ a.duration_ms if a.duration_ms is not none else '—' }}{{ a.message or '—' }}
+ {% endif %} +
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_jobs.html b/scripts/generator/dportsv3/tracker/templates/agentic_jobs.html new file mode 100644 index 00000000000..8048add67a2 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_jobs.html @@ -0,0 +1,45 @@ + + + + + + Jobs — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Jobs"}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

Jobs

+

Triage + patch jobs in the agentic queue.

+ + {% with show_state = True %}{% include "_target_filter.html" %}{% endwith %} + + + + + + + {% for j in jobs %} + + + + + + + + + {% else %} + + {% endfor %} + +
JobStateTypeOriginTargetCreated
{{ j.job_id }} + {% set cls = {'pending':'ignored','inflight':'skipped','done':'built','failed':'failed'}.get(j.state,'total') %} + {{ j.state }} + {{ j.type or '—' }}{{ j.origin or '—' }}{{ j.target or '—' }}{{ j.created_ts_utc or '—' }}
No jobs match this filter.
+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_run.html b/scripts/generator/dportsv3/tracker/templates/agentic_run.html new file mode 100644 index 00000000000..ec5e0e05d97 --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_run.html @@ -0,0 +1,50 @@ + + + + + + {{ run.run_id }} — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": run.run_id}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

{{ run.run_id }}

+ +
+ + + + + + + +
Profile{{ run.profile or '—' }}
Target{{ run.target or '—' }}
Started{{ run.ts_start or '—' }}
Ended{{ run.ts_end or '—' }}
Last seen{{ run.last_seen_at or '—' }}
Path{{ run.path or '—' }}
+
+ +

Bundles in this run

+ {% if bundles %} + + + + {% for b in bundles %} + + + + + + + + {% endfor %} + +
BundleOriginTargetResultWhen
{{ b.bundle_id }}{{ b.origin }}{{ b.target or '—' }}{{ b.result }}{{ b.ts_utc }}
+ {% else %} +
No bundles recorded for this run.
+ {% endif %} +
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/agentic_runner.html b/scripts/generator/dportsv3/tracker/templates/agentic_runner.html new file mode 100644 index 00000000000..6bd193c427a --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/agentic_runner.html @@ -0,0 +1,33 @@ + + + + + + Runner — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Agentic", "url": request.url_for('agentic_index')}, {"label": "Runner"}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + +
+

Runner status

+

Heartbeat from the agent queue runner.

+ +
+ + + + + + + {% if runner.extra_json %} + + {% endif %} +
Status{{ runner.status }}
Current job{% if runner.job_id %}{{ runner.job_id }}{% else %}—{% endif %}
Stage{{ runner.current_stage or '—' }}
Started at{{ runner.started_at or '—' }}
Updated at{{ runner.updated_at or '—' }}
Extra
{{ runner.extra_json }}
+
+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/base.html b/scripts/generator/dportsv3/tracker/templates/base.html deleted file mode 100644 index 97adb415b0b..00000000000 --- a/scripts/generator/dportsv3/tracker/templates/base.html +++ /dev/null @@ -1,66 +0,0 @@ - - - - - - {{ title }} - DeltaPorts Tracker - {% if refresh_seconds %} - - {% endif %} - - - - - - - -
- {% block content %}{% endblock %} -
- - - - diff --git a/scripts/generator/dportsv3/tracker/templates/build_compare.html b/scripts/generator/dportsv3/tracker/templates/build_compare.html index 69763b384ba..30de09edbf0 100644 --- a/scripts/generator/dportsv3/tracker/templates/build_compare.html +++ b/scripts/generator/dportsv3/tracker/templates/build_compare.html @@ -1,66 +1,59 @@ -{% extends "base.html" %} -{% set active_page = "builds" %} + + + + + + Build compare — DeltaPorts Tracker + + + + + + {% with nav_crumbs = [{"label": "Builds", "url": request.url_for('dashboard_builds')}, {"label": "Compare"}] %} + {% include "_tracker_nav.html" %} + {% endwith %} -{% block content %} -

Build Compare

-

{{ compare.run_a.target }} {{ compare.run_a.build_type }} run {{ compare.run_a.id }} vs {{ compare.run_b.target }} {{ compare.run_b.build_type }} run {{ compare.run_b.id }}

+
+

Build compare

+

{{ compare.run_a.target }} {{ compare.run_a.build_type }} run {{ compare.run_a.id }} vs {{ compare.run_b.target }} {{ compare.run_b.build_type }} run {{ compare.run_b.id }}

-
-
- - - - - - - - +
+
BucketCount
{% for key, value in compare.summary.items() %} - - + + {% endfor %} - -
{{ key }}{{ value }}{{ key }}{{ value }}
-
-
- -
- {% for section in ['new_failures', 'new_successes', 'still_failing', 'added', 'removed', 'version_changes'] %} -
-

- -

-
-
-
- - - - - - - - - - {% for row in compare[section] %} - - - - - - {% endfor %} - -
OriginRun ARun B
{{ row.origin }}{{ row.result_a or '-' }} {{ row.version_a or '-' }}{{ row.result_b or '-' }} {{ row.version_b or '-' }}
-
-
+
+ + {% for section in ['new_failures', 'new_successes', 'still_failing', 'added', 'removed', 'version_changes'] %} +
+ {{ section }}({{ compare[section]|length }}) + + + + + + {% for row in compare[section] %} + + + + + + {% else %} + + {% endfor %} + +
OriginRun ARun B
{{ row.origin }}{{ row.result_a or '—' }} {{ row.version_a or '' }}{{ row.result_b or '—' }} {{ row.version_b or '' }}
none
+
+ {% endfor %}
- {% endfor %} -
-{% endblock %} + + diff --git a/scripts/generator/dportsv3/tracker/templates/build_detail.html b/scripts/generator/dportsv3/tracker/templates/build_detail.html deleted file mode 100644 index 8b9aa7d1263..00000000000 --- a/scripts/generator/dportsv3/tracker/templates/build_detail.html +++ /dev/null @@ -1,86 +0,0 @@ -{% extends "base.html" %} -{% set active_page = "builds" %} - -{% block content %} -

Build {{ build.id }}

-

{{ build.target }} / {{ build.build_type }}

- -
-
-
-
Started
-
{{ build.started_at }}
- -
Finished
-
{{ build.finished_at or 'running' }}
- -
Results
-
{{ build.success_count }} success, {{ build.failure_count }} failure, {{ build.result_count }} recorded
- - {% if build.commit_sha %} -
Commit
-
{{ build.commit_sha }} on {{ build.commit_branch or '-' }}, pushed {{ build.commit_pushed_at or '-' }}
- {% endif %} -
- {% if build.queued_count or build.building_count %} - {% set done = build.success_count + build.failure_count + build.skipped_count + build.ignored_count %} - {% set total = build.total_expected or build.result_count %} -
-
-
-
-
- {{ done }} / {{ total }} -
- {{ build.building_count }} building, {{ build.queued_count }} queued, {{ build.success_count }} success, {{ build.failure_count }} failure - {% endif %} -
-
- -
-
- - -
-
- -
-
- -
- - - - - - - - - - - - {% for row in results %} - - - - - - - - {% endfor %} - -
OriginVersionResultLogRecorded
{{ row.origin }}{{ row.version }} - {% if row.status in ('building', 'queued') %} - {{ row.status }} - {% else %} - {{ row.result }} - {% endif %} - {% if row.log_url %}View log{% else %}-{% endif %}{{ row.recorded_at }}
-
-{% endblock %} diff --git a/scripts/generator/dportsv3/tracker/templates/builds.html b/scripts/generator/dportsv3/tracker/templates/builds.html index 9bdd812401f..ea5f070b043 100644 --- a/scripts/generator/dportsv3/tracker/templates/builds.html +++ b/scripts/generator/dportsv3/tracker/templates/builds.html @@ -1,51 +1,62 @@ -{% extends "base.html" %} -{% set active_page = "builds" %} + + + + + + Builds — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Builds"}] %}{% include "_tracker_nav.html" %}{% endwith %} -{% block content %} -

Builds

-

Recent build runs across targets and build types.

+
+

Builds

+

Recent build runs across targets and build types.

-
- - - - - - - - - - - - - - {% for run in runs %} - - - - - - - - - - {% endfor %} - -
RunTargetTypeStartedFinishedPass/FailCompare
{{ run.id }}{{ run.target }}{{ run.build_type }}{{ run.started_at }} - {% if run.finished_at %} - {{ run.finished_at }} - {% else %} - running{% if run.total_expected %} {{ run.success_count + run.failure_count + run.skipped_count + run.ignored_count }}/{{ run.total_expected }}{% endif %} - {% endif %} - - {{ run.success_count }} - {{ run.failure_count }} - - {% if compare_links.get(run.id) %} - vs {{ compare_links[run.id] }} - {% else %} - - - {% endif %} -
-
-{% endblock %} + + + + + + + + + + + + + + {% for run in runs %} + + + + + + + + + + {% else %} + + {% endfor %} + +
RunTargetTypeStartedFinishedPass / FailCompare
{{ run.id }}{{ run.target }}{{ run.build_type }}{{ run.started_at }} + {% if run.finished_at %} + {{ run.finished_at }} + {% else %} + running{% if run.total_expected %} {{ run.success_count + run.failure_count + run.skipped_count + run.ignored_count }}/{{ run.total_expected }}{% endif %} + {% endif %} + + {{ run.success_count }} + {{ run.failure_count }} + + {% if compare_links.get(run.id) %} + vs {{ compare_links[run.id] }} + {% else %} + — + {% endif %} +
No build runs yet.
+
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/diff.html b/scripts/generator/dportsv3/tracker/templates/diff.html index 32b1a34746e..af63518ed5c 100644 --- a/scripts/generator/dportsv3/tracker/templates/diff.html +++ b/scripts/generator/dportsv3/tracker/templates/diff.html @@ -1,81 +1,75 @@ -{% extends "base.html" %} -{% set active_page = "diff" %} + + + + + + Diff — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": "Diff"}] %}{% include "_tracker_nav.html" %}{% endwith %} -{% block content %} -

Target Diff

-

Compare current status across two targets.

+
+

Target diff

+

Compare current status across two targets.

-
-
- - -
-
- - -
-
- -
-
+
+
+
Target A
+ +
+
+
Target B
+ +
+ +
-{% if diff %} -

Different

-
- - - - - - - - - - {% for row in diff.differ %} - - - - - - {% endfor %} - {% for row in diff.only_a %} - - - - - - {% endfor %} - {% for row in diff.only_b %} - - - - - - {% endfor %} - -
Origin{{ target_a }}{{ target_b }}
{{ row.origin }} - {{ row.result_a }} - {{ row.version_a }} - - {{ row.result_b }} - {{ row.version_b }} -
{{ row.origin }} - {{ row.result }} - {{ row.version }} - -
{{ row.origin }}- - {{ row.result }} - {{ row.version }} -
-
-{% endif %} -{% endblock %} + {% if diff %} +

Different

+ + + + + + {% for row in diff.differ %} + + + + + + {% endfor %} + {% for row in diff.only_a %} + + + + + + {% endfor %} + {% for row in diff.only_b %} + + + + + + {% endfor %} + {% if not diff.differ and not diff.only_a and not diff.only_b %} + + {% endif %} + +
Origin{{ target_a }}{{ target_b }}
{{ row.origin }}{{ row.result_a }} {{ row.version_a }}{{ row.result_b }} {{ row.version_b }}
{{ row.origin }}{{ row.result }} {{ row.version }}—
{{ row.origin }}—{{ row.result }} {{ row.version }}
No differences between the selected targets.
+ {% endif %} +
+ + diff --git a/scripts/generator/dportsv3/tracker/templates/index.html b/scripts/generator/dportsv3/tracker/templates/index.html index bd11dbefa53..be1803a5ee0 100644 --- a/scripts/generator/dportsv3/tracker/templates/index.html +++ b/scripts/generator/dportsv3/tracker/templates/index.html @@ -1,15 +1,24 @@ -{% extends "base.html" %} -{% set active_page = "targets" %} + + + + + + DeltaPorts Tracker + + + {% if refresh_seconds %}{% endif %} + + -{% block content %} -

Targets

-

Current per-target status and most recent build activity.

+ {% include "_tracker_nav.html" %} -{% if active_builds %} -
-
Active Builds
-
- +
+

Targets

+

Per-target build status. Click a target for the live progress view.

+ + {% if active_builds %} +

Active builds

+
@@ -22,58 +31,54 @@

Targets

{% for ab in active_builds %} + {% set total = ab.total_expected or 0 %} + {% set done = ab.done_count %} + {% set pct = (done * 100 // total) if total else 0 %} - - + + {% endfor %}
Target
{{ ab.target }} {{ ab.build_type }} {{ ab.started_at }} - {% set done = ab.done_count %} - {% set total = ab.total_expected or 0 %} -
-
-
-
- {{ done }} / {{ total or '?' }} +
+
+ {{ done }} / {{ total or '?' }}
{% if ab.building_count %}{{ ab.building_count }}{% else %}-{% endif %}{% if ab.failure_count %}{{ ab.failure_count }}{% else %}0{% endif %} + {% if ab.building_count %}{{ ab.building_count }}{% else %}—{% endif %} + + {% if ab.failure_count %}{{ ab.failure_count }}{% else %}0{% endif %} +
-
-
-{% endif %} + {% endif %} -
- - - - - - - - - - - +

All targets

+ {% if targets %} +
{% for row in targets %} -
- - - - - - + +
{{ row.target }}
+
+ {% if row.last_build_at %}last build {{ row.last_build_at }}{% else %}no builds yet{% endif %} +
+
+ Σ {{ row.total_ports }} + {% if row.successes %}✓ {{ row.successes }}{% endif %} + {% if row.failures %}✗ {{ row.failures }}{% endif %} + {% if row.skipped %}⇢ {{ row.skipped }}{% endif %} + {% if row.ignored %}⊘ {{ row.ignored }}{% endif %} +
+ {% endfor %} - -
TargetTotalPassFailLast Build
{{ row.target }}{{ row.total_ports }}{{ row.successes }}{{ row.failures }} - {% if row.last_build_id %} - {{ row.last_build_at }} - {% else %} - - - {% endif %} -
-
-{% endblock %} +
+ {% else %} +
No targets recorded yet.
+ {% endif %} + + + + diff --git a/scripts/generator/dportsv3/tracker/templates/port_detail.html b/scripts/generator/dportsv3/tracker/templates/port_detail.html index a95333d8bac..ae040c327fc 100644 --- a/scripts/generator/dportsv3/tracker/templates/port_detail.html +++ b/scripts/generator/dportsv3/tracker/templates/port_detail.html @@ -1,55 +1,53 @@ -{% extends "base.html" %} -{% set active_page = "targets" %} + + + + + + {{ origin }} — DeltaPorts Tracker + + + + + {% with nav_crumbs = [{"label": target, "url": request.url_for('dashboard_target', target=target)}, {"label": origin}] %} + {% include "_tracker_nav.html" %} + {% endwith %} -{% block content %} -

{{ origin }}

-

{{ target }}

+
+

{{ origin }}

+

{{ target }}

-
-
-
-
Current result
-
- {% set r = status.last_attempt_result or 'unknown' %} - {{ r }} -
+
+ {% set r = status.last_attempt_result or 'unknown' %} + + + + + + + +
Current result{{ r }}
Last attempt{{ status.last_attempt_version or '—' }} in run {{ status.last_attempt_run_id or '—' }} at {{ status.last_attempt_at or '—' }}
Last success{{ status.last_success_version or '—' }} in run {{ status.last_success_run_id or '—' }} at {{ status.last_success_at or '—' }}
+
-
Last attempt
-
{{ status.last_attempt_version or '-' }} in run {{ status.last_attempt_run_id or '-' }} at {{ status.last_attempt_at or '-' }}
- -
Last success
-
{{ status.last_success_version or '-' }} in run {{ status.last_success_run_id or '-' }} at {{ status.last_success_at or '-' }}
-
+

Recent history

+ + + + + + {% for row in history %} + + + + + + + + + {% else %} + + {% endfor %} + +
RunTypeVersionResultLogRecorded
{{ row.build_run_id }}{{ row.build_type }}{{ row.version }}{{ row.result }}{% if row.log_url %}View log{% else %}—{% endif %}{{ row.recorded_at }}
No build history for this origin.
-
- -

Recent History

-
- - - - - - - - - - - - - {% for row in history %} - - - - - - - - - {% endfor %} - -
RunTypeVersionResultLogRecorded
{{ row.build_run_id }}{{ row.build_type }}{{ row.version }} - {{ row.result }} - {% if row.log_url %}View log{% else %}-{% endif %}{{ row.recorded_at }}
-
-{% endblock %} + + diff --git a/scripts/generator/dportsv3/tracker/templates/progress.html b/scripts/generator/dportsv3/tracker/templates/progress.html new file mode 100644 index 00000000000..cf0d13587de --- /dev/null +++ b/scripts/generator/dportsv3/tracker/templates/progress.html @@ -0,0 +1,188 @@ +{# Lifted from www/example/index.html; Jinja-ified. + + This template renders the dsynth-progress UI against tracker data + for one target. Progress.js fetches summary.json + _history.json + with paths relative to the page; the tag pins those to the + /target/{target}/progress/ endpoint so the browser resolves them + correctly regardless of trailing slash. +#} + + + + + + + dsynth — {{ target }} + + + + + + + {% with nav_crumbs = [{"label": target}] %} + {% include "_tracker_nav.html" %} + {% endwith %} + + + +
+ +
+ + + + + + + + + + + +
IDDurationBuild PhaseOriginLines
+
+ +
+
+
+ biggest blockers + fetch errors + patch errors + build errors + ignored + recent failures +
+ +
+ + +
+ + + + + + + + + + + + + + +
No.ElapsedIDResultOriginInformationSkipDuration
+
+ +
+ + + + + diff --git a/scripts/generator/dportsv3/tracker/templates/target.html b/scripts/generator/dportsv3/tracker/templates/target.html deleted file mode 100644 index 23ad31f1c18..00000000000 --- a/scripts/generator/dportsv3/tracker/templates/target.html +++ /dev/null @@ -1,61 +0,0 @@ -{% extends "base.html" %} -{% set active_page = "targets" %} - -{% block content %} -

{{ target }}

-

{{ total_rows }} ports tracked.

- -
-
- - -
-
- - -
-
- - -
-
- -
-
- -
- - - - - - - - - - - {% for row in rows %} - - - - - - - {% endfor %} - -
OriginVersionResultLast Success
- - {{ row.origin }} - - {{ row.last_attempt_version or '-' }} - {% set r = row.last_attempt_result or 'unknown' %} - {{ r }} - {{ row.last_success_version or '-' }}
-
- -

Page {{ page }} of {{ page_count }}

-{% endblock %} diff --git a/scripts/generator/pyproject.toml b/scripts/generator/pyproject.toml index e989f431fec..8de807efb9c 100644 --- a/scripts/generator/pyproject.toml +++ b/scripts/generator/pyproject.toml @@ -32,10 +32,12 @@ dependencies = [ [project.optional-dependencies] tracker = ["fastapi", "uvicorn[standard]", "jinja2"] +agent = ["litellm"] dev = ["pytest", "mypy", "httpx"] [project.scripts] dportsv3 = "dportsv3.cli:main" +artifact-store = "dportsv3.artifact_store:main" [project.urls] Homepage = "https://github.com/DragonFlyBSD/DeltaPorts" diff --git a/scripts/generator/tests/test_state_db_concurrency.py b/scripts/generator/tests/test_state_db_concurrency.py new file mode 100644 index 00000000000..4b106161b46 --- /dev/null +++ b/scripts/generator/tests/test_state_db_concurrency.py @@ -0,0 +1,159 @@ +"""Smoke test for two-writer access to state.db under SQLite WAL. + +Phase 4 step 4 puts both artifact-store and the tracker on the same +state.db file. SQLite WAL allows one writer + N readers concurrently; +writers serialize via the WAL lock with a busy_timeout window. This +test stresses that contract by hammering both writers in parallel +threads — if WAL or busy_timeout weren't set, we'd see ``database is +locked`` errors here. + +Run with: pytest scripts/generator/tests/test_state_db_concurrency.py +""" + +from __future__ import annotations + +import sqlite3 +import threading +import time +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from dportsv3.db.schema import init_db as init_state_db + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _artifact_store_writes(db_path: Path, n: int, stop_after_first_error: bool = True) -> list[str]: + """Simulate artifact-store: insert bundles + events.""" + errors: list[str] = [] + conn = sqlite3.connect(str(db_path), check_same_thread=False, timeout=10) + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA busy_timeout=5000") + try: + for i in range(n): + try: + with conn: + conn.execute( + """INSERT INTO bundles (bundle_id, run_id, origin, ts_utc, result, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?)""", + (f"as-bundle-{i}", "as-run-1", "devel/x", _now(), "fail", _now()), + ) + conn.execute( + "INSERT INTO events (ts, type, data_json) VALUES (?, ?, ?)", + (_now(), "bundle_upserted", "{}"), + ) + except sqlite3.Error as exc: + errors.append(f"as[{i}]: {exc}") + if stop_after_first_error: + break + finally: + conn.close() + return errors + + +def _tracker_writes(db_path: Path, n: int, stop_after_first_error: bool = True) -> list[str]: + """Simulate tracker: create a build_run, write build_results.""" + errors: list[str] = [] + conn = sqlite3.connect(str(db_path), check_same_thread=False, timeout=10) + conn.execute("PRAGMA foreign_keys=ON") + conn.execute("PRAGMA busy_timeout=5000") + try: + # Open one run + with conn: + cur = conn.execute( + "INSERT INTO build_runs (target, build_type, started_at) VALUES (?, ?, ?)", + ("@concurrency", "test", _now()), + ) + run_id = cur.lastrowid + # Now hammer build_results + for i in range(n): + try: + with conn: + conn.execute( + """INSERT INTO build_results + (build_run_id, origin, version, result, recorded_at, status) + VALUES (?, ?, ?, ?, ?, ?)""", + (run_id, f"devel/p{i:03d}", "1.0", "success", _now(), "recorded"), + ) + except sqlite3.Error as exc: + errors.append(f"tk[{i}]: {exc}") + if stop_after_first_error: + break + finally: + conn.close() + return errors + + +def test_two_writers_no_lock_errors(tmp_path: Path) -> None: + db_path = tmp_path / "state.db" + conn = sqlite3.connect(str(db_path)) + init_state_db(conn) + conn.close() + + n_per_writer = 60 + as_errors: list[str] = [] + tk_errors: list[str] = [] + + def run_as() -> None: + as_errors.extend(_artifact_store_writes(db_path, n_per_writer)) + + def run_tk() -> None: + tk_errors.extend(_tracker_writes(db_path, n_per_writer)) + + t_as = threading.Thread(target=run_as) + t_tk = threading.Thread(target=run_tk) + t0 = time.monotonic() + t_as.start() + t_tk.start() + t_as.join(timeout=30) + t_tk.join(timeout=30) + elapsed = time.monotonic() - t0 + + assert not as_errors, f"artifact-store writes had errors: {as_errors[:3]}" + assert not tk_errors, f"tracker writes had errors: {tk_errors[:3]}" + + # Verify all writes landed. + reader = sqlite3.connect(str(db_path)) + n_bundles = reader.execute("SELECT count(*) FROM bundles").fetchone()[0] + n_events = reader.execute( + "SELECT count(*) FROM events WHERE type='bundle_upserted'" + ).fetchone()[0] + n_runs = reader.execute("SELECT count(*) FROM build_runs").fetchone()[0] + n_results = reader.execute("SELECT count(*) FROM build_results").fetchone()[0] + reader.close() + + assert n_bundles == n_per_writer, f"expected {n_per_writer} bundles, got {n_bundles}" + assert n_events == n_per_writer, f"expected {n_per_writer} events, got {n_events}" + assert n_runs == 1, f"expected 1 build_run, got {n_runs}" + assert n_results == n_per_writer, f"expected {n_per_writer} build_results, got {n_results}" + + # Sanity bound: 60 writes per side × ~5ms each + WAL contention <<< 30s + # If we're anywhere near 30s we have a serialization disaster + assert elapsed < 15, f"two-writer load took {elapsed:.1f}s — investigate" + + +def test_fk_enforcement_on_tracker_write(tmp_path: Path) -> None: + """Tracker connections must respect the FK constraint introduced by + the build_results -> build_runs reference. If foreign_keys=ON gets + skipped (per-connection pragma), this insert would succeed and the + bug would only surface much later.""" + db_path = tmp_path / "state.db" + conn = sqlite3.connect(str(db_path)) + init_state_db(conn) + conn.close() + + conn = sqlite3.connect(str(db_path)) + conn.execute("PRAGMA foreign_keys=ON") + with pytest.raises(sqlite3.IntegrityError): + with conn: + conn.execute( + """INSERT INTO build_results + (build_run_id, origin, version, result, recorded_at, status) + VALUES (99999, 'devel/x', '1.0', 'success', ?, 'recorded')""", + (_now(),), + ) + conn.close() diff --git a/scripts/generator/tests/test_tracker_agentic_endpoints.py b/scripts/generator/tests/test_tracker_agentic_endpoints.py new file mode 100644 index 00000000000..c1064224b68 --- /dev/null +++ b/scripts/generator/tests/test_tracker_agentic_endpoints.py @@ -0,0 +1,188 @@ +"""Tests for the Phase 4 step 5 agentic-read endpoints. + +The tracker absorbs state-server's read API onto the same state.db. +These tests seed rows directly via SQL (matching what artifact-store +and state-server would write) and exercise the new ``/api/...`` routes. +""" + +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +fastapi = pytest.importorskip("fastapi") +from fastapi.testclient import TestClient + +from dportsv3.db.schema import init_db as init_state_db +from dportsv3.tracker.server import create_app + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@pytest.fixture +def seeded_state_db(tmp_path: Path) -> Path: + """Build a state.db with bundles/jobs/runs covering target filters.""" + db_path = tmp_path / "state.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + init_state_db(conn) + + now = _now() + conn.executemany( + """INSERT INTO runs (run_id, profile, target, ts_start, last_seen_at) + VALUES (?, ?, ?, ?, ?)""", + [ + ("run-2026Q2-001", "2026Q2", "@2026Q2", now, now), + ("run-main-002", "main", "@main", now, now), + ("run-legacy-003", "legacy", None, now, now), + ], + ) + conn.executemany( + """INSERT INTO bundles + (bundle_id, run_id, origin, flavor, ts_utc, result, target, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + [ + ("b-q2-a", "run-2026Q2-001", "devel/foo", "", now, "fail", "@2026Q2", now), + ("b-q2-b", "run-2026Q2-001", "devel/bar", "", now, "fail", "@2026Q2", now), + ("b-main-a", "run-main-002", "devel/foo", "", now, "fail", "@main", now), + ("b-legacy", "run-legacy-003", "devel/baz", "", now, "fail", None, now), + ], + ) + conn.executemany( + """INSERT INTO jobs + (job_id, state, type, origin, flavor, bundle_dir, + created_ts_utc, path, last_seen_at, target) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + [ + ("job-q2-a", "pending", "triage", "devel/foo", "", "", now, "", now, "@2026Q2"), + ("job-q2-b", "done", "triage", "devel/bar", "", "", now, "", now, "@2026Q2"), + ("job-main", "pending", "triage", "devel/foo", "", "", now, "", now, "@main"), + ("job-legacy", "pending","triage", "devel/baz", "", "", now, "", now, None), + ], + ) + conn.execute( + """INSERT INTO events (ts, type, data_json) VALUES (?, ?, ?)""", + (now, "bundle_upserted", json.dumps({"bundle_id": "b-q2-a", "target": "@2026Q2"})), + ) + conn.execute( + """INSERT INTO events (ts, type, data_json) VALUES (?, ?, ?)""", + (now, "bundle_upserted", json.dumps({"bundle_id": "b-main-a", "target": "@main"})), + ) + conn.execute( + """INSERT INTO runner_status (id, status, updated_at) + VALUES (1, 'idle', ?)""", + (now,), + ) + conn.commit() + conn.close() + return db_path + + +@pytest.fixture +def client(seeded_state_db: Path) -> TestClient: + app = create_app(seeded_state_db) + with TestClient(app) as test_client: + yield test_client + + +def test_api_health(client: TestClient) -> None: + resp = client.get("/api/health") + assert resp.status_code == 200 + assert resp.json() == {"status": "ok"} + + +def test_api_agentic_status_counts(client: TestClient) -> None: + body = client.get("/api/agentic-status").json() + assert body["bundles"] == 4 + assert body["runs"] == 3 + assert body["jobs"]["pending"] == 3 + assert body["jobs"]["done"] == 1 + + +def test_api_runs_filters_by_target(client: TestClient) -> None: + all_runs = client.get("/api/runs").json() + assert len(all_runs) == 3 + q2_runs = client.get("/api/runs", params={"target": "@2026Q2"}).json() + assert [r["run_id"] for r in q2_runs] == ["run-2026Q2-001"] + + +def test_api_run_detail_404(client: TestClient) -> None: + resp = client.get("/api/runs/run-nope") + assert resp.status_code == 404 + + +def test_api_jobs_filters_by_state_and_target(client: TestClient) -> None: + pending = client.get("/api/jobs", params={"state": "pending"}).json() + assert {j["job_id"] for j in pending} == {"job-q2-a", "job-main", "job-legacy"} + + q2_pending = client.get( + "/api/jobs", params={"state": "pending", "target": "@2026Q2"} + ).json() + assert [j["job_id"] for j in q2_pending] == ["job-q2-a"] + + +def test_api_jobs_legacy_null_target_only_in_unfiltered(client: TestClient) -> None: + unfiltered = client.get("/api/jobs").json() + assert any(j["job_id"] == "job-legacy" for j in unfiltered) + + filtered = client.get("/api/jobs", params={"target": "@main"}).json() + assert all(j["job_id"] != "job-legacy" for j in filtered) + + +def test_api_bundles_filters(client: TestClient) -> None: + q2 = client.get("/api/bundles", params={"target": "@2026Q2"}).json() + assert {b["bundle_id"] for b in q2} == {"b-q2-a", "b-q2-b"} + + by_origin = client.get( + "/api/bundles", params={"origin": "devel/foo"} + ).json() + assert {b["bundle_id"] for b in by_origin} == {"b-q2-a", "b-main-a"} + + +def test_api_bundle_detail_includes_artifacts_list(client: TestClient) -> None: + body = client.get("/api/bundles/b-q2-a").json() + assert body["bundle_id"] == "b-q2-a" + assert body["target"] == "@2026Q2" + assert "artifacts" in body + assert isinstance(body["artifacts"], list) + + +def test_api_port_history_target_scoped(client: TestClient) -> None: + all_foo = client.get("/api/ports/devel/foo").json() + assert {b["bundle_id"] for b in all_foo} == {"b-q2-a", "b-main-a"} + + q2_foo = client.get( + "/api/ports/devel/foo", params={"target": "@2026Q2"} + ).json() + assert {b["bundle_id"] for b in q2_foo} == {"b-q2-a"} + + +def test_api_runner_status_returns_singleton(client: TestClient) -> None: + body = client.get("/api/runner-status").json() + assert body["status"] == "idle" + + +def test_api_events_filters_by_target_payload( + client: TestClient, seeded_state_db: Path +) -> None: + # The SSE endpoint streams forever; verify target filtering by + # exercising the query layer directly against the seeded DB. + from dportsv3.tracker.agentic_queries import events_since + from dportsv3.tracker.db import open_db + + conn = open_db(seeded_state_db) + try: + all_events = events_since(conn, last_id=0) + q2_events = events_since(conn, last_id=0, target="@2026Q2") + finally: + conn.close() + + assert len(all_events) == 2 + assert len(q2_events) == 1 + assert json.loads(q2_events[0]["data_json"])["target"] == "@2026Q2" diff --git a/scripts/generator/tests/test_tracker_agentic_views.py b/scripts/generator/tests/test_tracker_agentic_views.py new file mode 100644 index 00000000000..557da9cb214 --- /dev/null +++ b/scripts/generator/tests/test_tracker_agentic_views.py @@ -0,0 +1,178 @@ +"""Tests for the Phase 4 step 6 agentic HTML views. + +These are page-shape assertions only — Jinja rendering succeeds, the +key fields are present in the HTML, target filters work end-to-end. +The data layer is covered by ``test_tracker_agentic_endpoints.py``. +""" + +from __future__ import annotations + +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +fastapi = pytest.importorskip("fastapi") +from fastapi.testclient import TestClient + +from dportsv3.db.schema import init_db as init_state_db +from dportsv3.tracker.server import create_app + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@pytest.fixture +def seeded_state_db(tmp_path: Path) -> Path: + db_path = tmp_path / "state.db" + conn = sqlite3.connect(str(db_path)) + conn.row_factory = sqlite3.Row + init_state_db(conn) + + now = _now() + conn.executemany( + """INSERT INTO runs (run_id, profile, target, ts_start, last_seen_at) + VALUES (?, ?, ?, ?, ?)""", + [ + ("run-q2-001", "2026Q2", "@2026Q2", now, now), + ("run-main-002", "main", "@main", now, now), + ], + ) + conn.executemany( + """INSERT INTO bundles + (bundle_id, run_id, origin, flavor, ts_utc, result, target, last_seen_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + [ + ("b-q2-foo", "run-q2-001", "devel/foo", "", now, "fail", "@2026Q2", now), + ("b-q2-bar", "run-q2-001", "devel/bar", "", now, "fail", "@2026Q2", now), + ("b-main-foo", "run-main-002", "devel/foo", "", now, "fail", "@main", now), + ], + ) + conn.executemany( + """INSERT INTO artifact_refs + (bundle_id, relpath, backend, sha256, kind, size, created_at) + VALUES (?, ?, 'blob', ?, ?, ?, ?)""", + [ + ("b-q2-foo", "meta.txt", "abc123def456", "text/plain", 42, now), + ("b-q2-foo", "logs/errors.txt", "ffee1122", "text/plain", 100, now), + ], + ) + conn.executemany( + """INSERT INTO jobs + (job_id, state, type, origin, flavor, bundle_dir, + created_ts_utc, path, last_seen_at, target) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", + [ + ("job-q2-foo", "pending", "triage", "devel/foo", "", "", + now, "/tmp/job-q2-foo.job", now, "@2026Q2"), + ("job-main-foo", "done", "triage", "devel/foo", "", "", + now, "/tmp/job-main-foo.job", now, "@main"), + ], + ) + conn.execute( + """INSERT INTO activity_log (ts, job_id, stage, message, duration_ms) + VALUES (?, ?, ?, ?, ?)""", + (now, "job-q2-foo", "triage_start", "began triage", 12), + ) + conn.execute( + """INSERT INTO runner_status (id, status, job_id, updated_at) + VALUES (1, 'idle', NULL, ?)""", + (now,), + ) + conn.commit() + conn.close() + return db_path + + +@pytest.fixture +def client(seeded_state_db: Path) -> TestClient: + app = create_app(seeded_state_db) + with TestClient(app) as test_client: + yield test_client + + +def test_view_agentic_index(client: TestClient) -> None: + resp = client.get("/agentic") + assert resp.status_code == 200 + body = resp.text + assert "Agentic" in body + assert "b-q2-foo" in body + assert "job-q2-foo" in body + # Counts panel + assert ">3<" in body or "3
" in body # bundles count + + +def test_view_agentic_bundles_filter(client: TestClient) -> None: + all_resp = client.get("/agentic/bundles") + assert all_resp.status_code == 200 + assert "b-q2-foo" in all_resp.text + assert "b-main-foo" in all_resp.text + + q2 = client.get("/agentic/bundles", params={"target": "@2026Q2"}) + assert q2.status_code == 200 + assert "b-q2-foo" in q2.text + assert "b-main-foo" not in q2.text + + +def test_view_agentic_bundle_detail_lists_artifacts(client: TestClient) -> None: + resp = client.get("/agentic/bundles/b-q2-foo") + assert resp.status_code == 200 + body = resp.text + assert "b-q2-foo" in body + assert "meta.txt" in body + assert "logs/errors.txt" in body + # Link to artifact stream endpoint + assert "/api/bundles/b-q2-foo/artifacts/meta.txt" in body + + +def test_view_agentic_bundle_detail_404(client: TestClient) -> None: + assert client.get("/agentic/bundles/does-not-exist").status_code == 404 + + +def test_view_agentic_jobs_state_filter(client: TestClient) -> None: + pending = client.get( + "/agentic/jobs", params={"state": "pending"} + ) + assert pending.status_code == 200 + assert "job-q2-foo" in pending.text + assert "job-main-foo" not in pending.text + + +def test_view_agentic_job_detail_shows_activity(client: TestClient) -> None: + resp = client.get("/agentic/jobs/job-q2-foo") + assert resp.status_code == 200 + body = resp.text + assert "triage_start" in body + assert "began triage" in body + + +def test_view_agentic_runner(client: TestClient) -> None: + resp = client.get("/agentic/runner") + assert resp.status_code == 200 + assert "idle" in resp.text + + +def test_view_agentic_activity(client: TestClient) -> None: + resp = client.get("/agentic/activity") + assert resp.status_code == 200 + body = resp.text + assert "triage_start" in body + assert "job-q2-foo" in body + + +def test_view_agentic_run_detail(client: TestClient) -> None: + resp = client.get("/agentic/runs/run-q2-001") + assert resp.status_code == 200 + body = resp.text + assert "run-q2-001" in body + assert "b-q2-foo" in body + assert "b-q2-bar" in body + + +def test_view_nav_includes_agentic(client: TestClient) -> None: + resp = client.get("/") + assert resp.status_code == 200 + assert ">Agentic<" in resp.text diff --git a/scripts/generator/tests/test_tracker_api.py b/scripts/generator/tests/test_tracker_api.py index 0b87e7fdeda..01c1c0c3828 100644 --- a/scripts/generator/tests/test_tracker_api.py +++ b/scripts/generator/tests/test_tracker_api.py @@ -8,11 +8,12 @@ from fastapi.testclient import TestClient from dportsv3.tracker.server import create_app +import dportsv3.tracker.server as tracker_server @pytest.fixture def client(tmp_path: Path) -> TestClient: - app = create_app(tmp_path / "tracker.db") + app = create_app(tmp_path / "state.db") with TestClient(app) as test_client: yield test_client @@ -215,3 +216,37 @@ def test_api_status_failures_and_diff_endpoints(client: TestClient) -> None: assert [row["origin"] for row in failures.json()] == ["devel/foo"] assert diff.status_code == 200 assert [row["origin"] for row in diff.json()["differ"]] == ["devel/foo"] + + +def test_api_uses_fresh_db_connection_per_request( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + open_count = 0 + real_open_db = tracker_server.open_db + + def _counting_open_db(db_path: str | Path): + nonlocal open_count + open_count += 1 + return real_open_db(db_path) + + monkeypatch.setattr(tracker_server, "open_db", _counting_open_db) + + app = create_app(tmp_path / "state.db") + with TestClient(app) as test_client: + start = test_client.post( + "/api/builds", + json={"target": "@main", "build_type": "test"}, + ) + assert start.status_code == 200 + run_id = start.json()["id"] + + enqueue = test_client.post( + f"/api/builds/{run_id}/queue", + json={"ports": [{"origin": "devel/foo", "version": "1.0"}]}, + ) + assert enqueue.status_code == 200 + + detail = test_client.get(f"/api/builds/{run_id}") + assert detail.status_code == 200 + + assert open_count >= 3 diff --git a/scripts/generator/tests/test_tracker_integration.py b/scripts/generator/tests/test_tracker_integration.py index d17480f7a7b..5ac99aeae04 100644 --- a/scripts/generator/tests/test_tracker_integration.py +++ b/scripts/generator/tests/test_tracker_integration.py @@ -31,7 +31,7 @@ def __exit__(self, exc_type, exc, tb) -> bool: @pytest.fixture def test_client(tmp_path: Path) -> TestClient: - app = create_app(tmp_path / "tracker.db") + app = create_app(tmp_path / "state.db") with TestClient(app) as client: yield client diff --git a/scripts/generator/tests/test_tracker_progress.py b/scripts/generator/tests/test_tracker_progress.py new file mode 100644 index 00000000000..554f973fd54 --- /dev/null +++ b/scripts/generator/tests/test_tracker_progress.py @@ -0,0 +1,154 @@ +"""Tests for the Phase 5 step 1 dsynth-progress adapter. + +Seeds a build_run + build_results, hits the summary.json / +NN_history.json endpoints, and checks shape parity with the +dsynth-progress UI's expectations. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +fastapi = pytest.importorskip("fastapi") +from fastapi.testclient import TestClient + +from dportsv3.tracker.db import init_db, open_db +from dportsv3.tracker.server import create_app + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@pytest.fixture +def seeded_state_db(tmp_path: Path) -> Path: + db_path = tmp_path / "state.db" + conn = init_db(db_path) + now = _now() + cur = conn.execute( + "INSERT INTO build_runs(target, build_type, started_at, total_expected) VALUES (?, ?, ?, ?)", + ("@2026Q2", "test", now, 5), + ) + run_id = cur.lastrowid + rows = [ + (run_id, "devel/foo", "1.0", "success", now, "recorded"), + (run_id, "devel/bar", "1.0", "failure", now, "recorded"), + (run_id, "devel/baz", "1.0", "skipped", now, "recorded"), + (run_id, "devel/qux", "1.0", "ignored", now, "recorded"), + (run_id, "devel/inprogress", "1.0", "", now, "building"), + ] + conn.executemany( + """INSERT INTO build_results + (build_run_id, origin, version, result, recorded_at, status) + VALUES (?, ?, ?, ?, ?, ?)""", + rows, + ) + conn.commit() + conn.close() + return db_path + + +@pytest.fixture +def client(seeded_state_db: Path) -> TestClient: + app = create_app(seeded_state_db) + with TestClient(app) as test_client: + yield test_client + + +def test_progress_html_serves(client: TestClient) -> None: + resp = client.get("/target/@2026Q2") + assert resp.status_code == 200 + body = resp.text + # Pinned base + key dsynth-progress hooks + assert '' in body + assert "progress.css" in body + assert "progress.js" in body + assert 'id="stats_built"' in body + + +def test_progress_summary_shape(client: TestClient) -> None: + body = client.get("/api/progress/@2026Q2/summary.json").json() + assert body["profile"] == "@2026Q2" + assert body["active"] == 1 # finished_at IS NULL + stats = body["stats"] + assert stats["built"] == 1 + assert stats["failed"] == 1 + assert stats["skipped"] == 1 + assert stats["ignored"] == 1 + assert stats["queued"] == 5 # total_expected + assert stats["meta"] == 0 + # 5 results total, chunk size 1000 → kfiles >= 1 + assert body["kfiles"] >= 1 + # One building row → one virtual builder slot + assert len(body["builders"]) == 1 + assert body["builders"][0]["origin"] == "devel/inprogress" + + +def test_progress_history_chunk_one(client: TestClient) -> None: + body = client.get("/api/progress/@2026Q2/01_history.json").json() + # Excludes the 'building' row (active builder, not history) + assert isinstance(body, list) + assert len(body) == 4 + assert "devel/inprogress" not in {row["origin"] for row in body} + # dsynth vocabulary + results = {row["origin"]: row["result"] for row in body} + assert results["devel/foo"] == "built" + assert results["devel/bar"] == "failed" + assert results["devel/baz"] == "skipped" + assert results["devel/qux"] == "ignored" + + +def test_progress_history_past_last_chunk(client: TestClient) -> None: + body = client.get("/api/progress/@2026Q2/99_history.json").json() + assert body == [] + + +def test_progress_summary_unknown_target(client: TestClient) -> None: + body = client.get("/api/progress/@nonexistent/summary.json").json() + assert body["kfiles"] == 0 + assert body["stats"]["built"] == 0 + assert body["builders"] == [] + + +def test_build_progress_html_serves(client: TestClient, seeded_state_db: Path) -> None: + import sqlite3 + conn = sqlite3.connect(str(seeded_state_db)) + run_id = conn.execute("SELECT id FROM build_runs LIMIT 1").fetchone()[0] + conn.close() + + resp = client.get(f"/builds/{run_id}") + assert resp.status_code == 200 + body = resp.text + assert f'' in body + assert "progress.css" in body + + +def test_build_progress_summary_and_history( + client: TestClient, seeded_state_db: Path +) -> None: + import sqlite3 + conn = sqlite3.connect(str(seeded_state_db)) + run_id = conn.execute("SELECT id FROM build_runs LIMIT 1").fetchone()[0] + conn.close() + + summary = client.get(f"/api/progress/build/{run_id}/summary.json").json() + assert summary["stats"]["built"] == 1 + assert summary["stats"]["failed"] == 1 + assert summary["stats"]["skipped"] == 1 + assert summary["stats"]["ignored"] == 1 + assert len(summary["builders"]) == 1 + + history = client.get(f"/api/progress/build/{run_id}/01_history.json").json() + assert isinstance(history, list) + assert len(history) == 4 # excludes building row + + +def test_build_progress_unknown_run_404(client: TestClient) -> None: + assert client.get("/builds/99999").status_code == 404 + assert client.get("/api/progress/build/99999/summary.json").status_code == 404 + # History endpoint silently returns [] for unknown run (no run guard). + body = client.get("/api/progress/build/99999/01_history.json").json() + assert body == [] diff --git a/scripts/generator/tests/test_tracker_queue.py b/scripts/generator/tests/test_tracker_queue.py index 4022fa023f2..fabbf756b58 100644 --- a/scripts/generator/tests/test_tracker_queue.py +++ b/scripts/generator/tests/test_tracker_queue.py @@ -29,7 +29,7 @@ def conn(tmp_path: Path): @pytest.fixture def client(tmp_path: Path) -> TestClient: - app = create_app(tmp_path / "tracker.db") + app = create_app(tmp_path / "state.db") with TestClient(app) as tc: yield tc diff --git a/scripts/snippet-extractor b/scripts/snippet-extractor new file mode 100755 index 00000000000..5fa913b2a9f --- /dev/null +++ b/scripts/snippet-extractor @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 +""" +snippet-extractor: Extract bounded source/log snippets for agent context. + +Usage: + snippet-extractor --bundle [options] + +Options: + --bundle PATH Evidence bundle directory (required) + --round N Snippet round number (default: 1) + --distfiles-dir PATH Override distfiles directory + --buildbase-dir PATH Override buildbase directory (for workdir scan) + --max-per-snippet N Max bytes per snippet (default: 51200) + --max-total N Max total bytes per round (default: 204800) + --prefer-workdir Prefer preserved workdir over distfiles + --dry-run Parse requests but don't extract + --verbose Verbose output + +Exit codes: + 0 - Success, at least some snippets extracted + 1 - No snippet requests found + 2 - All requests failed (nothing extracted) + 3 - Configuration/usage error + +The extractor reads snippet requests from analysis/triage.md or analysis/patch.md +and writes results to analysis/snippets/round_N/. +""" + +import argparse +import gzip +import hashlib +import json +import os +import re +import shutil +import sys +import tarfile +import zipfile +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional + + +# ----------------------------------------------------------------------------- +# Data structures +# ----------------------------------------------------------------------------- + +@dataclass +class SnippetRequest: + raw: str + type: str # source, buildsystem, configure, log + path: Optional[str] = None + line: Optional[int] = None + context: Optional[int] = None # ±N lines + start_line: Optional[int] = None # for log requests + end_line: Optional[int] = None + status: str = "pending" + output: Optional[str] = None + bytes: int = 0 + actual_lines: Optional[list] = None + note: Optional[str] = None + + +@dataclass +class RoundResult: + round: int + source: str # "workdir", "distfiles", "log" + distfile: Optional[str] = None + workdir_path: Optional[str] = None + requests: list = field(default_factory=list) + total_bytes: int = 0 + budget_remaining: int = 0 + truncated: bool = False + + +@dataclass +class Manifest: + rounds: list = field(default_factory=list) + total_rounds: int = 0 + total_bytes_all_rounds: int = 0 + + +# ----------------------------------------------------------------------------- +# Logging +# ----------------------------------------------------------------------------- + +VERBOSE = False + +def log(level: str, msg: str): + ts = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + print(f"{ts} {level:5} {msg}", file=sys.stderr) + +def log_verbose(msg: str): + if VERBOSE: + log("DEBUG", msg) + +def log_info(msg: str): + log("INFO", msg) + +def log_warn(msg: str): + log("WARN", msg) + +def log_error(msg: str): + log("ERROR", msg) + + +# ----------------------------------------------------------------------------- +# Request parsing +# ----------------------------------------------------------------------------- + +def parse_snippet_requests(content: str) -> list[SnippetRequest]: + """Parse ## Snippet Requests section from markdown.""" + # Find the section + match = re.search( + r"^##\s*Snippet Requests\s*\n(.*?)(?=^##|\Z)", + content, + re.MULTILINE | re.DOTALL | re.IGNORECASE + ) + if not match: + return [] + + section = match.group(1) + requests = [] + + for line in section.split("\n"): + line = line.strip() + if not line: + continue + + # Parse: - `request:spec` — description + m = re.match(r"-\s*`([^`]+)`", line) + if not m: + continue + + spec = m.group(1) + req = parse_request_spec(spec) + if req: + requests.append(req) + + return requests + + +def parse_request_spec(spec: str) -> Optional[SnippetRequest]: + """Parse a single request specification.""" + parts = spec.split(":") + if len(parts) < 2: + return None + + req_type = parts[0].lower() + + if req_type == "source": + # source:path:line:context or source:path:all + if len(parts) < 3: + return None + path = parts[1] + if len(parts) >= 3 and parts[2].lower() == "all": + return SnippetRequest( + raw=spec, type="source", path=path, + line=None, context=None + ) + try: + line = int(parts[2]) + context = 30 # default + if len(parts) >= 4: + ctx_str = parts[3].strip() + if ctx_str.startswith("±") or ctx_str.startswith("+/-"): + ctx_str = ctx_str.lstrip("±+/-") + context = int(ctx_str) + return SnippetRequest( + raw=spec, type="source", path=path, + line=line, context=context + ) + except ValueError: + return None + + elif req_type in ("buildsystem", "configure"): + # buildsystem:path or configure:path + path = ":".join(parts[1:]) # rejoin in case path has colons + return SnippetRequest(raw=spec, type=req_type, path=path) + + elif req_type == "log": + # log:start:end + if len(parts) < 3: + return None + try: + start = int(parts[1]) + end = int(parts[2]) + return SnippetRequest( + raw=spec, type="log", + start_line=start, end_line=end + ) + except ValueError: + return None + + return None + + +# ----------------------------------------------------------------------------- +# Meta parsing +# ----------------------------------------------------------------------------- + +def parse_meta_file(path: Path) -> dict: + """Parse key=value meta file.""" + data = {} + if not path.exists(): + return data + with open(path) as f: + for line in f: + line = line.strip() + if "=" in line: + key, _, value = line.partition("=") + data[key.strip()] = value.strip() + return data + + +def parse_distinfo(path: Path) -> list[tuple[str, str, int]]: + """Parse distinfo file, return list of (filename, hash, size).""" + results = [] + if not path.exists(): + return results + + # Format: SHA256 (filename) = hash + # SIZE (filename) = size + sha_pattern = re.compile(r"SHA256\s+\(([^)]+)\)\s*=\s*(\w+)") + size_pattern = re.compile(r"SIZE\s+\(([^)]+)\)\s*=\s*(\d+)") + + sha_map = {} + size_map = {} + + with open(path) as f: + for line in f: + m = sha_pattern.match(line) + if m: + sha_map[m.group(1)] = m.group(2) + continue + m = size_pattern.match(line) + if m: + size_map[m.group(1)] = int(m.group(2)) + + for fname, sha in sha_map.items(): + size = size_map.get(fname, 0) + results.append((fname, sha, size)) + + return results + + +# ----------------------------------------------------------------------------- +# Workdir detection +# ----------------------------------------------------------------------------- + +def find_workdir(buildbase: Path, origin: str) -> Optional[Path]: + """Scan SL* directories for preserved workdir matching origin.""" + cat, port = origin.split("/", 1) if "/" in origin else (origin, "") + + for slot_dir in sorted(buildbase.glob("SL*")): + # Look for construction///work + work_path = slot_dir / "construction" / cat / port / "work" + if work_path.is_dir(): + log_info(f"Found preserved workdir: {work_path}") + return work_path + + return None + + +# ----------------------------------------------------------------------------- +# Distfile extraction +# ----------------------------------------------------------------------------- + +def find_distfile(distfiles_dir: Path, distinfo_entries: list) -> Optional[Path]: + """Find the primary distfile in the distfiles directory.""" + for fname, sha, size in distinfo_entries: + # Try common locations + candidates = [ + distfiles_dir / fname, + distfiles_dir / fname.split("/")[-1], # basename only + ] + for candidate in candidates: + if candidate.exists(): + log_verbose(f"Found distfile: {candidate}") + return candidate + + return None + + +def extract_distfile_to_workdir(distfile: Path, workdir: Path) -> bool: + """Extract distfile to temporary workdir.""" + workdir.mkdir(parents=True, exist_ok=True) + + name = distfile.name.lower() + + try: + if name.endswith((".tar.gz", ".tgz")): + with tarfile.open(distfile, "r:gz") as tf: + tf.extractall(workdir, filter="data") + return True + elif name.endswith((".tar.xz", ".txz")): + with tarfile.open(distfile, "r:xz") as tf: + tf.extractall(workdir, filter="data") + return True + elif name.endswith((".tar.bz2", ".tbz2", ".tbz")): + with tarfile.open(distfile, "r:bz2") as tf: + tf.extractall(workdir, filter="data") + return True + elif name.endswith(".tar"): + with tarfile.open(distfile, "r:") as tf: + tf.extractall(workdir, filter="data") + return True + elif name.endswith(".zip"): + with zipfile.ZipFile(distfile, "r") as zf: + zf.extractall(workdir) + return True + else: + log_warn(f"Unknown archive format: {name}") + return False + except Exception as e: + log_error(f"Failed to extract {distfile}: {e}") + return False + + +def find_file_in_tree(root: Path, target_path: str) -> Optional[Path]: + """Find a file in extracted tree, handling top-level directory variations.""" + # Direct path + direct = root / target_path + if direct.exists(): + return direct + + # Try under first subdirectory (common for tarballs) + subdirs = [d for d in root.iterdir() if d.is_dir()] + for subdir in subdirs: + candidate = subdir / target_path + if candidate.exists(): + return candidate + + # Try basename search as last resort + basename = Path(target_path).name + for found in root.rglob(basename): + if found.is_file(): + return found + + return None + + +# ----------------------------------------------------------------------------- +# Log extraction +# ----------------------------------------------------------------------------- + +def extract_log_lines(log_gz_path: Path, start: int, end: int) -> tuple[str, list]: + """Extract lines from gzipped log file.""" + if not log_gz_path.exists(): + return "", [] + + lines = [] + try: + with gzip.open(log_gz_path, "rt", errors="replace") as f: + for i, line in enumerate(f, 1): + if i >= start and i <= end: + lines.append(line.rstrip("\n")) + if i > end: + break + except Exception as e: + log_error(f"Failed to read log: {e}") + return "", [] + + content = "\n".join(lines) + actual_lines = [start, min(start + len(lines) - 1, end)] if lines else [] + return content, actual_lines + + +# ----------------------------------------------------------------------------- +# Source extraction +# ----------------------------------------------------------------------------- + +def extract_source_snippet( + file_path: Path, + line: Optional[int], + context: Optional[int], + max_bytes: int +) -> tuple[str, list]: + """Extract snippet from source file.""" + if not file_path.exists(): + return "", [] + + try: + with open(file_path, "r", errors="replace") as f: + all_lines = f.readlines() + except Exception as e: + log_error(f"Failed to read {file_path}: {e}") + return "", [] + + if line is None: + # Extract entire file + content = "".join(all_lines) + if len(content) > max_bytes: + content = content[:max_bytes] + "\n[...truncated...]\n" + return content, [1, len(all_lines)] + + # Extract lines around target + start_idx = max(0, line - context - 1) + end_idx = min(len(all_lines), line + context) + + selected = all_lines[start_idx:end_idx] + content = "".join(selected) + + if len(content) > max_bytes: + content = content[:max_bytes] + "\n[...truncated...]\n" + + actual_lines = [start_idx + 1, start_idx + len(selected)] + return content, actual_lines + + +# ----------------------------------------------------------------------------- +# Safe filename generation +# ----------------------------------------------------------------------------- + +def safe_filename(path: str) -> str: + """Convert path to safe filename.""" + # Replace path separators and special chars + safe = path.replace("/", "_").replace("\\", "_") + safe = re.sub(r"[^a-zA-Z0-9._-]", "_", safe) + # Truncate if too long + if len(safe) > 200: + safe = safe[:200] + return safe + + +# ----------------------------------------------------------------------------- +# Main extraction logic +# ----------------------------------------------------------------------------- + +def process_requests( + requests: list[SnippetRequest], + bundle_dir: Path, + output_dir: Path, + source_root: Path, + source_type: str, + distfile_name: Optional[str], + log_gz_path: Path, + max_per_snippet: int, + max_total: int +) -> RoundResult: + """Process all snippet requests.""" + result = RoundResult( + round=0, # Will be set by caller + source=source_type, + distfile=distfile_name, + workdir_path=str(source_root) if source_type == "workdir" else None, + budget_remaining=max_total + ) + + total_bytes = 0 + + for req in requests: + if total_bytes >= max_total: + req.status = "budget_exceeded" + req.note = "Budget exhausted" + result.truncated = True + result.requests.append(asdict(req)) + continue + + remaining_budget = min(max_per_snippet, max_total - total_bytes) + + if req.type == "log": + # Log extraction + content, actual_lines = extract_log_lines( + log_gz_path, req.start_line, req.end_line + ) + if content: + out_name = f"lines_{req.start_line}-{req.end_line}.txt" + out_path = output_dir / "log" / out_name + out_path.parent.mkdir(parents=True, exist_ok=True) + + if len(content) > remaining_budget: + content = content[:remaining_budget] + "\n[...truncated...]\n" + + out_path.write_text(content) + req.status = "ok" + req.output = f"log/{out_name}" + req.bytes = len(content) + req.actual_lines = actual_lines + total_bytes += len(content) + else: + req.status = "not_found" + req.note = "Could not extract log lines" + + elif req.type in ("source", "buildsystem", "configure"): + # Source/build file extraction + found_path = find_file_in_tree(source_root, req.path) + + if found_path: + content, actual_lines = extract_source_snippet( + found_path, req.line, req.context, remaining_budget + ) + if content: + subdir = req.type if req.type != "source" else "source" + out_name = safe_filename(req.path) + ".txt" + out_path = output_dir / subdir / out_name + out_path.parent.mkdir(parents=True, exist_ok=True) + + out_path.write_text(content) + req.status = "ok" + req.output = f"{subdir}/{out_name}" + req.bytes = len(content) + req.actual_lines = actual_lines + total_bytes += len(content) + else: + req.status = "empty" + req.note = "File exists but is empty or unreadable" + else: + req.status = "not_found" + req.note = f"File not found in {source_type}" + + else: + req.status = "unknown_type" + req.note = f"Unknown request type: {req.type}" + + result.requests.append(asdict(req)) + + result.total_bytes = total_bytes + result.budget_remaining = max_total - total_bytes + return result + + +def load_existing_manifest(bundle_dir: Path) -> Manifest: + """Load existing manifest if present.""" + manifest_path = bundle_dir / "analysis" / "snippets" / "manifest.json" + if manifest_path.exists(): + try: + with open(manifest_path) as f: + data = json.load(f) + return Manifest( + rounds=data.get("rounds", []), + total_rounds=data.get("total_rounds", 0), + total_bytes_all_rounds=data.get("total_bytes_all_rounds", 0) + ) + except Exception: + pass + return Manifest() + + +def save_manifest(manifest: Manifest, bundle_dir: Path): + """Save manifest to bundle.""" + manifest_path = bundle_dir / "analysis" / "snippets" / "manifest.json" + manifest_path.parent.mkdir(parents=True, exist_ok=True) + with open(manifest_path, "w") as f: + json.dump(asdict(manifest), f, indent=2) + + +def save_round_manifest(result: RoundResult, output_dir: Path): + """Save per-round manifest.""" + manifest_path = output_dir / "manifest.json" + with open(manifest_path, "w") as f: + json.dump(asdict(result), f, indent=2) + + +# ----------------------------------------------------------------------------- +# Main +# ----------------------------------------------------------------------------- + +def main(): + global VERBOSE + + parser = argparse.ArgumentParser( + description="Extract bounded source/log snippets for agent context" + ) + parser.add_argument("--bundle", required=True, help="Evidence bundle directory") + parser.add_argument("--round", type=int, default=1, help="Snippet round number") + parser.add_argument("--distfiles-dir", help="Override distfiles directory") + parser.add_argument("--buildbase-dir", help="Override buildbase directory") + parser.add_argument("--max-per-snippet", type=int, default=51200, + help="Max bytes per snippet (default: 51200)") + parser.add_argument("--max-total", type=int, default=204800, + help="Max total bytes per round (default: 204800)") + parser.add_argument("--prefer-workdir", action="store_true", + help="Prefer preserved workdir over distfiles") + parser.add_argument("--dry-run", action="store_true", + help="Parse requests but don't extract") + parser.add_argument("--verbose", action="store_true", help="Verbose output") + args = parser.parse_args() + + VERBOSE = args.verbose + + bundle_dir = Path(args.bundle) + if not bundle_dir.exists(): + log_error(f"Bundle directory does not exist: {bundle_dir}") + sys.exit(3) + + # Parse meta.txt for paths + meta = parse_meta_file(bundle_dir / "meta.txt") + origin = meta.get("origin", meta.get("origin_base", "")) + if not origin: + log_error("Could not determine origin from meta.txt") + sys.exit(3) + + distfiles_dir = Path(args.distfiles_dir) if args.distfiles_dir else Path(meta.get("dir_distfiles", "/build/synth/distfiles")) + buildbase_dir = Path(args.buildbase_dir) if args.buildbase_dir else Path(meta.get("dir_buildbase", "/build/synth")) + + log_info(f"Bundle: {bundle_dir}") + log_info(f"Origin: {origin}") + log_info(f"Round: {args.round}") + log_verbose(f"Distfiles dir: {distfiles_dir}") + log_verbose(f"Buildbase dir: {buildbase_dir}") + + # Find snippet requests in triage.md or patch.md + requests = [] + for source_file in ["triage.md", "patch.md"]: + source_path = bundle_dir / "analysis" / source_file + if source_path.exists(): + content = source_path.read_text() + requests = parse_snippet_requests(content) + if requests: + log_info(f"Found {len(requests)} requests in {source_file}") + break + + if not requests: + log_info("No snippet requests found") + sys.exit(1) + + if args.dry_run: + log_info("Dry run - parsed requests:") + for req in requests: + print(f" {req.type}: {req.raw}") + sys.exit(0) + + # Determine source: workdir or distfiles + source_root = None + source_type = None + distfile_name = None + tmp_workdir = None + + if args.prefer_workdir: + source_root = find_workdir(buildbase_dir, origin) + if source_root: + source_type = "workdir" + + if not source_root: + # Try distfiles + distinfo_path = bundle_dir / "port" / "distinfo" + distinfo_entries = parse_distinfo(distinfo_path) + + if distinfo_entries: + distfile = find_distfile(distfiles_dir, distinfo_entries) + if distfile: + # Extract to temporary workdir + tmp_workdir = bundle_dir / "analysis" / "snippets" / ".workdir" + if tmp_workdir.exists(): + # Reuse existing extraction + source_root = tmp_workdir + source_type = "distfiles" + distfile_name = distfile.name + log_info(f"Reusing extracted workdir: {tmp_workdir}") + else: + log_info(f"Extracting distfile: {distfile}") + if extract_distfile_to_workdir(distfile, tmp_workdir): + source_root = tmp_workdir + source_type = "distfiles" + distfile_name = distfile.name + + if not source_root and not args.prefer_workdir: + # Fallback: try workdir + source_root = find_workdir(buildbase_dir, origin) + if source_root: + source_type = "workdir" + + if not source_root: + log_warn("No source tree available (no workdir, no distfiles)") + source_root = Path("/nonexistent") # Will fail gracefully + source_type = "none" + + log_info(f"Source type: {source_type}") + + # Output directory for this round + output_dir = bundle_dir / "analysis" / "snippets" / f"round_{args.round}" + output_dir.mkdir(parents=True, exist_ok=True) + + # Log path + log_gz_path = bundle_dir / "logs" / "full.log.gz" + + # Process requests + result = process_requests( + requests, + bundle_dir, + output_dir, + source_root, + source_type, + distfile_name, + log_gz_path, + args.max_per_snippet, + args.max_total + ) + result.round = args.round + + # Save round manifest + save_round_manifest(result, output_dir) + + # Update cumulative manifest + manifest = load_existing_manifest(bundle_dir) + manifest.rounds.append(asdict(result)) + manifest.total_rounds = len(manifest.rounds) + manifest.total_bytes_all_rounds = sum(r.get("total_bytes", 0) for r in manifest.rounds) + save_manifest(manifest, bundle_dir) + + # Report results + success_count = sum(1 for r in result.requests if r.get("status") == "ok") + fail_count = len(result.requests) - success_count + + log_info(f"Extracted {success_count}/{len(result.requests)} snippets ({result.total_bytes} bytes)") + + if success_count == 0: + log_error("All requests failed") + sys.exit(2) + + if fail_count > 0: + log_warn(f"{fail_count} requests failed") + + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/scripts/tools/dev-env/dports_dev_env/cli.py b/scripts/tools/dev-env/dports_dev_env/cli.py index 3a59766a33a..b871d2b13ad 100644 --- a/scripts/tools/dev-env/dports_dev_env/cli.py +++ b/scripts/tools/dev-env/dports_dev_env/cli.py @@ -74,6 +74,66 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help="Confirm tear-down of every mount under the cache root (required)", ) + + exec_ = subparsers.add_parser("exec", help="Run a command inside an environment non-interactively") + exec_.add_argument("--cwd", default="/work/DeltaPorts", help="Working directory inside the chroot") + exec_.add_argument("--quiet", action="store_true", help="Suppress INFO mount-prep output") + exec_.add_argument("name", help="Environment name") + exec_.add_argument("argv", nargs=argparse.REMAINDER, help="-- CMD [ARGS...] to run inside the env") + + status = subparsers.add_parser("status", help="Print one environment's state as a single JSON line") + status.add_argument("name", help="Environment name") + + update_ = subparsers.add_parser("update", help="Refresh repo mirrors and fast-forward the env's git checkouts") + update_.add_argument("--force", action="store_true", + help="Pull even when the env's checkouts have uncommitted changes") + update_.add_argument("name", help="Environment name") + + path_ = subparsers.add_parser("path", help="Print one environment's host-side path") + path_.add_argument( + "--writable", + action="store_true", + help="Print env_dir/writable (the agent's edit overlay) instead of env_dir", + ) + path_.add_argument("name", help="Environment name") + + # ----- hooks: install/uninstall/status the dsynth hooks inside an env ----- + hi = subparsers.add_parser( + "hooks-install", + help="Install dsynth hooks into the env's writable etc/dsynth", + ) + hi.add_argument("name", help="Environment name") + hi.add_argument( + "--source", + help="Override source dir (default: scripts/dsynth-hooks/ in the repo)", + ) + hi.add_argument( + "--force", + action="store_true", + help="Overwrite an existing dportsv3-hooks.conf", + ) + + hu = subparsers.add_parser( + "hooks-uninstall", + help="Remove dsynth hooks installed by dports-dev-env", + ) + hu.add_argument("name", help="Environment name") + hu.add_argument( + "--purge", + action="store_true", + help="Also remove dportsv3-hooks.conf", + ) + + hs = subparsers.add_parser( + "hooks-status", + help="Report whether hooks are installed in the env, and if any are stale", + ) + hs.add_argument("name", help="Environment name") + hs.add_argument( + "--source", + help="Override source dir for staleness comparison", + ) + return parser @@ -88,6 +148,74 @@ def cmd_list(_args: argparse.Namespace) -> int: return 0 +def cmd_status(args: argparse.Namespace) -> int: + import json + import subprocess + require_root() + config = load_config() + validate_cache_root(config.cache_root) + store = EnvironmentStore(config) + state = store.load(args.name) + env_dir = store.env_dir(args.name) + root_mounted = bool(mounts_under(state.root_dir)) + writable = store.writable_dir(args.name) + + # Per-repo git status (branch + short HEAD) for repos that live in + # the env's writable overlay. Best-effort: missing or broken repos + # are reported as null. + def _git_info(repo_rel: str) -> dict | None: + repo = writable / repo_rel + if not (repo / ".git").exists(): + return None + def _run(*args: str) -> str: + r = subprocess.run( + ["git", "-C", str(repo)] + list(args), + text=True, capture_output=True, + ) + return r.stdout.strip() if r.returncode == 0 else "" + return { + "branch": _run("rev-parse", "--abbrev-ref", "HEAD"), + "commit": _run("rev-parse", "--short=12", "HEAD"), + "dirty": bool(_run("status", "--porcelain")), + } + + print(json.dumps({ + "name": state.name, + "target": state.target, + "origin": state.origin, + "status": state.status, + "backend": state.backend, + "oracle_profile": state.oracle_profile, + "root_mounted": root_mounted, + "env_dir": str(env_dir), + "deltaports": _git_info("work/DeltaPorts"), + "freebsd_ports": _git_info("work/freebsd-ports"), + })) + return 0 + + +def cmd_update(args: argparse.Namespace) -> int: + require_root() + config = load_config() + validate_cache_root(config.cache_root) + store = EnvironmentStore(config) + from .update import update_env + update_env(config, store, args.name, force=args.force) + return 0 + + +def cmd_path(args: argparse.Namespace) -> int: + require_root() + config = load_config() + validate_cache_root(config.cache_root) + store = EnvironmentStore(config) + if not store.env_dir(args.name).is_dir(): + raise UsageError(f"environment not found: {args.name}") + target = store.writable_dir(args.name) if args.writable else store.env_dir(args.name) + print(str(target)) + return 0 + + def cmd_cleanup_mounts(args: argparse.Namespace) -> int: require_root() config = load_config() @@ -179,6 +307,24 @@ def cmd_shell(args: argparse.Namespace) -> int: return 0 +def cmd_exec(args: argparse.Namespace) -> int: + import os + require_root() + if args.quiet: + os.environ["DPORTS_DEV_ENV_QUIET"] = "1" + config = load_config() + validate_cache_root(config.cache_root) + store = EnvironmentStore(config) + argv = list(args.argv) + if argv and argv[0] == "--": + argv = argv[1:] + if not argv: + raise UsageError("dev-env exec requires a command after '--'") + session = EnvironmentSession(config, store) + state = session.prepare(args.name) + return session.exec_command(state, argv, cwd=args.cwd) + + def cmd_create(args: argparse.Namespace) -> int: require_root() config = load_config() @@ -208,6 +354,34 @@ def cmd_create(args: argparse.Namespace) -> int: return result.exit_code +def _hooks_resolve_state(args: argparse.Namespace): + require_root() + config = load_config() + validate_cache_root(config.cache_root) + store = EnvironmentStore(config) + if not store.env_dir(args.name).is_dir(): + raise UsageError(f"environment not found: {args.name}") + return store.load(args.name) + + +def cmd_hooks_install(args: argparse.Namespace) -> int: + from .hooks import cmd_hooks_install as _impl + + return _impl(args, _hooks_resolve_state(args)) + + +def cmd_hooks_uninstall(args: argparse.Namespace) -> int: + from .hooks import cmd_hooks_uninstall as _impl + + return _impl(args, _hooks_resolve_state(args)) + + +def cmd_hooks_status(args: argparse.Namespace) -> int: + from .hooks import cmd_hooks_status as _impl + + return _impl(args, _hooks_resolve_state(args)) + + def dispatch(args: argparse.Namespace) -> int: if args.action is None: build_parser().print_help() @@ -215,10 +389,17 @@ def dispatch(args: argparse.Namespace) -> int: commands = { "create": cmd_create, "shell": cmd_shell, + "exec": cmd_exec, "destroy": cmd_destroy, "sync-dirty": cmd_sync_dirty, "list": cmd_list, "cleanup-mounts": cmd_cleanup_mounts, + "status": cmd_status, + "update": cmd_update, + "path": cmd_path, + "hooks-install": cmd_hooks_install, + "hooks-uninstall": cmd_hooks_uninstall, + "hooks-status": cmd_hooks_status, } return commands[args.action](args) diff --git a/scripts/tools/dev-env/dports_dev_env/dsynth.py b/scripts/tools/dev-env/dports_dev_env/dsynth.py index 9002ad88546..b1bff1e30f3 100644 --- a/scripts/tools/dev-env/dports_dev_env/dsynth.py +++ b/scripts/tools/dev-env/dports_dev_env/dsynth.py @@ -11,8 +11,19 @@ def dsynth_profile_name(state: EnvironmentState) -> str: return sanitize_name(state.name) +def env_dsynth_etc_dir(state: EnvironmentState) -> Path: + """Per-env /etc/dsynth path (mounted view). + + Single source of truth for "where dsynth configuration lives in an + env." Used by ``write_dsynth_config`` and by hook install/status. + Requires the env to be mounted — when unmounted, this path either + doesn't exist or points at the read-only base layer. + """ + return state.root_dir / "etc/dsynth" + + def write_dsynth_config(config: DevEnvConfig, state: EnvironmentState) -> None: - config_dir = state.root_dir / "etc/dsynth" + config_dir = env_dsynth_etc_dir(state) dsynth_root = state.root_dir / "work/dsynth" profile_name = dsynth_profile_name(state) for path in [ diff --git a/scripts/tools/dev-env/dports_dev_env/helpers.py b/scripts/tools/dev-env/dports_dev_env/helpers.py index c055cf1e14b..56596fc21da 100644 --- a/scripts/tools/dev-env/dports_dev_env/helpers.py +++ b/scripts/tools/dev-env/dports_dev_env/helpers.py @@ -110,27 +110,36 @@ def write_helper_scripts(root_dir, *, bin_dir: str | None = None) -> None: path.chmod(0o755) -def write_shell_rc(state: EnvironmentState) -> None: +def build_env_dict(state: EnvironmentState) -> dict[str, str]: profile_name = dsynth_profile_name(state) + helper_bin = HELPER_BIN_DIR + return { + "DELTAPORTS_ROOT": "/work/DeltaPorts", + "FREEBSD_PORTS_ROOT": "/work/freebsd-ports", + "DPORTS_DEV_ENV": state.name, + "DPORTS_TARGET": state.target, + "DPORTS_ORIGIN": state.origin, + "DPORTS_COMPOSE_ROOT": f"/work/artifacts/compose/{state.target}", + "DPORTS_LOCK_ROOT": "/work/DPorts", + "DPORTS_DSYNTH_ROOT": "/work/dsynth", + "DPORTS_DSYNTH_PROFILE": profile_name, + "DPORTS_TOUCHED_ORIGINS_FILE": str(TOUCHED_ORIGINS_PATH), + "DPORTS_HELPER_BIN": helper_bin, + "DPORTS_ORACLE_PROFILE": state.oracle_profile, + "DISTDIR": "/usr/distfiles", + "DPORTS_DOC_USER_GUIDE": "https://github.com/DragonFlyBSD/DeltaPorts/blob/master/docs/dportsv3-user-guide.md", + "DPORTS_DOC_DEV_ENV": "https://github.com/DragonFlyBSD/DeltaPorts/blob/master/docs/dev-chroot-environment.md", + "PATH": f"{helper_bin}:/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin", + } + + +def write_shell_rc(state: EnvironmentState) -> None: + env = build_env_dict(state) + exports = "\n".join(f"export {k}={quote(v)}" for k, v in env.items()) root_file = state.root_dir / "root/.dports-dev-env.sh" root_file.parent.mkdir(parents=True, exist_ok=True) root_file.write_text( - f"""export DELTAPORTS_ROOT=/work/DeltaPorts -export FREEBSD_PORTS_ROOT=/work/freebsd-ports -export DPORTS_DEV_ENV={quote(state.name)} -export DPORTS_TARGET={quote(state.target)} -export DPORTS_ORIGIN={quote(state.origin)} -export DPORTS_COMPOSE_ROOT={quote(f'/work/artifacts/compose/{state.target}')} -export DPORTS_LOCK_ROOT=/work/DPorts -export DPORTS_DSYNTH_ROOT=/work/dsynth -export DPORTS_DSYNTH_PROFILE={quote(profile_name)} -export DPORTS_TOUCHED_ORIGINS_FILE={quote(str(TOUCHED_ORIGINS_PATH))} -export DPORTS_HELPER_BIN={quote(HELPER_BIN_DIR)} -export DPORTS_ORACLE_PROFILE={quote(state.oracle_profile)} -export DISTDIR=/usr/distfiles -export DPORTS_DOC_USER_GUIDE=https://github.com/DragonFlyBSD/DeltaPorts/blob/master/docs/dportsv3-user-guide.md -export DPORTS_DOC_DEV_ENV=https://github.com/DragonFlyBSD/DeltaPorts/blob/master/docs/dev-chroot-environment.md -export PATH="$DPORTS_HELPER_BIN:/usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin" + f"""{exports} if [ -n "$DPORTS_ORIGIN" ] && [ -d "$DPORTS_COMPOSE_ROOT/$DPORTS_ORIGIN" ]; then cd "$DPORTS_COMPOSE_ROOT/$DPORTS_ORIGIN" diff --git a/scripts/tools/dev-env/dports_dev_env/hooks.py b/scripts/tools/dev-env/dports_dev_env/hooks.py new file mode 100644 index 00000000000..aa4a22ce3e0 --- /dev/null +++ b/scripts/tools/dev-env/dports_dev_env/hooks.py @@ -0,0 +1,249 @@ +"""dsynth hook install/uninstall/status for a dev-env. + +Shares the same path resolution (``env_dsynth_etc_dir`` in +``dsynth.py``) as ``write_dsynth_config`` so there's one source of +truth for "where dsynth configuration lives in an env." That path is +the mounted view (``state.root_dir/etc/dsynth``), which the bind-mount +on ``writable/etc_dsynth`` makes writable while the env is mounted. + +Hooks-install therefore requires the env to be mounted, matching the +existing dsynth-config convention. The CLI handler errors with a +helpful message when the env isn't mounted. +""" + +from __future__ import annotations + +import shutil +import stat +from argparse import Namespace +from pathlib import Path + +from .dsynth import env_dsynth_etc_dir +from .mounts import mounts_under +from .state import EnvironmentState + +# Files we ship as executable hook scripts (chmod 0755 on install). +HOOK_SCRIPTS: tuple[str, ...] = ( + "hook_common.sh", + "hook_pkg_failure", + "hook_pkg_ignored", + "hook_pkg_skipped", + "hook_pkg_start", + "hook_pkg_started", + "hook_pkg_success", + "hook_run_end", + "hook_run_start", +) + +# Example config — copied as ``dportsv3-hooks.conf`` only if no +# operator-edited config exists yet. +CONF_EXAMPLE = "dportsv3-hooks.conf.example" +CONF_TARGET = "dportsv3-hooks.conf" + + +def repo_hook_source() -> Path: + """Path to the repo's scripts/dsynth-hooks/ directory. + + Walks up from this module: parents[0]=dports_dev_env, + parents[1]=dev-env, parents[2]=tools, parents[3]=scripts. + """ + return Path(__file__).resolve().parents[3] / "dsynth-hooks" + + +def install_hooks( + target_dir: Path, + source_dir: Path | None = None, + *, + force: bool = False, +) -> tuple[list[str], list[str]]: + """Copy hook scripts + (optionally) a default conf into ``target_dir``. + + Returns (written_files, skipped_notes). ``dportsv3-hooks.conf`` is + written from the example only if it doesn't exist or ``force`` is + set. Hook scripts are always replaced (they're code, not config). + """ + src = source_dir or repo_hook_source() + if not src.is_dir(): + raise FileNotFoundError(f"source dir not found: {src}") + target_dir.mkdir(parents=True, exist_ok=True) + + written: list[str] = [] + skipped: list[str] = [] + + for name in HOOK_SCRIPTS: + sfile = src / name + if not sfile.is_file(): + raise FileNotFoundError(f"missing hook in source: {sfile}") + dfile = target_dir / name + shutil.copy2(sfile, dfile) + dfile.chmod( + dfile.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + ) + written.append(name) + + src_conf = src / CONF_EXAMPLE + dst_conf = target_dir / CONF_TARGET + if src_conf.is_file(): + if dst_conf.exists() and not force: + skipped.append(f"{CONF_TARGET} (exists; --force to overwrite)") + else: + shutil.copy2(src_conf, dst_conf) + written.append(CONF_TARGET) + + return written, skipped + + +def uninstall_hooks( + target_dir: Path, *, purge: bool = False +) -> list[str]: + """Remove hook scripts (and config if ``purge``). Returns removed names.""" + if not target_dir.is_dir(): + return [] + removed: list[str] = [] + for name in HOOK_SCRIPTS: + path = target_dir / name + if path.exists(): + path.unlink() + removed.append(name) + conf = target_dir / CONF_TARGET + if conf.exists() and purge: + conf.unlink() + removed.append(CONF_TARGET) + return removed + + +def status_hooks( + target_dir: Path, source_dir: Path | None = None +) -> dict[str, object]: + """Return a dict describing what's installed vs. the source. + + Keys: present, missing, stale, conf_present. + """ + if not target_dir.is_dir(): + return { + "present": [], + "missing": list(HOOK_SCRIPTS), + "stale": [], + "conf_present": False, + "exists": False, + } + src = source_dir or repo_hook_source() + src_mtimes: dict[str, float] = {} + if src.is_dir(): + for name in HOOK_SCRIPTS: + sfile = src / name + if sfile.is_file(): + src_mtimes[name] = sfile.stat().st_mtime + + present: list[str] = [] + missing: list[str] = [] + stale: list[str] = [] + for name in HOOK_SCRIPTS: + path = target_dir / name + if not path.exists(): + missing.append(name) + continue + present.append(name) + src_mtime = src_mtimes.get(name) + if src_mtime is not None and path.stat().st_mtime < src_mtime: + stale.append(name) + + return { + "present": present, + "missing": missing, + "stale": stale, + "conf_present": (target_dir / CONF_TARGET).exists(), + "exists": True, + } + + +# ---- CLI argparse handlers ---- + + +def _require_mounted(state: EnvironmentState) -> Path: + """Resolve the env's /etc/dsynth dir, refusing if env isn't mounted.""" + if not mounts_under(state.root_dir): + raise RuntimeError( + f"env '{state.name}' is not mounted. Run " + f"`dportsv3 dev-env shell {state.name}` first to mount it, " + f"then re-run." + ) + return env_dsynth_etc_dir(state) + + +def cmd_hooks_install(args: Namespace, state: EnvironmentState) -> int: + try: + target = _require_mounted(state) + except RuntimeError as exc: + print(str(exc)) + return 1 + source = Path(args.source) if getattr(args, "source", None) else None + try: + written, skipped = install_hooks( + target, source_dir=source, force=bool(args.force) + ) + except FileNotFoundError as exc: + print(str(exc)) + return 1 + + print(f"Installed {len(written)} files into {target}:") + for name in written: + print(f" - {name}") + for note in skipped: + print(f" skipped: {note}") + print() + print("Next steps:") + print(f" 1. Edit {target}/{CONF_TARGET} — set ARTIFACT_STORE_URL,") + print(" DPORTSV3_TRACKER_URL, DPORTSV3_TRACKER_TARGET.") + print(" 2. Hooks are live at /etc/dsynth inside the chroot.") + return 0 + + +def cmd_hooks_uninstall(args: Namespace, state: EnvironmentState) -> int: + try: + target = _require_mounted(state) + except RuntimeError as exc: + print(str(exc)) + return 1 + if not target.is_dir(): + print(f"Nothing to remove: {target} does not exist") + return 0 + removed = uninstall_hooks(target, purge=bool(args.purge)) + if not removed: + print(f"No dportsv3-installed hooks found in {target}") + return 0 + print(f"Removed {len(removed)} files from {target}:") + for name in removed: + print(f" - {name}") + if not args.purge and (target / CONF_TARGET).exists(): + print(f"Preserved {target}/{CONF_TARGET} (pass --purge to remove it too)") + return 0 + + +def cmd_hooks_status(args: Namespace, state: EnvironmentState) -> int: + try: + target = _require_mounted(state) + except RuntimeError as exc: + print(str(exc)) + return 1 + source = Path(args.source) if getattr(args, "source", None) else None + info = status_hooks(target, source_dir=source) + + if not info["exists"]: + print(f"{target}: missing (hooks not installed)") + return 1 + + for name in info["present"]: + marker = " (stale: source is newer)" if name in info["stale"] else "" + print(f" x {name}{marker}") + for name in info["missing"]: + print(f" missing: {name}") + print(f" {'✓' if info['conf_present'] else 'missing:'} {CONF_TARGET}") + print() + present = len(info["present"]) + missing = len(info["missing"]) + stale = len(info["stale"]) + print( + f"{target}: {present} hook(s) installed, {missing} missing, {stale} stale" + ) + return 0 if missing == 0 else 1 diff --git a/scripts/tools/dev-env/dports_dev_env/log.py b/scripts/tools/dev-env/dports_dev_env/log.py index e55045f36e0..44540471e65 100644 --- a/scripts/tools/dev-env/dports_dev_env/log.py +++ b/scripts/tools/dev-env/dports_dev_env/log.py @@ -1,12 +1,24 @@ from __future__ import annotations +import os import sys import time from contextlib import contextmanager from typing import Iterator +def _quiet() -> bool: + """Suppress INFO output when running non-interactively (agent harness). + + Set DPORTS_DEV_ENV_QUIET=1 in the environment to silence INFO lines. + WARN and ERROR are never silenced. + """ + return os.environ.get("DPORTS_DEV_ENV_QUIET") == "1" + + def info(message: str) -> None: + if _quiet(): + return print(f"INFO: {message}", file=sys.stderr) diff --git a/scripts/tools/dev-env/dports_dev_env/runtime.py b/scripts/tools/dev-env/dports_dev_env/runtime.py index 26580424d92..ba65dd62d84 100644 --- a/scripts/tools/dev-env/dports_dev_env/runtime.py +++ b/scripts/tools/dev-env/dports_dev_env/runtime.py @@ -72,4 +72,14 @@ def prepare_root_runtime(config: Config, root_dir: Path, *, refresh_resolv_conf: distfiles_target = root_dir / "usr/distfiles" if mount_null(config.host_distdir, distfiles_target): mounted_targets.append(distfiles_target) + # Bind-mount the repo mirror cache so the env's git origin URLs + # (recorded at clone time as host paths under config.repos_dir) + # resolve from inside the chroot. Read-only — operators inside the + # env shouldn't mutate the shared cache; `dportsv3 dev-env update` + # is the supported way to refresh it. + if config.repos_dir.is_dir(): + repos_target = root_dir / config.repos_dir.relative_to("/") + repos_target.parent.mkdir(parents=True, exist_ok=True) + if mount_null(config.repos_dir, repos_target, read_only=True): + mounted_targets.append(repos_target) return mounted_targets diff --git a/scripts/tools/dev-env/dports_dev_env/session.py b/scripts/tools/dev-env/dports_dev_env/session.py index 49e1b2c4ac7..e32447f3e0d 100644 --- a/scripts/tools/dev-env/dports_dev_env/session.py +++ b/scripts/tools/dev-env/dports_dev_env/session.py @@ -1,13 +1,14 @@ from __future__ import annotations +import shlex from pathlib import Path from .base import find_ready_provisioned_base -from .chroot import command_exists, exec_shell +from .chroot import ChrootRunner, chroot_env, command_exists, exec_shell from .config import DevEnvConfig from .dsynth import write_dsynth_config from .errors import UsageError -from .helpers import HELPER_BIN_DIR, write_helper_scripts, write_shell_rc +from .helpers import HELPER_BIN_DIR, build_env_dict, write_helper_scripts, write_shell_rc from .log import info, warn from .mounts import mounts_under from .runtime import mount_env_root, mount_env_writable_dirs, prepare_env_writable_dirs, prepare_root_runtime @@ -20,7 +21,7 @@ def __init__(self, config: DevEnvConfig, store: EnvironmentStore) -> None: self.config = config self.store = store - def enter(self, name: str, *, refresh: bool = False) -> None: + def prepare(self, name: str, *, refresh: bool = False) -> EnvironmentState: state = self.store.load(name) if state.backend != "chroot": raise UsageError(f"unsupported backend in environment: {state.backend}") @@ -30,10 +31,10 @@ def enter(self, name: str, *, refresh: bool = False) -> None: "wait for the in-flight operation or destroy the partial env first" ) if state.status == "failed": - warn(f"environment {name} is marked failed; entering anyway for inspection") + warn(f"environment {name} is marked failed; proceeding anyway for inspection") env_dir = self.store.env_dir(name) - info(f"preparing shell entry for environment {name}") + info(f"preparing environment {name}") self.ensure_root_mounted(env_dir, state) if refresh or not (state.root_dir / "etc/dsynth/dsynth.ini").is_file(): write_dsynth_config(self.config, state) @@ -43,12 +44,25 @@ def enter(self, name: str, *, refresh: bool = False) -> None: if not (state.root_dir / HELPER_BIN_DIR.lstrip("/") / "dbuild").exists(): warn("helper scripts missing in /usr/local/bin; recreate the env to pick up current helpers") prepare_root_runtime(self.config, state.root_dir, refresh_resolv_conf=refresh) + return state + + def enter(self, name: str, *, refresh: bool = False) -> None: + state = self.prepare(name, refresh=refresh) info(f"entering shell for env={name} target={state.target} origin={state.origin or ''}") info(f"compose root will be /work/artifacts/compose/{state.target}") if not command_exists(state.root_dir, "bash"): warn("bash is unavailable in the environment; falling back to /bin/sh") exec_shell(state.root_dir) + def exec_command(self, state: EnvironmentState, argv: list[str], *, cwd: str) -> int: + if not argv: + raise UsageError("dev-env exec requires a command to run") + env = chroot_env() | build_env_dict(state) + wrapped = ["/bin/sh", "-c", f'cd {shlex.quote(cwd)} && exec "$@"', "_", *argv] + info(f"exec in env={state.name} cwd={cwd}: {' '.join(argv)}") + result = ChrootRunner(state.root_dir).run(wrapped, env=env) + return result.returncode + def ensure_root_mounted(self, env_dir: Path, state: EnvironmentState) -> None: if mounts_under(state.root_dir): prepare_env_writable_dirs(env_dir) diff --git a/scripts/tools/dev-env/dports_dev_env/update.py b/scripts/tools/dev-env/dports_dev_env/update.py new file mode 100644 index 00000000000..54b62762fe8 --- /dev/null +++ b/scripts/tools/dev-env/dports_dev_env/update.py @@ -0,0 +1,116 @@ +"""Refresh the env's git repos from the host's mirror cache. + +Two-phase operation: + 1. Refresh the bare mirrors under ``config.repos_dir`` from the host's + working trees (``RepoCache.refresh_all`` — same logic the builder + uses at env create time). + 2. From the host, run ``git fetch`` + ``git pull --ff-only`` against + each repo's host-side checkout under ``env_dir/writable/work/``. + +The bind-mount of ``repos_dir`` into the chroot (added in runtime.py) +lets ``git pull`` work from inside the env shell too — but the +operator-facing ``dportsv3 dev-env update`` command does the work from +the host, so it doesn't depend on the chroot being mounted. +""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from .builder import default_delta_root +from .config import DevEnvConfig +from .errors import CommandError, UsageError +from .log import info, step_timer +from .repos import RepoCache +from .state import EnvironmentState +from .store import EnvironmentStore + + +# Repos in the env to fast-forward. Each tuple is (label, +# writable-relative checkout path). DPorts is deliberately omitted — +# it's compose-generated, not a git checkout. +ENV_REPOS: list[tuple[str, str]] = [ + ("DeltaPorts", "work/DeltaPorts"), + ("freebsd-ports", "work/freebsd-ports"), +] + + +def _run(cmd: list[str]) -> str: + result = subprocess.run(cmd, text=True, capture_output=True) + if result.returncode != 0: + raise CommandError( + f"command failed: {' '.join(cmd)}\nstderr: {result.stderr.strip()}" + ) + return result.stdout.strip() + + +def _git(repo: Path, *args: str) -> str: + return _run(["git", "-C", str(repo)] + list(args)) + + +def _is_dirty(repo: Path) -> bool: + """True when the working tree has any uncommitted changes.""" + out = _run(["git", "-C", str(repo), "status", "--porcelain"]) + return bool(out.strip()) + + +def _current_branch(repo: Path) -> str: + try: + out = _git(repo, "rev-parse", "--abbrev-ref", "HEAD") + return out.strip() or "HEAD" + except CommandError: + return "HEAD" + + +def update_env( + config: DevEnvConfig, + store: EnvironmentStore, + name: str, + *, + force: bool = False, +) -> None: + """Refresh mirrors + fast-forward the env's git checkouts.""" + state: EnvironmentState = store.load(name) + delta_root = ( + Path(state.source.delta_root) if state.source.delta_root else default_delta_root() + ) + if not delta_root.is_dir(): + raise UsageError(f"delta_root not found: {delta_root}") + + # 1. Refresh mirrors from the host's working trees. + info(f"[1/2] Refreshing repo mirrors from {delta_root}") + with step_timer("refresh mirrors"): + RepoCache(config).refresh_all(delta_root) + + # 2. Fast-forward each repo checkout in the env. + writable = store.writable_dir(name) + info("[2/2] Fast-forwarding env repos") + for label, rel in ENV_REPOS: + repo_path = writable / rel + if not (repo_path / ".git").exists(): + info(f" {label}: skipped (no .git at {repo_path})") + continue + branch = _current_branch(repo_path) + if _is_dirty(repo_path): + if not force: + raise UsageError( + f"{label} has uncommitted changes at {repo_path} " + f"(branch {branch}); rerun with --force to override" + ) + info(f" {label}: dirty (branch {branch}); --force given, proceeding") + before = _git(repo_path, "rev-parse", "HEAD") + _git(repo_path, "fetch", "--prune", "origin") + try: + _git(repo_path, "pull", "--ff-only", "origin", branch) + except CommandError as exc: + raise UsageError( + f"{label} branch {branch} cannot fast-forward — " + f"divergent history or wrong branch. Resolve manually: {exc}" + ) + after = _git(repo_path, "rev-parse", "HEAD") + if before == after: + info(f" {label}: already at {before[:12]} (branch {branch})") + else: + info(f" {label}: {before[:12]} -> {after[:12]} (branch {branch})") + info(f"environment {name} updated")