diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml new file mode 100644 index 0000000..64d72e8 --- /dev/null +++ b/.github/workflows/test-unit.yml @@ -0,0 +1,14 @@ +name: Unit tests + +on: + push: + branches: ["**"] + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run unit tests + run: ./test-unit.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..474544d --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ + +# local scratch: OpenRouter credentials, not for commit +open_router +test.sh + +__pycache__/ +*.pyc + +tests/outputs/ diff --git a/README.md b/README.md index 4056238..8dd1bbb 100644 --- a/README.md +++ b/README.md @@ -85,12 +85,125 @@ api-cli run module/loki1/get-configuration { "retention_days": 7, "active_from": "2021-05-28T15:49:27Z+00:00", - "active_to": "2021-05-28T15:49:27Z+00:00" + "active_to": "2021-05-28T15:49:27Z+00:00", + "insights": { + "status": "active", + "base_url": "https://insights.nethesis.it", + "verify_tls": true, + "subscription_configured": true, + "last_run": "Wed 2026-08-07 14:00:11 UTC" + } } ``` Note: `active_to` field WILL miss if the instance is still active. +### `set-insights` + +Configure the insights collector. On its 15-minute timer it collects a window +of the cluster journal, scrubs likely secrets, masks variable text, +deduplicates the result into counted templates, and ships the bundle to the +Nethesis insights service, where the actual (LLM-based) analysis happens. +The node performs no analysis and holds no LLM credential. Disabled by +default. + +#### Parameters + +- `active`: enable or disable `insights-collector.timer`. Required. +- `base_url`: base URL of the insights server. Required when `active` is + `true`. +- `verify_tls`: verify the server TLS certificate. Optional, default `true`. + Set to `false` only for a self-signed test server — never against a + production endpoint. + +No API key is required or accepted any more. Identity comes from the node's +existing NethServer subscription: the collector reads `system_id` and its +secret from the `cluster/subscription` Redis hash at run time and +authenticates as `Authorization: Basic base64(system_id:secret)`. A node with +no subscription ships nothing and says so in the journal. + +#### Example + +```bash +api-cli run module/loki1/set-insights --data '{ + "active": true, + "base_url": "https://insights.nethesis.it" +}' +``` + +Disable it again: + +```bash +api-cli run module/loki1/set-insights --data '{"active": false}' +``` + +#### Findings + +Findings are no longer written to the local journal: analysis happens on the +insights server, and findings are read back through its API, not through this +module. The node's only journal output is operational, one line per window +under `SYSLOG_IDENTIFIER=loki1/insights-collector`: a +`shipped N templates, M lines -> 202` line on success, an error line +otherwise. + +Check the collector's own health with: + +```bash +runagent -m loki1 journalctl --user -u insights-collector +``` + +#### Manual execution + +The collector is also a plain CLI with three flags, so systemd invokes it +with none: + +```bash +# See exactly what would leave the node before enabling anything. +# No subscription needed, no server URL needed, nothing is shipped. +runagent -m loki1 ../bin/insights-collector --print +``` + +`runagent` changes directory to the module state directory, hence the +`../bin/` prefix. + +| Flag | Effect | +|------|--------| +| `--print` | build the bundle and write it to stdout instead of shipping; needs no subscription and no server URL | +| `--max-lines N` | cap on log lines read per window, before deduplication. Default `500` | +| `--minutes N` | window size in minutes. Default `15` | + +A run covers one window and exits. A failure is loud and costs exactly one +window: the next timer fire retries. + +#### Sizing + +What ships is deduplicated *templates*, not raw log lines, so the outbound +volume is far below the raw line count of a window. `--max-lines` caps how +many lines are read per window before deduplication, 500 by default. Check +your own figure with `--print` before enabling the timer. + +#### Privacy + +What leaves the node is masked, deduplicated log templates plus per-module +counts. A template still carries the fixed text of the log messages it stands +for — that text is the signal — but the variable parts are replaced and +identical events collapse into one counted entry, so no line is sent verbatim. +Two passes run before anything is sent: +`imageroot/pypkg/insights/scrub.py` removes likely secrets +(`password=`, `token=`, `api_key=`, `Authorization` headers, long base64 +runs, email addresses), and `imageroot/pypkg/insights/masking.py` replaces +variable text (timestamps, PIDs, addresses, UUIDs and similar) so that +repeated events collapse to one template. This is defence in depth, not a +guarantee. + +The destination is the Nethesis insights service, authenticated with the +subscription identity (`system_id` and secret) the node already holds — +nothing new to provision or store. This is an explicit improvement over the +previous design: no third-party LLM API key is stored on any node any more, +and `state/secrets.env` no longer exists. + +The feature is disabled by default. + ## Uninstall To uninstall the instance: diff --git a/docs/dev-environment-restore.md b/docs/dev-environment-restore.md new file mode 100644 index 0000000..afaaeaf --- /dev/null +++ b/docs/dev-environment-restore.md @@ -0,0 +1,104 @@ +# Restoring the rl1 dev environment + +Steps to rebuild the `rl1.leader.default.gs.nethserver.net` dev box back to +the state it was in during the `anomaly_detector` branch work, after it gets +torn down. Written for an agent with no memory of this session. + +**Deliberately excluded** (ask the user/operator, never store in this repo): +the cluster subscription `auth_token`, and any `system_id` value read back +from `cluster/subscription`. Everything else below is reproducible from +public repo state. + +## 1. Provision + install NS8 + +Use the `accessing-nethserver-test-vps` skill (or, if unavailable, +`ns8-terraform-infra`'s `tofu apply -var 'leader_node={"dn1":"rl1"}'`, then +NS8 core install + `create-cluster`). Set the real admin password per that +skill — do not leave the default cluster-admin password from +`create-cluster` in place. + +## 2. Enroll the cluster subscription + +Required for the insights feature below to actually authenticate — without +it, `insights-collector` logs "no subscription found" and ships nothing. + +```bash +api-cli run cluster/set-subscription --data '{"subscription":{"auth_token":""}}' +``` + +`` is a Nethesis subscription auth token, ≥32 chars — get it from the +operator, not from any file in this repo. Confirm it landed with: + +```bash +api-cli run cluster/get-subscription +``` + +## 3. Install modules + +```bash +add-module ghcr.io/nethserver/loki:latest 1 # -> loki1 +add-module ghcr.io/nethserver/crowdsec:latest 1 # -> crowdsec1 (+ its firewall-bouncer companion) +``` + +`crowdsec1` was present on the box but **untouched** in this session — no +config changes were made to it. It matters only if continuing the +blocked-IP-evidence work; see +`/home/giacomo/projects/ns8/ns8-crowdsec/crowdsec.plan` on this machine (not +in this repo, not on rl1 — a local planning note) for that follow-on design. + +## 4. Update loki1 to the branch build + +The `anomaly_detector` branch publishes its image via CI on every push — +no local build needed. Command actually used: + +```bash +update-module ghcr.io/nethserver/loki:anomaly_detector loki1 --force +``` + +## 5. Configure the insights collector + +```bash +api-cli run module/loki1/set-insights --data '{ + "active": true, + "base_url": "https://controller.gs.nethserver.net/insights", + "verify_tls": false +}' +``` + +The `/insights` path suffix is required — the server on +`controller.gs.nethserver.net` is path-mounted, not on the bare host (bare +host `/v1/bundles` 404s; `curl -k https://controller.gs.nethserver.net/insights/healthz` +should return `200`). `verify_tls: false` matches that server's self-signed +cert; use `true` against a properly-certified endpoint. + +## 6. Verify + +```bash +# Confirm config landed +api-cli run module/loki1/get-configuration +# -> "insights": {"status": "active", "base_url": "https://controller.gs.nethserver.net/insights", ...} + +# Zero-cost payload check, no shipping +runagent -m loki1 ../bin/insights-collector --print + +# Actually ship one window +runagent -m loki1 ../bin/insights-collector +# or, as the timer would: +runagent -m loki1 systemctl --user start insights-collector.service +runagent -m loki1 journalctl --user -u insights-collector +``` + +## Known state at time of writing (not yet resolved — do not assume fixed) + +- **Real end-to-end shipping against `controller.gs.nethserver.net` was not + confirmed working.** The server accepts auth and rejects malformed bodies + fast, but a real, well-formed bundle causes a request that hangs until the + client's 60s read timeout — looks like a server-side issue (likely a + downstream queue/broker not responding in that dev deployment), not + something fixable from `ns8-loki`. Re-check this before relying on it. +- Local unit tests: `./test-unit.sh` → 106 passed. +- CI on the branch: green as of commit `8b3e86b` (port-collision fix in + `tests/20__insights.robot` — the e2e stub was colliding with + node_exporter's default port 9100 on the test node; moved to 19100). +- PR: https://github.com/NethServer/ns8-loki/pull/70 (draft, no assignee, no + reviewer requested as of last check). diff --git a/docs/superpowers/plans/2026-07-29-loki-anomaly-detector.md b/docs/superpowers/plans/2026-07-29-loki-anomaly-detector.md new file mode 100644 index 0000000..09a9143 --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-loki-anomaly-detector.md @@ -0,0 +1,3399 @@ +# Loki Anomaly Detector Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add an hourly, opt-in job to `ns8-loki` that sends a scrubbed digest of the cluster's journal to an OpenAI-compatible LLM and writes the findings back into the journal. + +**Architecture:** One Python script (`imageroot/bin/anomaly-detector`) driven by a `Type=oneshot` service plus an hourly timer. The script queries Loki (logcli for log lines, the Loki HTTP instant-query API for the rate digest and 7-day baseline), scrubs secrets, renders a prompt, POSTs it to `${ANOMALY_LLM_BASE_URL}/chat/completions`, and prints findings as JSON lines on stdout — which under systemd *is* the journal, so findings round-trip back into Loki and serve as the detector's own memory. A single action (`set-anomaly-detector`) writes config to `state/environment`, secrets to `state/secrets.env`, and enables/disables the timer. + +**Tech Stack:** Python 3.11+ (the `runagent` interpreter), `requests` + `urllib3.util.Retry` (already used by `cloud-log-manager-forwarder`), `logcli`, the NS8 `agent` Python SDK, systemd user units, `pytest` (new to this repo), Robot Framework + SSHLibrary. + +## Global Constraints + +- Every new Python file starts with `#!/usr/bin/env python3` then the exact 4-line header: + ``` + # + # Copyright (C) 2026 Nethesis S.r.l. + # SPDX-License-Identifier: GPL-3.0-or-later + # + ``` +- Executable bits: scripts under `imageroot/bin/` and action steps (`10set`, `10get`, …) are mode **755**. `*.json` schemas and `imageroot/systemd/user/*` unit files are mode **644**. +- All JSON Schema files use draft-04: `"$schema": "http://json-schema.org/draft-04/schema#"` and `"$id": "http://schema.nethserver.org/loki/.json"`. +- Systemd unit files use `%E` (never `%S`) — established by commit `045a6cd`. `%E` is the module's install/config dir; `%E/state` is `AGENT_STATE_DIR`. +- `imageroot/` is added to the image wholesale by `build-images.sh` (`buildah add "${container}" imageroot /imageroot`). **No `build-images.sh` change is needed for any new file.** +- The detector is **disabled by default**. `imageroot/actions/create-module/20systemd` is NOT modified. +- Diagnostics go to **stderr only**. stdout carries findings and the summary line and nothing else, because `identifier="/anomaly-detector"` Loki queries must return findings only. +- Log-line ordering constant used everywhere: the display format is + ` [node_id:module_id:identifier] message`. +- Secrets (`ANOMALY_LLM_API_KEY`, `ANOMALY_WEBHOOK_TOKEN`) NEVER go through `agent.set_env` — that writes `state/environment`, which is mirrored into the Redis hash `module//environment`. + +## Deviations from the design spec (deliberate, with reasons) + +These five points differ from `docs/superpowers/specs/2026-07-29-loki-anomaly-detector-design.md`. They are baked into the tasks below; do not "fix" them back. + +1. **The script has a `main()` and an `if __name__ == "__main__":` guard, and `import agent` is lazy** (inside `_read_state_envfile`). The two existing forwarders are bare top-level scripts. The spec requires `scrub`, `build_digest`, `render_prompt` and `parse_findings` to be unit-testable with no I/O; that is impossible if the module body runs on import, and the `agent` SDK is not installed in the test container. +2. **The rate digest and baseline use the Loki HTTP instant-query API (`GET /loki/api/v1/query`) via `requests`, not `logcli`.** The spec says "three `logcli` invocations". `logcli`'s output format for *metric* queries is not a documented machine-stable contract, whereas `/loki/api/v1/query` returns a typed vector. Log *lines* still go through `logcli query --forward -o jsonl`, matching `cloud-log-manager-forwarder`. Both paths use the same address and basic-auth credentials, so nothing extra is configured. +3. **Scrub rule order is: keyword → authorization → email → base64/hex blob.** The spec's table lists the blob rule before email; running blob first turns a long email local-part into `` and loses the more informative `` marker. +4. **The nominal early-exit still prints the summary line to stdout** (with `"llm_called": false`), then exits 0. The spec says only "log `nominal, no LLM call`, exit 0". Emitting unconditionally means every window is recorded in Loki (so "quiet hours" are graphable) and makes the Robot test deterministic — otherwise a quiet test node produces no journal output and the `SyslogIdentifier` assertion has nothing to match. +5. **Unit tests use `pytest` in a container via a new `test-unit.sh`, modelled on this repo's `test-module.sh`, not on `core/agent/test-agent.sh`.** The spec cites `core/agent/test-agent.sh` as the pytest-in-a-container template; that script actually runs Robot Framework, and no pytest harness exists anywhere in ns8-core. `test-module.sh` is the real local precedent for "venv in a cached podman volume". + +## Verification on the test node + +A live single-node cluster is available for step-by-step verification: + +```bash +ssh root@rl1.leader.default.gs.nethserver.net +``` + +Facts established by probing it on 2026-07-29 (do not re-derive these): + +| Fact | Value | +|------|-------| +| Module instance | `loki1`, running (`loki`, `loki-server`, `traefik` all active) | +| `%E` | `/home/loki1/.config` | +| `AGENT_STATE_DIR` (action/`runagent` CWD) | `/home/loki1/.config/state` | +| Unit directory | `/home/loki1/.config/systemd/user/` | +| `LOKI_HTTP_PORT` | `20000` | +| `logcli` | `/usr/local/bin/logcli`, on PATH under `runagent` | +| Python | 3.11.13 | +| Other modules present | `crowdsec1`, `nethvoice2`, `nethvoice-proxy1`, `metrics1`, `samba2`, `traefik1` | + +Already verified against this node, so treat these as settled: + +- **The instant metric query works.** `GET /loki/api/v1/query` with + `sum by (module_id, priority) (count_over_time({node_id=~".+"} | json priority="PRIORITY" [3600s]))` + returns `status: success`, `resultType: vector`, 11 series. +- **Some series have no `module_id`** (host-level logs, e.g. `sshd-session`), so + the metric label map is `{"priority": "3"}` with no `module_id` key. This is why + `parse_metric_response` maps a missing label to `"unknown"`. +- **The prefiltered lines query works** through `logcli query --forward -o jsonl` + with the full pipeline, including the `identifier !=` exclusion and + `| priority < 5 or category="security"`. Priority-6 `sshd-session` lines are + correctly returned via the `category="security"` branch. +- **Size reality check:** one hour returned **382 lines / ~36k characters / + ~9k tokens** for the `LINES` block alone. That is above the spec's 4–6k target + per window. Keep the documented default of `max_lines: 500` — the spec fixes it + — but expect ~9k tokens on a node of this size and say so in the final + verification, so tuning `max_lines` down is an informed operator choice rather + than a surprise. + +### The LLM endpoint used for verification + +Real-LLM verification uses OpenRouter. The credentials live in `./open_router` in the +repo root, which is git-ignored — **never commit the key, never paste it into a +report, a test file, or a plan**. Read it at use time: + +```bash +ORKEY=$(grep -oE 'sk-or-[A-Za-z0-9._-]+' open_router | head -1) +``` + +| Setting | Value | +|---------|-------| +| `ANOMALY_LLM_BASE_URL` | `https://openrouter.ai/api/v1` | +| `ANOMALY_LLM_MODEL` | `google/gemma-4-26b-a4b-it:free` | + +The base URL is a configured value, never hardcoded: it arrives via the action's +`base_url` field or the `ANOMALY_LLM_BASE_URL` variable, and `ask_llm` appends +`/chat/completions` to it. Any OpenAI-compatible endpoint works unchanged. + +Verified against this endpoint on 2026-07-29 with the plan's exact +`RESPONSE_SCHEMA` and a prompt built from real `rl1` log lines: + +- **`response_format: {"type": "json_schema", "strict": true}` is honoured.** The + reply parsed as JSON and matched the schema, including the `severity` and + `window_assessment` enums. +- **It is load-bearing, not optional.** The identical request *without* + `response_format` came back wrapped in a ```` ```json ```` fence with an invented + shape (`{"deviations": [...]}`), which `parse_findings` would correctly reject. + Never make `response_format` conditional. +- **Prompt quality is real**, not just well-formed: given a `crowdsec1` priority-3 + rate 14× above baseline plus three matching log lines, the model returned one + `high` finding titled "SSH Brute-Force Attack Detected" naming the offending IP, + with `window_assessment: degraded`. +- **Known prompt-adherence wrinkle:** the model put a `RATES` row into `evidence` + even though `SYSTEM_PROMPT` says to quote evidence verbatim from `LINES`. + `parse_findings` does not verify evidence provenance, and this does not block + anything — do not add provenance validation, it is out of scope. + +CI never touches this endpoint. The Robot suite in Task 9 uses the offline stub, so +tests stay deterministic with no egress and no cost. + +### Deploying to the node between tasks + +The module is installed, so files can be synced in place instead of rebuilding +the image. Run from the repo root: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +rsync -a --rsync-path='rsync' imageroot/ ${NODE}:/tmp/imageroot-staged/ +ssh ${NODE} 'cp -a /tmp/imageroot-staged/. /home/loki1/.config/ \ + && chown -R loki1:loki1 /home/loki1/.config \ + && runagent -m loki1 systemctl --user daemon-reload' +``` + +For a script-only change, the spec's documented manual path needs no sync at all: + +```bash +scp imageroot/bin/anomaly-detector ${NODE}:/tmp/ +ssh ${NODE} runagent -m loki1 python3 /tmp/anomaly-detector --dry-run --since 2h +``` + +Each task below states what to verify on the node. Tasks 1–3 are pure functions +with no node step. Tasks 4, 5, 6, 7 and 8 each end with a node check. + +## File Structure + +**Create:** + +| Path | Responsibility | +|------|----------------| +| `imageroot/bin/anomaly-detector` | the whole job + the CLI; pure helpers at top level, all I/O behind `main()` | +| `imageroot/systemd/user/anomaly-detector.service` | `Type=oneshot`, one window per invocation | +| `imageroot/systemd/user/anomaly-detector.timer` | `OnCalendar=hourly`, `Persistent=true` | +| `imageroot/actions/set-anomaly-detector/validate-input.json` | draft-04 schema, `oneOf` on `active` | +| `imageroot/actions/set-anomaly-detector/10set` | write env + secrets, enable/disable the timer | +| `imageroot/update-module.d/15systemd` | `systemctl --user daemon-reload` so new units land on an already-installed module | +| `tests/unit/conftest.py` | loads the extension-less script as an importable module | +| `tests/unit/requirements.txt` | `pytest`, `requests` | +| `tests/unit/test_anomaly_detector.py` | pytest suite for the pure helpers | +| `test-unit.sh` | runs pytest in a `python:3.11-alpine` container | +| `.github/workflows/test-unit.yml` | runs `./test-unit.sh` on push/PR | +| `tests/llm-stub.py` | canned OpenAI-shaped HTTP server, copied to the node by the Robot test | +| `tests/20__anomaly_detector.robot` | end-to-end test against the stub | + +**Modify:** + +| Path | Change | +|------|--------| +| `imageroot/actions/get-configuration/10get` | add the `anomaly_detector` object | +| `imageroot/actions/get-configuration/validate-output.json` | declare `anomaly_detector` | +| `imageroot/etc/state-include.conf` | add `state/secrets.env` | +| `README.md` | new `### set-anomaly-detector` API section, manual-run section, privacy statement | + +**Function inventory of `imageroot/bin/anomaly-detector`** (built up across Tasks 1–5; every later task's `Interfaces` block repeats the signatures it needs): + +``` +# pure — unit tested +scrub(line) -> str +sanitize_line(raw) -> str +parse_duration(text) -> timedelta +compute_window(now, since=None) -> (datetime, datetime) +parse_metric_response(payload) -> dict[(str, str), float] +build_digest(rates, baseline) -> list[dict] +is_nominal(lines, digest, tolerance=3.0) -> bool +estimate_tokens(text) -> int +render_prompt(window, digest, recent_findings, lines, truncated) -> str +build_request_body(model, user_prompt) -> dict +extract_content(payload) -> str +parse_findings(body) -> (list[dict], str) +render_findings(findings, assessment, window, truncated, llm_called, pretty) -> list[str] +load_config(args, environ, reader) -> dict + +# I/O +_read_state_envfile(path) -> dict +make_session() -> requests.Session +query_metric(session, addr, auth, query, at) -> dict[(str, str), float] +query_lines(session_unused, window, max_lines, module_id) -> (list[dict], bool) +recall_findings(module_id, limit=10) -> list[dict] +ask_llm(session, config, prompt) -> str +post_webhook(session, config, payload) -> None +main(argv=None) -> int +``` + +--- + +### Task 1: Unit-test harness and `scrub()` + +Creates the script file, its import-safe skeleton, the pytest harness, and the first pure function. + +**Files:** +- Create: `imageroot/bin/anomaly-detector` +- Create: `tests/unit/requirements.txt` +- Create: `tests/unit/conftest.py` +- Create: `tests/unit/test_anomaly_detector.py` +- Create: `test-unit.sh` +- Create: `.github/workflows/test-unit.yml` + +**Interfaces:** +- Consumes: nothing. +- Produces: `scrub(line) -> str`; `sanitize_line(raw) -> str`; the module-level constant `SCRUB_RULES`; the conftest fixture `ad` (the loaded module object) available to every later test file. + +- [ ] **Step 1: Write the failing test** + +Create `tests/unit/requirements.txt`: + +``` +pytest +requests +``` + +Create `tests/unit/conftest.py`: + +```python +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import importlib.machinery +import importlib.util +import pathlib + +import pytest + +SCRIPT = pathlib.Path(__file__).resolve().parents[2] / "imageroot" / "bin" / "anomaly-detector" + + +def _load(): + loader = importlib.machinery.SourceFileLoader("anomaly_detector", str(SCRIPT)) + spec = importlib.util.spec_from_loader("anomaly_detector", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def ad(): + """The anomaly-detector script loaded as a module. + + The file has no .py extension, so it cannot be imported normally. + Loading it must not perform I/O nor import the `agent` SDK. + """ + return _load() +``` + +Create `tests/unit/test_anomaly_detector.py`: + +```python +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import pytest + + +class TestScrub: + @pytest.mark.parametrize("raw", [ + 'login failed password=hunter2', + 'login failed password: hunter2', + 'sent Bearer eyJhbGciOi', + 'using api_key=abc123def', + 'using api-key: abc123def', + 'set SECRET="s3kr1t"', + 'cfg passwd=root pwd=root', + 'token=ghp_0123456789', + ]) + def test_secret_keywords_are_redacted(self, ad, raw): + out = ad.scrub(raw) + assert "" in out + for leaked in ("hunter2", "eyJhbGciOi", "abc123def", "s3kr1t", "ghp_0123456789"): + assert leaked not in out + + @pytest.mark.parametrize("raw", [ + 'password reset requested for user bob', + 'the token bucket is full', + 'secret santa module started', + ]) + def test_keyword_without_assignment_is_kept(self, ad, raw): + assert ad.scrub(raw) == raw + + def test_authorization_header_is_redacted(self, ad): + out = ad.scrub('GET /api Authorization: Basic bG9raTpwYXNz') + assert out == 'GET /api authorization: ' + + def test_authorization_word_alone_is_kept(self, ad): + raw = 'authorization succeeded for node 3' + assert ad.scrub(raw) == raw + + def test_long_blob_is_redacted(self, ad): + raw = 'cookie 0123456789abcdef0123456789abcdef0123' + assert ad.scrub(raw) == 'cookie ' + + def test_short_hex_is_kept(self, ad): + raw = 'commit 6417fba failed' + assert ad.scrub(raw) == raw + + def test_email_is_redacted(self, ad): + out = ad.scrub('bounce to admin@example.org failed') + assert out == 'bounce to failed' + + def test_bare_at_sign_is_kept(self, ad): + raw = 'resolved loki@cluster to loki1' + assert ad.scrub(raw) == raw + + def test_signal_carrying_values_survive(self, ad): + raw = 'nethvoice2: 192.168.1.44 -> rl1.example.com refused for user bob' + out = ad.scrub(raw) + assert '192.168.1.44' in out + assert 'nethvoice2' in out + assert 'rl1.example.com' in out + assert 'bob' in out + + def test_email_rule_wins_over_blob_rule(self, ad): + # A 32+ char local part must still be reported as an email, not a blob. + out = ad.scrub('mail to abcdefghijabcdefghijabcdefghijabc@example.org') + assert out == 'mail to ' + + def test_scrub_is_idempotent(self, ad): + once = ad.scrub('password=hunter2 and admin@example.org') + assert ad.scrub(once) == once + + +class TestSanitizeLine: + def test_embedded_newlines_are_collapsed(self, ad): + # Observed on a live node: crowdsec messages carry a trailing \n, and a + # multi-line message would otherwise forge extra lines in the prompt's + # LINES block. + raw = '<3> [1:crowdsec1:crowdsec1] alert\nlevel=info msg="x"\n' + assert ad.sanitize_line(raw) == '<3> [1:crowdsec1:crowdsec1] alert level=info msg="x"' + + def test_carriage_returns_and_tabs_are_collapsed(self, ad): + assert ad.sanitize_line('a\r\nb\tc') == 'a b c' + + def test_runs_of_whitespace_collapse_to_one_space(self, ad): + assert ad.sanitize_line('a b') == 'a b' + + def test_it_also_scrubs(self, ad): + assert ad.sanitize_line('token=abc\ndef') == 'token= def' + + def test_empty_and_blank(self, ad): + assert ad.sanitize_line('') == '' + assert ad.sanitize_line(' \n ') == '' + + def test_a_forged_fence_cannot_escape_the_block(self, ad): + # A log message must never be able to close the LINES fence. + out = ad.sanitize_line('boom\n```\nWINDOW\n```') + assert '\n' not in out +``` + +Create `test-unit.sh` (mode 755): + +```bash +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# Run the anomaly-detector unit tests in a container. +# +# ./test-unit.sh [PYTEST ARG]... + +set -e + +venvroot=/usr/local/venv + +exec podman run -i --rm \ + --volume=.:/srv/source:z \ + --volume=pytest-cache:${venvroot}:z \ + --replace --name=pytest-unit \ + --env=venvroot \ + docker.io/python:3.11-alpine \ + ash -l -s -- "${@}" <<'EOF' +set -e +if [ ! -x ${venvroot}/bin/pytest ] ; then + python3 -mvenv ${venvroot} --upgrade + ${venvroot}/bin/pip3 install -q -r /srv/source/tests/unit/requirements.txt +fi +cd /srv/source +exec ${venvroot}/bin/pytest -q "${@}" tests/unit/ +EOF +``` + +Create `.github/workflows/test-unit.yml`: + +```yaml +name: Unit tests + +on: + push: + branches: ["**"] + pull_request: + +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run unit tests + run: ./test-unit.sh +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +chmod +x test-unit.sh +./test-unit.sh +``` + +Expected: collection error — `FileNotFoundError` / `No such file or directory: '.../imageroot/bin/anomaly-detector'`. + +- [ ] **Step 3: Write the minimal implementation** + +Create `imageroot/bin/anomaly-detector` (mode 755): + +```python +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# +# Analyse one hour of NS8 journal logs with a remote LLM and write the +# findings back to the journal. Diagnostics go to stderr; stdout carries +# findings only. +# + +import argparse +import json +import os +import re +import subprocess +import sys +from datetime import datetime, timedelta, timezone + +#-------------------------------- SCRUB ---------------------------------# + +# Ordered: keyword assignments, then Authorization headers, then email +# addresses, then long opaque blobs. Email runs before the blob rule so a +# long local part is still reported as an email. +SCRUB_RULES = [ + ( + re.compile(r'(?i)\b(bearer|tokens?|api[-_]?keys?|secrets?|passwords?|passwd|pwd)\b[=:\s"\']+\S+'), + r'\1=', + ), + ( + re.compile(r'(?i)\bauthorization:\s*\S+(?:\s+\S+)?'), + 'authorization: ', + ), + ( + re.compile(r'\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)+\b'), + '', + ), + ( + re.compile(r'\b[A-Za-z0-9+/]{32,}={0,2}\b'), + '', + ), +] + + +WHITESPACE_RUN = re.compile(r'\s+') + + +def scrub(line): + """Remove likely secrets from a log line. + + Defence in depth, not a guarantee. IP addresses, hostnames, module IDs + and usernames are deliberately preserved: they carry the signal. + """ + for pattern, replacement in SCRUB_RULES: + line = pattern.sub(replacement, line) + return line + + +def sanitize_line(raw): + """Flatten a collected log line to exactly one prompt line, then scrub. + + Journal messages can contain newlines. Left alone they would break the + one-record-per-line structure of the prompt's LINES block, letting a log + message forge additional lines or close the fence. + """ + return scrub(WHITESPACE_RUN.sub(' ', raw).strip()) +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +./test-unit.sh +``` + +Expected: all `TestScrub` tests PASS. + +If `test_secret_keywords_are_redacted` fails on `'sent Bearer eyJhbGciOi'` because the replacement emits `Bearer=` — that is correct and expected; the assertion only checks that `` is present and the value is gone. + +- [ ] **Step 5: Commit** + +```bash +git add imageroot/bin/anomaly-detector tests/unit test-unit.sh .github/workflows/test-unit.yml +git commit -m "feat(anomaly-detector): add script skeleton, scrub() and pytest harness" +``` + +--- + +### Task 2: Window arithmetic and the rate digest + +**Files:** +- Modify: `imageroot/bin/anomaly-detector` (append after `scrub`) +- Modify: `tests/unit/test_anomaly_detector.py` (append test classes) + +**Interfaces:** +- Consumes: nothing from Task 1 beyond the module skeleton. +- Produces: + - `parse_duration(text) -> timedelta` — accepts `30m`, `6h`, `2d`; raises `ValueError` otherwise. + - `compute_window(now, since=None) -> (datetime, datetime)` + - `parse_metric_response(payload) -> dict[(module_id, priority) -> float]` + - `build_digest(rates, baseline) -> list[dict]` with keys `module_id`, `priority`, `observed`, `expected`, `ratio` (`ratio` is `None` when `expected == 0`). + - `is_nominal(lines, digest, tolerance=3.0) -> bool` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/unit/test_anomaly_detector.py`: + +```python +from datetime import datetime, timedelta, timezone + + +class TestParseDuration: + @pytest.mark.parametrize("text,expected", [ + ("30m", timedelta(minutes=30)), + ("6h", timedelta(hours=6)), + ("2d", timedelta(days=2)), + (" 1h ", timedelta(hours=1)), + ]) + def test_valid(self, ad, text, expected): + assert ad.parse_duration(text) == expected + + @pytest.mark.parametrize("text", ["", "h", "1w", "1.5h", "-1h", "1 h", "abc"]) + def test_invalid(self, ad, text): + with pytest.raises(ValueError): + ad.parse_duration(text) + + +class TestComputeWindow: + def test_default_is_the_previous_full_hour(self, ad): + now = datetime(2026, 7, 29, 14, 7, 33, tzinfo=timezone.utc) + start, end = ad.compute_window(now) + assert start == datetime(2026, 7, 29, 13, 0, 0, tzinfo=timezone.utc) + assert end == datetime(2026, 7, 29, 14, 0, 0, tzinfo=timezone.utc) + + def test_exactly_on_the_hour(self, ad): + now = datetime(2026, 7, 29, 14, 0, 0, tzinfo=timezone.utc) + start, end = ad.compute_window(now) + assert start == datetime(2026, 7, 29, 13, 0, 0, tzinfo=timezone.utc) + assert end == now + + def test_since_ends_at_now(self, ad): + now = datetime(2026, 7, 29, 14, 7, 33, tzinfo=timezone.utc) + start, end = ad.compute_window(now, since="2h") + assert end == now + assert start == datetime(2026, 7, 29, 12, 7, 33, tzinfo=timezone.utc) + + +class TestParseMetricResponse: + def test_extracts_labels_and_values(self, ad): + payload = { + "status": "success", + "data": { + "resultType": "vector", + "result": [ + {"metric": {"module_id": "nethvoice2", "priority": "6"}, + "value": [1769000000, "656"]}, + {"metric": {"module_id": "loki1", "priority": "3"}, + "value": [1769000000, "4"]}, + ], + }, + } + assert ad.parse_metric_response(payload) == { + ("nethvoice2", "6"): 656.0, + ("loki1", "3"): 4.0, + } + + def test_missing_labels_become_unknown(self, ad): + payload = {"data": {"result": [{"metric": {}, "value": [0, "2"]}]}} + assert ad.parse_metric_response(payload) == {("unknown", "unknown"): 2.0} + + def test_empty_result(self, ad): + assert ad.parse_metric_response({"data": {"result": []}}) == {} + + def test_missing_data_key_raises(self, ad): + with pytest.raises(ValueError): + ad.parse_metric_response({"status": "error"}) + + +class TestBuildDigest: + def test_pairs_observed_with_expected(self, ad): + rates = {("nethvoice2", "6"): 656.0, ("loki1", "3"): 4.0} + baseline = {("nethvoice2", "6"): 600.0, ("mail1", "4"): 10.0} + assert ad.build_digest(rates, baseline) == [ + {"module_id": "loki1", "priority": "3", + "observed": 4.0, "expected": 0.0, "ratio": None}, + {"module_id": "mail1", "priority": "4", + "observed": 0.0, "expected": 10.0, "ratio": 0.0}, + {"module_id": "nethvoice2", "priority": "6", + "observed": 656.0, "expected": 600.0, "ratio": 1.09}, + ] + + def test_rounds_to_two_decimals(self, ad): + rates = {("a", "6"): 1.0 / 3.0} + baseline = {("a", "6"): 1.0 / 7.0} + row = ad.build_digest(rates, baseline)[0] + assert row["observed"] == 0.33 + assert row["expected"] == 0.14 + assert row["ratio"] == 2.33 + + def test_empty_inputs(self, ad): + assert ad.build_digest({}, {}) == [] + + +class TestIsNominal: + def test_lines_present_is_never_nominal(self, ad): + assert ad.is_nominal([{"line": "x"}], []) is False + + def test_no_lines_and_rates_on_baseline(self, ad): + digest = [{"module_id": "a", "priority": "6", + "observed": 100.0, "expected": 90.0, "ratio": 1.11}] + assert ad.is_nominal([], digest) is True + + def test_spike_above_tolerance(self, ad): + digest = [{"module_id": "a", "priority": "6", + "observed": 400.0, "expected": 100.0, "ratio": 4.0}] + assert ad.is_nominal([], digest) is False + + def test_new_pair_with_no_baseline_is_not_nominal(self, ad): + digest = [{"module_id": "new1", "priority": "6", + "observed": 5.0, "expected": 0.0, "ratio": None}] + assert ad.is_nominal([], digest) is False + + def test_pair_that_went_silent_is_nominal(self, ad): + digest = [{"module_id": "a", "priority": "6", + "observed": 0.0, "expected": 50.0, "ratio": 0.0}] + assert ad.is_nominal([], digest) is True + + def test_everything_empty_is_nominal(self, ad): + assert ad.is_nominal([], []) is True +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +./test-unit.sh tests/unit/test_anomaly_detector.py -k "Duration or Window or Metric or Digest or Nominal" +``` + +Expected: FAIL with `AttributeError: module 'anomaly_detector' has no attribute 'parse_duration'`. + +- [ ] **Step 3: Write the minimal implementation** + +Append to `imageroot/bin/anomaly-detector`: + +```python +#-------------------------------- WINDOW --------------------------------# + +DURATION_UNITS = {'m': 'minutes', 'h': 'hours', 'd': 'days'} +DURATION_RE = re.compile(r'^(\d+)([mhd])$') + +# A pair whose observed count exceeds expected by this factor is a spike. +NOMINAL_TOLERANCE = 3.0 + +# Hours in 7 days: the baseline query divisor. +BASELINE_HOURS = 168 + + +def parse_duration(text): + """Parse a duration such as 30m, 6h or 2d into a timedelta.""" + match = DURATION_RE.match(text.strip()) + if not match: + raise ValueError(f"invalid duration {text!r}: expected [mhd]") + return timedelta(**{DURATION_UNITS[match.group(2)]: int(match.group(1))}) + + +def compute_window(now, since=None): + """Return the [start, end) window to analyse. + + Default: the previous full hour, derived from the wall clock, so there + is no cursor file and no drift. With `since`: [now - since, now]. + """ + if since: + end = now + start = end - parse_duration(since) + else: + end = now.replace(minute=0, second=0, microsecond=0) + start = end - timedelta(hours=1) + return start, end + + +def parse_metric_response(payload): + """Turn a Loki instant-query vector into {(module_id, priority): value}.""" + try: + result = payload["data"]["result"] + except (KeyError, TypeError) as exc: + raise ValueError(f"unexpected metric response shape: {exc}") from exc + rates = {} + for entry in result: + metric = entry.get("metric") or {} + key = (metric.get("module_id") or "unknown", metric.get("priority") or "unknown") + rates[key] = float(entry["value"][1]) + return rates + + +def build_digest(rates, baseline): + """Join observed counts with expected counts, sorted for a stable prompt.""" + rows = [] + for key in sorted(set(rates) | set(baseline)): + module_id, priority = key + observed = rates.get(key, 0.0) + expected = baseline.get(key, 0.0) + rows.append({ + "module_id": module_id, + "priority": priority, + "observed": round(observed, 2), + "expected": round(expected, 2), + "ratio": round(observed / expected, 2) if expected > 0 else None, + }) + return rows + + +def is_nominal(lines, digest, tolerance=NOMINAL_TOLERANCE): + """True when the window is not worth an LLM call. + + Requires zero prefiltered lines and no rate pair above tolerance. A + pair with no baseline at all counts as a deviation if it produced + anything: it is new behaviour. + """ + if lines: + return False + for row in digest: + if row["ratio"] is None: + if row["observed"] > 0: + return False + elif row["ratio"] > tolerance: + return False + return True +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +./test-unit.sh +``` + +Expected: every test PASSES. + +- [ ] **Step 5: Commit** + +```bash +git add imageroot/bin/anomaly-detector tests/unit/test_anomaly_detector.py +git commit -m "feat(anomaly-detector): add window arithmetic and rate digest" +``` + +--- + +### Task 3: Prompt rendering and response parsing + +**Files:** +- Modify: `imageroot/bin/anomaly-detector` (append) +- Modify: `tests/unit/test_anomaly_detector.py` (append) + +**Interfaces:** +- Consumes: `scrub(line)` (Task 1); `build_digest` output rows (Task 2). +- Produces: + - `estimate_tokens(text) -> int` + - `SYSTEM_PROMPT` (str constant) + - `RESPONSE_SCHEMA` (dict constant) + - `render_prompt(window, digest, recent_findings, lines, truncated) -> str` — `window` is the `(start, end)` tuple; `recent_findings` is a list of `{"severity", "title"}`; `lines` is a list of already-formatted strings. + - `build_request_body(model, user_prompt) -> dict` + - `extract_content(payload) -> str` + - `parse_findings(body) -> (findings, assessment)`; raises `ValueError` on any schema violation. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/unit/test_anomaly_detector.py`: + +```python +import json + + +WINDOW = ( + datetime(2026, 7, 29, 13, 0, tzinfo=timezone.utc), + datetime(2026, 7, 29, 14, 0, tzinfo=timezone.utc), +) + + +class TestEstimateTokens: + def test_roughly_four_chars_per_token(self, ad): + assert ad.estimate_tokens("a" * 400) == 100 + + def test_empty(self, ad): + assert ad.estimate_tokens("") == 0 + + +class TestRenderPrompt: + def _render(self, ad, **kw): + params = { + "window": WINDOW, + "digest": [{"module_id": "nethvoice2", "priority": "3", + "observed": 40.0, "expected": 2.0, "ratio": 20.0}], + "recent_findings": [{"severity": "high", "title": "asterisk restart loop"}], + "lines": ["<3> [1:nethvoice2:asterisk] registration failed"], + "truncated": False, + } + params.update(kw) + return ad.render_prompt(**params) + + def test_contains_all_four_blocks(self, ad): + out = self._render(ad) + for block in ("WINDOW", "RATES", "RECENT_FINDINGS", "LINES"): + assert block in out + + def test_window_timestamps_are_iso(self, ad): + out = self._render(ad) + assert "2026-07-29T13:00:00+00:00" in out + assert "2026-07-29T14:00:00+00:00" in out + + def test_rate_row_shows_observed_and_expected(self, ad): + out = self._render(ad) + assert "nethvoice2" in out + assert "40.0" in out + assert "2.0" in out + + def test_truncation_is_declared(self, ad): + assert "truncated: yes" in self._render(ad, truncated=True) + assert "truncated: no" in self._render(ad, truncated=False) + + def test_empty_sections_are_explicit_not_blank(self, ad): + out = self._render(ad, recent_findings=[], lines=[], digest=[]) + assert "(none)" in out + + def test_recent_finding_titles_are_present(self, ad): + assert "asterisk restart loop" in self._render(ad) + + def test_prompt_body_is_scrubbed(self, ad): + out = self._render(ad, recent_findings=[ + {"severity": "high", "title": "leaked api_key=abc123def"}]) + assert "abc123def" not in out + + +class TestBuildRequestBody: + def test_shape(self, ad): + body = ad.build_request_body("gpt-4o-mini", "USER") + assert body["model"] == "gpt-4o-mini" + assert body["temperature"] == 0 + assert body["messages"][0]["role"] == "system" + assert body["messages"][1] == {"role": "user", "content": "USER"} + assert body["response_format"]["type"] == "json_schema" + assert body["response_format"]["json_schema"]["schema"] == ad.RESPONSE_SCHEMA + + +class TestExtractContent: + def test_reads_the_first_choice(self, ad): + payload = {"choices": [{"message": {"content": "{}"}}]} + assert ad.extract_content(payload) == "{}" + + @pytest.mark.parametrize("payload", [ + {}, {"choices": []}, {"choices": [{}]}, {"choices": [{"message": {}}]}, + ]) + def test_bad_shapes_raise(self, ad, payload): + with pytest.raises(ValueError): + ad.extract_content(payload) + + +class TestParseFindings: + def _body(self, **kw): + payload = { + "window_assessment": "degraded", + "findings": [{ + "severity": "high", + "title": "asterisk registration storm", + "summary": "40 failures against a baseline of 2", + "evidence": ["<3> [1:nethvoice2:asterisk] registration failed"], + "modules": ["nethvoice2"], + "suggested_action": "check SIP trunk credentials", + }], + } + payload.update(kw) + return json.dumps(payload) + + def test_valid_response(self, ad): + findings, assessment = ad.parse_findings(self._body()) + assert assessment == "degraded" + assert len(findings) == 1 + assert findings[0]["title"] == "asterisk registration storm" + + def test_empty_findings_is_valid(self, ad): + findings, assessment = ad.parse_findings( + self._body(findings=[], window_assessment="nominal")) + assert findings == [] + assert assessment == "nominal" + + @pytest.mark.parametrize("body", [ + "not json", + "[]", + '"a string"', + '{"findings": []}', + '{"window_assessment": "weird", "findings": []}', + '{"window_assessment": "nominal", "findings": {}}', + '{"window_assessment": "nominal", "findings": ["a string"]}', + ]) + def test_invalid_envelopes_raise(self, ad, body): + with pytest.raises(ValueError): + ad.parse_findings(body) + + @pytest.mark.parametrize("bad", [ + {"severity": "catastrophic"}, + {"title": ""}, + {"summary": None}, + ]) + def test_invalid_finding_fields_raise(self, ad, bad): + finding = { + "severity": "high", "title": "t", "summary": "s", + "evidence": [], "modules": [], "suggested_action": "a", + } + finding.update(bad) + with pytest.raises(ValueError): + ad.parse_findings(json.dumps( + {"window_assessment": "nominal", "findings": [finding]})) + + def test_missing_optional_lists_default_to_empty(self, ad): + finding = {"severity": "low", "title": "t", "summary": "s"} + findings, _ = ad.parse_findings(json.dumps( + {"window_assessment": "nominal", "findings": [finding]})) + assert findings[0]["evidence"] == [] + assert findings[0]["modules"] == [] + assert findings[0]["suggested_action"] == "" + + def test_findings_are_scrubbed_again(self, ad): + finding = { + "severity": "low", "title": "t", + "summary": "leaked password=hunter2", + "evidence": ["password=hunter2"], + } + findings, _ = ad.parse_findings(json.dumps( + {"window_assessment": "nominal", "findings": [finding]})) + assert "hunter2" not in findings[0]["summary"] + assert "hunter2" not in findings[0]["evidence"][0] +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +./test-unit.sh -k "FormatLine or EstimateTokens or RenderPrompt or RequestBody or ExtractContent or ParseFindings" +``` + +Expected: FAIL with `AttributeError: module 'anomaly_detector' has no attribute 'format_line'`. + +- [ ] **Step 3: Write the minimal implementation** + +Append to `imageroot/bin/anomaly-detector`: + +```python +#-------------------------------- PROMPT --------------------------------# + +SEVERITIES = ("critical", "high", "medium", "low") +ASSESSMENTS = ("nominal", "degraded", "incident") + +SYSTEM_PROMPT = ( + "You are a log analyst for a NethServer 8 cluster. You judge one hour of " + "journal logs against the supplied per-module baseline.\n" + "Report only actionable deviations from that baseline. Routine, expected " + "and self-healing events are not findings. An empty findings list is the " + "normal and expected answer.\n" + "Never restate a finding listed under RECENT_FINDINGS unless it has " + "clearly escalated; if it has, say so in the summary.\n" + "Quote evidence verbatim from the LINES block. Never invent a log line.\n" + "Keep each title short, specific and stable across hours so that repeats " + "can be deduplicated by title.\n" + "Answer with JSON matching the supplied schema and nothing else." +) + +RESPONSE_SCHEMA = { + "type": "object", + "additionalProperties": False, + "required": ["findings", "window_assessment"], + "properties": { + "findings": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": False, + "required": ["severity", "title", "summary", "evidence", + "modules", "suggested_action"], + "properties": { + "severity": {"type": "string", "enum": list(SEVERITIES)}, + "title": {"type": "string"}, + "summary": {"type": "string"}, + "evidence": {"type": "array", "items": {"type": "string"}}, + "modules": {"type": "array", "items": {"type": "string"}}, + "suggested_action": {"type": "string"}, + }, + }, + }, + "window_assessment": {"type": "string", "enum": list(ASSESSMENTS)}, + }, +} + + +def estimate_tokens(text): + """Rough token count for --dry-run reporting: 4 characters per token.""" + return len(text) // 4 + + +def _block(name, body): + return f"{name}\n```\n{body or '(none)'}\n```\n" + + +def render_prompt(window, digest, recent_findings, lines, truncated): + """Render the user message. Everything here is already scrubbed.""" + start, end = window + window_body = "\n".join([ + f"start: {start.isoformat()}", + f"end: {end.isoformat()}", + f"truncated: {'yes' if truncated else 'no'}", + ]) + + rates_body = "\n".join( + "{0} priority={1} observed={2} expected={3} ratio={4}".format( + row["module_id"], row["priority"], row["observed"], row["expected"], + "n/a" if row["ratio"] is None else row["ratio"], + ) + for row in digest + ) + + findings_body = "\n".join( + scrub("{0}: {1}".format(item.get("severity", "?"), item.get("title", ""))) + for item in recent_findings + ) + + return "".join([ + _block("WINDOW", window_body), + _block("RATES", rates_body), + _block("RECENT_FINDINGS", findings_body), + _block("LINES", "\n".join(lines)), + ]) + + +def build_request_body(model, user_prompt): + """The OpenAI-compatible chat completions request.""" + return { + "model": model, + "temperature": 0, + "messages": [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": user_prompt}, + ], + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "anomaly_report", + "strict": True, + "schema": RESPONSE_SCHEMA, + }, + }, + } + + +def extract_content(payload): + """Pull the assistant message out of a chat completions response.""" + try: + content = payload["choices"][0]["message"]["content"] + except (KeyError, IndexError, TypeError) as exc: + raise ValueError(f"unexpected completion shape: {exc}") from exc + if not isinstance(content, str): + raise ValueError("completion content is not a string") + return content + + +def _require_str(finding, key, allow_empty=False): + value = finding.get(key, "" if allow_empty else None) + if not isinstance(value, str) or (not allow_empty and not value): + raise ValueError(f"finding field {key!r} is invalid: {value!r}") + return scrub(value) + + +def _require_str_list(finding, key): + value = finding.get(key, []) + if not isinstance(value, list) or not all(isinstance(i, str) for i in value): + raise ValueError(f"finding field {key!r} is not a list of strings") + return [scrub(item) for item in value] + + +def parse_findings(body): + """Validate the LLM answer. Raise ValueError rather than emit a partial.""" + try: + payload = json.loads(body) + except json.JSONDecodeError as exc: + raise ValueError(f"response is not JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError("response is not a JSON object") + + assessment = payload.get("window_assessment") + if assessment not in ASSESSMENTS: + raise ValueError(f"invalid window_assessment: {assessment!r}") + + raw_findings = payload.get("findings") + if not isinstance(raw_findings, list): + raise ValueError("findings is not a list") + + findings = [] + for item in raw_findings: + if not isinstance(item, dict): + raise ValueError("finding is not an object") + severity = item.get("severity") + if severity not in SEVERITIES: + raise ValueError(f"invalid severity: {severity!r}") + findings.append({ + "severity": severity, + "title": _require_str(item, "title"), + "summary": _require_str(item, "summary"), + "evidence": _require_str_list(item, "evidence"), + "modules": _require_str_list(item, "modules"), + "suggested_action": _require_str(item, "suggested_action", allow_empty=True), + }) + return findings, assessment +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +./test-unit.sh +``` + +Expected: every test PASSES. + +- [ ] **Step 5: Commit** + +```bash +git add imageroot/bin/anomaly-detector tests/unit/test_anomaly_detector.py +git commit -m "feat(anomaly-detector): render the prompt and validate the response" +``` + +--- + +### Task 4: Loki collection + +**Files:** +- Modify: `imageroot/bin/anomaly-detector` (append) +- Modify: `tests/unit/test_anomaly_detector.py` (append) + +**Interfaces:** +- Consumes: `parse_metric_response` (Task 2), `scrub` (Task 1). +- Produces: + - `LINE_FORMAT` (str constant) — the LogQL `line_format` stage that renders + ` [node_id:module_id:identifier] message` server-side. This is the + single definition of the display format; nothing renders lines in Python. + - `logql_duration(delta) -> str` — a timedelta as LogQL seconds, e.g. `3600s`. + - `build_metric_query(range_text) -> str` + - `build_lines_query(module_id) -> str` + - `build_recall_query(module_id) -> str` + - `run_logcli(argv, timeout=LOGCLI_TIMEOUT) -> str` — raises `RuntimeError` on non-zero exit or timeout. + - `parse_jsonl_records(stdout) -> list[dict]` — one dict per `logcli -o jsonl` line, with keys `priority`, `node_id`, `module_id`, `identifier`, `message`. + - `query_metric(session, addr, auth, query, at) -> dict` + - `query_lines(window, max_lines, module_id) -> (records, truncated)` + - `recall_findings(window_end, module_id, limit=10) -> list[dict]` of `{"severity", "title"}` + - `loki_env() -> (addr, (username, password))` — also sets `LOKI_ADDR`/`LOKI_USERNAME`/`LOKI_PASSWORD` in `os.environ` for `logcli`. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/unit/test_anomaly_detector.py`: + +```python +import subprocess + + +class TestLogqlDuration: + def test_hour(self, ad): + assert ad.logql_duration(timedelta(hours=1)) == "3600s" + + def test_seven_days(self, ad): + assert ad.logql_duration(timedelta(days=7)) == "604800s" + + def test_sub_minute_rounds_up_to_one_second(self, ad): + assert ad.logql_duration(timedelta(milliseconds=1)) == "1s" + + +class TestQueryBuilders: + def test_line_format_is_the_single_display_format_definition(self, ad): + assert ad.LINE_FORMAT == ( + '| line_format "<{{.priority}}> ' + '[{{.node_id}}:{{.module_id}}:{{.identifier}}] {{.message}}"' + ) + + def test_metric_query(self, ad): + query = ad.build_metric_query("3600s") + assert query == ( + 'sum by (module_id, priority) ' + '(count_over_time({node_id=~".+"} | json priority="PRIORITY" [3600s]))' + ) + + def test_lines_query_excludes_own_identifier(self, ad): + query = ad.build_lines_query("loki1") + assert '| identifier != "loki1/anomaly-detector"' in query + assert 'priority < 5 or category="security"' in query + assert 'line_format' in query + # the exclusion must precede the priority filter, or the detector + # would feed on its own PRIORITY=3 diagnostics + assert query.index('identifier != ') < query.index('priority < 5') + + def test_recall_query_selects_only_own_output(self, ad): + query = ad.build_recall_query("loki1") + assert '{module_id="loki1"}' in query + assert '| identifier="loki1/anomaly-detector"' in query + + +class TestRunLogcli: + def test_returns_stdout(self, ad, monkeypatch): + def fake_run(argv, **kwargs): + assert argv[0] == "logcli" + return subprocess.CompletedProcess(argv, 0, stdout="ok\n", stderr="") + monkeypatch.setattr(ad.subprocess, "run", fake_run) + assert ad.run_logcli(["logcli", "query", "x"]) == "ok\n" + + def test_non_zero_exit_raises(self, ad, monkeypatch): + def fake_run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 1, stdout="", stderr="boom") + monkeypatch.setattr(ad.subprocess, "run", fake_run) + with pytest.raises(RuntimeError) as exc: + ad.run_logcli(["logcli", "query", "x"]) + assert "boom" in str(exc.value) + + def test_timeout_raises(self, ad, monkeypatch): + def fake_run(argv, **kwargs): + raise subprocess.TimeoutExpired(argv, 300) + monkeypatch.setattr(ad.subprocess, "run", fake_run) + with pytest.raises(RuntimeError): + ad.run_logcli(["logcli", "query", "x"]) + + +class TestParseJsonlRecords: + def test_parses_line_format_output(self, ad): + stdout = ( + '{"labels":{"node_id":"1","module_id":"nethvoice2"},' + '"line":"<3> [1:nethvoice2:asterisk] boom","timestamp":"t"}\n' + '{"labels":{},"line":"<4> [?:?:?] other","timestamp":"t"}\n' + ) + records = ad.parse_jsonl_records(stdout) + assert len(records) == 2 + assert records[0]["line"] == "<3> [1:nethvoice2:asterisk] boom" + + def test_skips_unparsable_and_blank_lines(self, ad): + stdout = '\nnot json\n{"line":"ok"}\n' + assert ad.parse_jsonl_records(stdout) == [{"line": "ok"}] + + def test_empty_stdout(self, ad): + assert ad.parse_jsonl_records("") == [] + + +class TestQueryMetric: + class _Response: + status_code = 200 + + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + class _Session: + def __init__(self, payload): + self._payload = payload + self.calls = [] + + def get(self, url, **kwargs): + self.calls.append((url, kwargs)) + return TestQueryMetric._Response(self._payload) + + def test_calls_the_instant_query_endpoint(self, ad): + payload = {"data": {"result": [ + {"metric": {"module_id": "a", "priority": "6"}, "value": [0, "10"]}]}} + session = self._Session(payload) + at = datetime(2026, 7, 29, 14, 0, tzinfo=timezone.utc) + rates = ad.query_metric(session, "http://127.0.0.1:3100", ("u", "p"), "Q", at) + url, kwargs = session.calls[0] + assert url == "http://127.0.0.1:3100/loki/api/v1/query" + assert kwargs["params"]["query"] == "Q" + assert kwargs["params"]["time"] == at.isoformat() + assert kwargs["auth"] == ("u", "p") + assert rates == {("a", "6"): 10.0} + + +class TestQueryLines: + def test_truncation_is_detected_at_the_cap(self, ad, monkeypatch): + stdout = "".join( + '{"line":"<3> [1:a:b] boom %d"}\n' % i for i in range(3)) + monkeypatch.setattr(ad, "run_logcli", lambda argv, **kw: stdout) + records, truncated = ad.query_lines(WINDOW, 3, "loki1") + assert len(records) == 3 + assert truncated is True + + def test_below_the_cap_is_not_truncated(self, ad, monkeypatch): + monkeypatch.setattr(ad, "run_logcli", lambda argv, **kw: '{"line":"x"}\n') + _, truncated = ad.query_lines(WINDOW, 500, "loki1") + assert truncated is False + + def test_argv_carries_window_and_limit(self, ad, monkeypatch): + seen = {} + def fake(argv, **kw): + seen["argv"] = argv + return "" + monkeypatch.setattr(ad, "run_logcli", fake) + ad.query_lines(WINDOW, 42, "loki1") + argv = seen["argv"] + assert argv[:2] == ["logcli", "query"] + assert "--limit" in argv and "42" in argv + assert "--forward" in argv + assert WINDOW[0].isoformat() in argv + assert WINDOW[1].isoformat() in argv + + +class TestRecallFindings: + def test_extracts_severity_and_title(self, ad, monkeypatch): + stdout = "\n".join([ + json.dumps({"line": json.dumps( + {"severity": "high", "title": "asterisk storm"})}), + json.dumps({"line": json.dumps( + {"window_assessment": "nominal", "findings_count": 0})}), + json.dumps({"line": "not json at all"}), + ]) + "\n" + monkeypatch.setattr(ad, "run_logcli", lambda argv, **kw: stdout) + found = ad.recall_findings(WINDOW[1], "loki1") + assert found == [{"severity": "high", "title": "asterisk storm"}] + + def test_keeps_only_the_last_n(self, ad, monkeypatch): + stdout = "".join( + json.dumps({"line": json.dumps( + {"severity": "low", "title": f"t{i}"})}) + "\n" + for i in range(15)) + monkeypatch.setattr(ad, "run_logcli", lambda argv, **kw: stdout) + found = ad.recall_findings(WINDOW[1], "loki1", limit=10) + assert len(found) == 10 + assert found[-1]["title"] == "t14" + + def test_a_failing_recall_is_not_fatal(self, ad, monkeypatch): + def boom(argv, **kw): + raise RuntimeError("loki down") + monkeypatch.setattr(ad, "run_logcli", boom) + assert ad.recall_findings(WINDOW[1], "loki1") == [] + + +class TestLokiEnv: + def test_builds_addr_and_exports_for_logcli(self, ad, monkeypatch): + monkeypatch.setattr(ad.os, "environ", { + "LOKI_HTTP_PORT": "3100", + "LOKI_API_AUTH_USERNAME": "loki", + "LOKI_API_AUTH_PASSWORD": "sekrit", + }) + addr, auth = ad.loki_env() + assert addr == "http://127.0.0.1:3100" + assert auth == ("loki", "sekrit") + assert ad.os.environ["LOKI_ADDR"] == "http://127.0.0.1:3100" + assert ad.os.environ["LOKI_USERNAME"] == "loki" + assert ad.os.environ["LOKI_PASSWORD"] == "sekrit" + + def test_missing_variable_raises(self, ad, monkeypatch): + monkeypatch.setattr(ad.os, "environ", {}) + with pytest.raises(RuntimeError): + ad.loki_env() +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +./test-unit.sh -k "Logql or QueryBuilders or RunLogcli or Jsonl or QueryMetric or QueryLines or Recall or LokiEnv" +``` + +Expected: FAIL with `AttributeError: module 'anomaly_detector' has no attribute 'logql_duration'`. + +- [ ] **Step 3: Write the minimal implementation** + +Append to `imageroot/bin/anomaly-detector`: + +```python +#------------------------------- COLLECT --------------------------------# + +LOGCLI_TIMEOUT = 300 +HTTP_TIMEOUT = (10, 300) + +# The whole journal record is stored as the log line, so every query needs +# a `| json` stage, exactly as cloud-log-manager-forwarder does today. +LINE_FORMAT = '| line_format "<{{.priority}}> [{{.node_id}}:{{.module_id}}:{{.identifier}}] {{.message}}"' + + +def logql_duration(delta): + """A timedelta as a LogQL range, in seconds.""" + return "{0}s".format(max(1, int(delta.total_seconds()))) + + +def build_metric_query(range_text): + return ( + 'sum by (module_id, priority) ' + '(count_over_time({{node_id=~".+"}} | json priority="PRIORITY" [{0}]))' + ).format(range_text) + + +def build_lines_query(module_id): + """Prefiltered lines for the window. + + The detector's own identifier is excluded BEFORE the priority filter. + Its diagnostics land in the journal at PRIORITY=3, so without this a + single failure would be re-analysed every hour and past evidence lines + would re-enter the prompt as fresh input. + """ + return " ".join([ + '{node_id=~".+"}', + '| json priority="PRIORITY", identifier="SYSLOG_IDENTIFIER", message="MESSAGE"', + '| identifier != "{0}/anomaly-detector"'.format(module_id), + '| priority < 5 or category="security"', + LINE_FORMAT, + ]) + + +def build_recall_query(module_id): + """The detector's own past findings — its self-hosted memory.""" + return " ".join([ + '{{module_id="{0}"}}'.format(module_id), + '| json identifier="SYSLOG_IDENTIFIER", message="MESSAGE"', + '| identifier="{0}/anomaly-detector"'.format(module_id), + '| line_format "{{.message}}"', + ]) + + +def loki_env(): + """Derive and export the Loki endpoint, as the forwarders do. + + The module environment sets LOKI_ADDR to the VPN IP address, which is + not a URL, so it is overwritten here with the local traefik endpoint. + """ + try: + addr = "http://127.0.0.1:{0}".format(os.environ['LOKI_HTTP_PORT']) + username = os.environ['LOKI_API_AUTH_USERNAME'] + password = os.environ['LOKI_API_AUTH_PASSWORD'] + except KeyError as exc: + raise RuntimeError(f"missing Loki variable {exc}; run under runagent") from exc + os.environ['LOKI_ADDR'] = addr + os.environ['LOKI_USERNAME'] = username + os.environ['LOKI_PASSWORD'] = password + return addr, (username, password) + + +def run_logcli(argv, timeout=LOGCLI_TIMEOUT): + """Run logcli and return its stdout. Fail loudly: one window is cheap.""" + try: + response = subprocess.run(argv, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired as exc: + raise RuntimeError(f"logcli timed out after {timeout}s") from exc + if response.returncode != 0: + raise RuntimeError( + "logcli exited {0}: {1}".format(response.returncode, response.stderr.strip())) + return response.stdout + + +def parse_jsonl_records(stdout): + """Parse `logcli -o jsonl` output, skipping anything unparsable.""" + records = [] + for raw in stdout.splitlines(): + raw = raw.strip() + if not raw: + continue + try: + records.append(json.loads(raw)) + except json.JSONDecodeError as exc: + print(f"Skipping unparsable logcli record: {exc}", file=sys.stderr) + return records + + +def query_metric(session, addr, auth, query, at): + """Run a Loki instant query and return {(module_id, priority): value}.""" + response = session.get( + addr + "/loki/api/v1/query", + params={"query": query, "time": at.isoformat()}, + auth=auth, + timeout=HTTP_TIMEOUT, + ) + response.raise_for_status() + return parse_metric_response(response.json()) + + +def query_lines(window, max_lines, module_id): + """Collect the prefiltered lines for the window. + + Returns (records, truncated). Truncation is never silent: the caller + puts it in the prompt and on stderr. + """ + start, end = window + argv = [ + "logcli", "query", + "--limit", str(max_lines), + "--forward", + "--timezone", "UTC", + "--from", start.isoformat(), + "--to", end.isoformat(), + "--no-labels", "-q", "-o", "jsonl", + build_lines_query(module_id), + ] + records = parse_jsonl_records(run_logcli(argv)) + return records, len(records) >= max_lines + + +def recall_findings(window_end, module_id, limit=10): + """The last `limit` finding titles from the detector's own journal output. + + A failure here is logged and treated as "no memory": losing dedup + context is not worth losing the whole window. + """ + argv = [ + "logcli", "query", + "--limit", "200", + "--forward", + "--timezone", "UTC", + "--from", (window_end - timedelta(hours=24)).isoformat(), + "--to", window_end.isoformat(), + "--no-labels", "-q", "-o", "jsonl", + build_recall_query(module_id), + ] + try: + stdout = run_logcli(argv) + except RuntimeError as exc: + print(f"Recall query failed, continuing without memory: {exc}", file=sys.stderr) + return [] + + recalled = [] + for record in parse_jsonl_records(stdout): + try: + finding = json.loads(record.get("line", "")) + except json.JSONDecodeError: + continue + if isinstance(finding, dict) and finding.get("title"): + recalled.append({ + "severity": finding.get("severity", "?"), + "title": finding["title"], + }) + return recalled[-limit:] +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +./test-unit.sh +``` + +Expected: every test PASSES. + +- [ ] **Step 5: Verify the real queries on the test node** + +The query builders must produce LogQL that a real Loki accepts. Generate the +queries from the code itself, so a typo in the constant cannot pass: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +python3 - > /tmp/probe.sh <<'PY' +import importlib.machinery, importlib.util +l = importlib.machinery.SourceFileLoader('ad', 'imageroot/bin/anomaly-detector') +s = importlib.util.spec_from_loader('ad', l) +m = importlib.util.module_from_spec(s); l.exec_module(m) +print('#!/bin/bash') +print('export LOKI_ADDR="http://127.0.0.1:$LOKI_HTTP_PORT"') +print('export LOKI_USERNAME="$LOKI_API_AUTH_USERNAME"') +print('export LOKI_PASSWORD="$LOKI_API_AUTH_PASSWORD"') +print('FROM=$(date -u -d "-1 hour" +%Y-%m-%dT%H:%M:%SZ)') +print('TO=$(date -u +%Y-%m-%dT%H:%M:%SZ)') +print('set -e') +print('echo "== metric query"') +print('curl -sSf -u "$LOKI_USERNAME:$LOKI_PASSWORD" -G "$LOKI_ADDR/loki/api/v1/query" \\') +print(' --data-urlencode {0} \\'.format(repr("query=" + m.build_metric_query("3600s")))) +print(' --data-urlencode "time=$TO" | head -c 200; echo') +print('echo "== lines query"') +print('logcli query --limit 500 --forward --timezone UTC --from "$FROM" --to "$TO" \\') +print(' --no-labels -q -o jsonl {0} | wc -l'.format(repr(m.build_lines_query("loki1")))) +print('echo "== recall query"') +print('logcli query --limit 200 --forward --timezone UTC --from "$FROM" --to "$TO" \\') +print(' --no-labels -q -o jsonl {0} | wc -l'.format(repr(m.build_recall_query("loki1")))) +PY +scp -q /tmp/probe.sh ${NODE}:/tmp/probe.sh +ssh ${NODE} runagent -m loki1 bash /tmp/probe.sh +``` + +Expected: the metric query prints `{"status":"success",...`; the lines query +prints a non-zero count (a few hundred on this node); the recall query prints +`0` because the detector has never run. **All three must exit 0** — `set -e` +plus `curl -sSf` makes a LogQL parse error a failure rather than a silent empty +result. If the recall query errors rather than returning 0, the query is +malformed; a genuinely empty result is not an error. + +- [ ] **Step 6: Commit** + +```bash +git add imageroot/bin/anomaly-detector tests/unit/test_anomaly_detector.py +git commit -m "feat(anomaly-detector): collect rates, lines and recalled findings from Loki" +``` + +--- + +### Task 5: Config precedence, LLM call, emit, webhook and `main()` + +This completes the script. After this task `--dry-run` works end to end on a real node. + +**Files:** +- Modify: `imageroot/bin/anomaly-detector` (append) +- Modify: `tests/unit/test_anomaly_detector.py` (append) + +**Interfaces:** +- Consumes: everything from Tasks 1–4. +- Produces: + - `CONFIG_KEYS` (tuple), `DEFAULT_MAX_LINES = 500` + - `_read_state_envfile(path) -> dict` — lazily imports `agent`; returns `{}` when the file is missing. + - `load_config(args, environ, reader=_read_state_envfile) -> dict` — precedence lowest→highest: `environment`, `secrets.env`, shell env, `--config` file, CLI flags. + - `make_session() -> requests.Session` + - `ask_llm(session, config, prompt) -> str` + - `render_findings(findings, assessment, window, truncated, llm_called, pretty) -> list[str]` + - `post_webhook(session, config, payload) -> None` + - `build_parser() -> argparse.ArgumentParser` + - `main(argv=None) -> int` + +- [ ] **Step 1: Write the failing test** + +Append to `tests/unit/test_anomaly_detector.py`: + +```python +import types + + +def _args(**kw): + defaults = { + "dry_run": False, "since": None, "config": None, "pretty": False, + "no_webhook": False, "max_lines": None, "print_prompt": False, + } + defaults.update(kw) + return types.SimpleNamespace(**defaults) + + +class TestLoadConfig: + def test_precedence_order(self, ad): + files = { + "environment": {"ANOMALY_LLM_MODEL": "from-environment", + "ANOMALY_LLM_BASE_URL": "from-environment"}, + "secrets.env": {"ANOMALY_LLM_MODEL": "from-secrets", + "ANOMALY_LLM_API_KEY": "from-secrets"}, + "/tmp/override.env": {"ANOMALY_LLM_MODEL": "from-config-file"}, + } + environ = {"ANOMALY_LLM_MODEL": "from-shell", "PATH": "/bin"} + config = ad.load_config( + _args(config="/tmp/override.env"), environ, reader=files.get) + # --config beats shell env beats secrets.env beats environment + assert config["ANOMALY_LLM_MODEL"] == "from-config-file" + assert config["ANOMALY_LLM_BASE_URL"] == "from-environment" + assert config["ANOMALY_LLM_API_KEY"] == "from-secrets" + + def test_shell_env_beats_state_files(self, ad): + files = {"environment": {"ANOMALY_LLM_MODEL": "old"}} + config = ad.load_config( + _args(), {"ANOMALY_LLM_MODEL": "new"}, reader=files.get) + assert config["ANOMALY_LLM_MODEL"] == "new" + + def test_cli_max_lines_wins(self, ad): + files = {"environment": {"ANOMALY_MAX_LINES": "10"}} + config = ad.load_config(_args(max_lines=7), {}, reader=files.get) + assert config["ANOMALY_MAX_LINES"] == "7" + + def test_max_lines_default(self, ad): + config = ad.load_config(_args(), {}, reader=lambda p: {}) + assert config["ANOMALY_MAX_LINES"] == str(ad.DEFAULT_MAX_LINES) + + def test_unrelated_variables_are_ignored(self, ad): + config = ad.load_config( + _args(), {"LOKI_API_AUTH_PASSWORD": "sekrit"}, reader=lambda p: {}) + assert "LOKI_API_AUTH_PASSWORD" not in config + + def test_missing_keys_default_to_empty_string(self, ad): + config = ad.load_config(_args(), {}, reader=lambda p: {}) + assert config["ANOMALY_LLM_API_KEY"] == "" + assert config["ANOMALY_WEBHOOK_URL"] == "" + + +class TestAskLlm: + class _Response: + def __init__(self, status_code, payload=None, text=""): + self.status_code = status_code + self._payload = payload or {} + self.text = text + + def json(self): + return self._payload + + class _Session: + def __init__(self, response): + self._response = response + self.calls = [] + + def post(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._response + + def _config(self, **kw): + config = { + "ANOMALY_LLM_BASE_URL": "https://api.example.org/v1", + "ANOMALY_LLM_MODEL": "gpt-4o-mini", + "ANOMALY_LLM_API_KEY": "sk-test", + } + config.update(kw) + return config + + def test_posts_to_chat_completions(self, ad): + response = self._Response( + 200, {"choices": [{"message": {"content": "{}"}}]}) + session = self._Session(response) + assert ad.ask_llm(session, self._config(), "PROMPT") == "{}" + url, kwargs = session.calls[0] + assert url == "https://api.example.org/v1/chat/completions" + assert kwargs["headers"]["Authorization"] == "Bearer sk-test" + assert kwargs["json"]["messages"][1]["content"] == "PROMPT" + + def test_trailing_slash_in_base_url(self, ad): + session = self._Session( + self._Response(200, {"choices": [{"message": {"content": "{}"}}]})) + ad.ask_llm(session, self._config( + ANOMALY_LLM_BASE_URL="https://api.example.org/v1/"), "P") + assert session.calls[0][0] == "https://api.example.org/v1/chat/completions" + + @pytest.mark.parametrize("status", [401, 403]) + def test_auth_errors_mention_the_api_key(self, ad, status): + session = self._Session(self._Response(status, text="nope")) + with pytest.raises(RuntimeError) as exc: + ad.ask_llm(session, self._config(), "P") + assert "check API key" in str(exc.value) + + def test_other_errors_truncate_the_body(self, ad): + session = self._Session(self._Response(500, text="x" * 900)) + with pytest.raises(RuntimeError) as exc: + ad.ask_llm(session, self._config(), "P") + assert len(str(exc.value)) < 700 + + +class TestRenderFindings: + def _lines(self, ad, **kw): + params = { + "findings": [{ + "severity": "high", "title": "asterisk storm", + "summary": "s", "evidence": ["e"], + "modules": ["nethvoice2"], "suggested_action": "a", + }], + "assessment": "degraded", + "window": WINDOW, + "truncated": False, + "llm_called": True, + "pretty": False, + } + params.update(kw) + return ad.render_findings(**params) + + def test_one_json_line_per_finding_plus_a_summary(self, ad): + lines = self._lines(ad) + assert len(lines) == 2 + finding = json.loads(lines[0]) + assert finding["title"] == "asterisk storm" + assert finding["window_start"] == WINDOW[0].isoformat() + summary = json.loads(lines[1]) + assert summary["window_assessment"] == "degraded" + assert summary["findings_count"] == 1 + assert summary["llm_called"] is True + assert summary["truncated"] is False + + def test_summary_line_is_emitted_even_with_no_findings(self, ad): + lines = self._lines(ad, findings=[], assessment="nominal", llm_called=False) + assert len(lines) == 1 + summary = json.loads(lines[0]) + assert summary["findings_count"] == 0 + assert summary["llm_called"] is False + + def test_pretty_mode_is_not_json(self, ad): + lines = self._lines(ad, pretty=True) + blob = "\n".join(lines) + assert "HIGH" in blob + assert "asterisk storm" in blob + with pytest.raises(json.JSONDecodeError): + json.loads(lines[0]) + + +class TestPostWebhook: + class _Session: + def __init__(self): + self.calls = [] + + class _Response: + status_code = 200 + + def raise_for_status(self): + pass + + def post(self, url, **kwargs): + self.calls.append((url, kwargs)) + return self._Response() + + def test_no_url_is_a_no_op(self, ad): + session = self._Session() + ad.post_webhook(session, {"ANOMALY_WEBHOOK_URL": ""}, {"a": 1}) + assert session.calls == [] + + def test_posts_with_bearer_token(self, ad): + session = self._Session() + ad.post_webhook(session, { + "ANOMALY_WEBHOOK_URL": "https://hook.example.org/h", + "ANOMALY_WEBHOOK_TOKEN": "t0ken", + }, {"a": 1}) + url, kwargs = session.calls[0] + assert url == "https://hook.example.org/h" + assert kwargs["headers"]["Authorization"] == "Bearer t0ken" + assert kwargs["json"] == {"a": 1} + + def test_posts_without_token(self, ad): + session = self._Session() + ad.post_webhook(session, { + "ANOMALY_WEBHOOK_URL": "https://hook.example.org/h"}, {"a": 1}) + assert "Authorization" not in session.calls[0][1]["headers"] + + +class TestParser: + def test_every_flag_is_optional(self, ad): + args = ad.build_parser().parse_args([]) + assert args.dry_run is False + assert args.since is None + assert args.max_lines is None + + def test_flags_parse(self, ad): + args = ad.build_parser().parse_args([ + "--dry-run", "--since", "2h", "--pretty", "--no-webhook", + "--max-lines", "50", "--print-prompt", "--config", "/tmp/x.env", + ]) + assert args.dry_run is True + assert args.since == "2h" + assert args.pretty is True + assert args.no_webhook is True + assert args.max_lines == 50 + assert args.print_prompt is True + assert args.config == "/tmp/x.env" +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +```bash +./test-unit.sh -k "LoadConfig or AskLlm or RenderFindings or PostWebhook or Parser" +``` + +Expected: FAIL with `AttributeError: module 'anomaly_detector' has no attribute 'load_config'`. + +- [ ] **Step 3: Write the minimal implementation** + +First add the two HTTP imports to the import block near the top of `imageroot/bin/anomaly-detector`, matching `cloud-log-manager-forwarder`: + +```python +from requests import Session +from requests.adapters import HTTPAdapter +from urllib3.util import Retry +``` + +Then append: + +```python +#-------------------------------- CONFIG --------------------------------# + +DEFAULT_MAX_LINES = 500 + +CONFIG_KEYS = ( + "ANOMALY_LLM_BASE_URL", + "ANOMALY_LLM_MODEL", + "ANOMALY_LLM_API_KEY", + "ANOMALY_MAX_LINES", + "ANOMALY_WEBHOOK_URL", + "ANOMALY_WEBHOOK_TOKEN", +) + +# Lowest precedence first. Shell env, --config and CLI flags are layered +# on top by load_config(). +STATE_ENVFILES = ("environment", "secrets.env") + + +def _read_state_envfile(path): + """Read an env file relative to AGENT_STATE_DIR, or {} if absent. + + `agent` is imported lazily so the module stays importable in the unit + test container, where the NS8 SDK is not installed. + """ + import agent + try: + return agent.read_envfile(path) + except FileNotFoundError: + return {} + + +def load_config(args, environ=None, reader=None): + """Resolve configuration. + + Precedence, highest first: CLI flag, --config file, shell environment, + secrets.env, environment. + """ + environ = os.environ if environ is None else environ + reader = _read_state_envfile if reader is None else reader + + merged = {} + for path in STATE_ENVFILES: + merged.update(reader(path) or {}) + merged.update({key: value for key, value in environ.items() if key in CONFIG_KEYS}) + if args.config: + merged.update(reader(args.config) or {}) + if args.max_lines is not None: + merged["ANOMALY_MAX_LINES"] = str(args.max_lines) + + config = {key: str(merged.get(key, "") or "") for key in CONFIG_KEYS} + if not config["ANOMALY_MAX_LINES"]: + config["ANOMALY_MAX_LINES"] = str(DEFAULT_MAX_LINES) + return config + + +#--------------------------------- ASK ----------------------------------# + +def make_session(): + """A Session that retries transient LLM and webhook failures.""" + session = Session() + retries = Retry( + total=3, + backoff_factor=2, + status_forcelist=[429, 500, 502, 503, 504], + allowed_methods=frozenset(["GET", "POST"]), + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + +def ask_llm(session, config, prompt): + """POST the prompt and return the raw assistant message.""" + url = config["ANOMALY_LLM_BASE_URL"].rstrip("/") + "/chat/completions" + response = session.post( + url, + headers={ + "Authorization": "Bearer " + config["ANOMALY_LLM_API_KEY"], + "Content-Type": "application/json", + }, + json=build_request_body(config["ANOMALY_LLM_MODEL"], prompt), + timeout=HTTP_TIMEOUT, + ) + if response.status_code in (401, 403): + raise RuntimeError(f"LLM returned {response.status_code}: check API key") + if response.status_code >= 400: + raise RuntimeError("LLM returned {0}: {1}".format( + response.status_code, response.text[:500])) + return extract_content(response.json()) + + +#--------------------------------- EMIT ---------------------------------# + +def render_findings(findings, assessment, window, truncated, llm_called, pretty): + """Render the output lines. One line per finding, then a summary line.""" + start, end = window + lines = [] + + for finding in findings: + record = dict(finding) + record["window_start"] = start.isoformat() + record["window_end"] = end.isoformat() + if pretty: + lines.append("[{0}] {1}".format(finding["severity"].upper(), finding["title"])) + lines.append(" modules: {0}".format(", ".join(finding["modules"]) or "-")) + lines.append(" summary: {0}".format(finding["summary"])) + lines.append(" action: {0}".format(finding["suggested_action"] or "-")) + for item in finding["evidence"]: + lines.append(" | {0}".format(item)) + lines.append("") + else: + lines.append(json.dumps(record, sort_keys=True)) + + summary = { + "window_assessment": assessment, + "findings_count": len(findings), + "llm_called": llm_called, + "truncated": truncated, + "window_start": start.isoformat(), + "window_end": end.isoformat(), + } + if pretty: + lines.append("window {0} .. {1}: {2}, {3} finding(s){4}".format( + start.isoformat(), end.isoformat(), assessment, len(findings), + ", window truncated" if truncated else "")) + else: + lines.append(json.dumps(summary, sort_keys=True)) + return lines + + +def post_webhook(session, config, payload): + """Best-effort delivery. Journald is the source of truth.""" + url = config.get("ANOMALY_WEBHOOK_URL", "") + if not url: + return + headers = {"Content-Type": "application/json"} + token = config.get("ANOMALY_WEBHOOK_TOKEN", "") + if token: + headers["Authorization"] = "Bearer " + token + response = session.post(url, headers=headers, json=payload, timeout=HTTP_TIMEOUT) + response.raise_for_status() + + +#--------------------------------- MAIN ---------------------------------# + +def build_parser(): + parser = argparse.ArgumentParser( + description="Detect anomalies in one window of NS8 journal logs.") + parser.add_argument("--dry-run", action="store_true", + help="collect and render the prompt, print it, make no LLM call") + parser.add_argument("--since", metavar="DURATION", + help="analyse [now-DURATION, now] instead of the previous full hour") + parser.add_argument("--config", metavar="FILE", + help="read ANOMALY_* from FILE instead of the module state") + parser.add_argument("--pretty", action="store_true", + help="print findings as indented text instead of JSON lines") + parser.add_argument("--no-webhook", action="store_true", + help="skip webhook delivery") + parser.add_argument("--max-lines", type=int, metavar="N", + help="override the prefiltered line cap for this run") + parser.add_argument("--print-prompt", action="store_true", + help="print the prompt to stderr alongside a real LLM call") + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + config = load_config(args) + module_id = os.environ.get('MODULE_ID', 'loki1') + + if not args.dry_run: + for key in ("ANOMALY_LLM_BASE_URL", "ANOMALY_LLM_MODEL", "ANOMALY_LLM_API_KEY"): + if not config[key]: + print(f"not configured: {key} is unset", file=sys.stderr) + return 1 + + addr, auth = loki_env() + window = compute_window(datetime.now(timezone.utc), args.since) + start, end = window + duration = end - start + max_lines = int(config["ANOMALY_MAX_LINES"]) + + session = make_session() + + rates = query_metric(session, addr, auth, + build_metric_query(logql_duration(duration)), end) + baseline_raw = query_metric(session, addr, auth, + build_metric_query(logql_duration(timedelta(days=7))), + start) + window_hours = duration.total_seconds() / 3600.0 + baseline = { + key: (value / BASELINE_HOURS) * window_hours + for key, value in baseline_raw.items() + } + digest = build_digest(rates, baseline) + + records, truncated = query_lines(window, max_lines, module_id) + if truncated: + print("Line cap of {0} reached: the window is truncated".format(max_lines), + file=sys.stderr) + # LINE_FORMAT already rendered each line server-side, so only flattening + # and scrubbing remain. Nothing formats log lines in Python. + lines = [sanitize_line(record.get("line", "")) for record in records] + lines = [line for line in lines if line] + + recalled = recall_findings(end, module_id) + + prompt = render_prompt(window, digest, recalled, lines, truncated) + + if args.dry_run: + print(prompt, file=sys.stderr) + print("--- {0} characters, ~{1} tokens, {2} lines, no LLM call".format( + len(prompt), estimate_tokens(prompt), len(lines)), file=sys.stderr) + return 0 + + if is_nominal(records, digest): + print("nominal, no LLM call", file=sys.stderr) + for line in render_findings([], "nominal", window, truncated, False, args.pretty): + print(line) + return 0 + + if args.print_prompt: + print(prompt, file=sys.stderr) + + findings, assessment = parse_findings(ask_llm(session, config, prompt)) + + for line in render_findings(findings, assessment, window, truncated, True, args.pretty): + print(line) + sys.stdout.flush() + + if not args.no_webhook: + post_webhook(session, config, { + "window_start": start.isoformat(), + "window_end": end.isoformat(), + "window_assessment": assessment, + "truncated": truncated, + "findings": findings, + }) + + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except Exception as exc: + print(f"anomaly-detector failed: {exc}", file=sys.stderr) + sys.exit(1) +``` + +- [ ] **Step 4: Run the tests to verify they pass** + +```bash +./test-unit.sh +``` + +Expected: every test PASSES. + +- [ ] **Step 5: Verify the CLI is wired up** + +```bash +python3 -c " +import importlib.machinery, importlib.util, sys +l = importlib.machinery.SourceFileLoader('ad', 'imageroot/bin/anomaly-detector') +s = importlib.util.spec_from_loader('ad', l); m = importlib.util.module_from_spec(s); l.exec_module(m) +m.build_parser().parse_args(['--help']) +" || true +``` + +Expected: the help text lists all seven flags and exits 0. + +- [ ] **Step 6: Verify `--dry-run` end to end on the test node** + +This is the first full run of stages 1–4 against real logs, with no LLM call and +no cost: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +scp -q imageroot/bin/anomaly-detector ${NODE}:/tmp/ +ssh ${NODE} runagent -m loki1 python3 /tmp/anomaly-detector --dry-run --since 1h +``` + +Expected, all on stderr, exit code 0: +- all four fenced blocks present: `WINDOW`, `RATES`, `RECENT_FINDINGS`, `LINES` +- `RECENT_FINDINGS` shows `(none)` — the detector has never run +- `RATES` lists real modules (`crowdsec1`, `nethvoice2`, `traefik1`, …) with + `observed=` and `expected=` values, and `ratio=n/a` for pairs with no 7-day + history +- the trailing line reports characters, approximate tokens and line count; + on this node expect roughly 40k characters / ~10k tokens / ~380 lines +- **no line in the `LINES` block contains a raw newline** — verify with + `... --dry-run --since 1h 2>&1 | sed -n '/^LINES/,$p' | wc -l` and check the + count matches the reported line count plus the two fence lines + +Then confirm the missing-configuration path: + +```bash +ssh ${NODE} runagent -m loki1 python3 /tmp/anomaly-detector --since 1h; echo "exit=$?" +``` + +Expected: `not configured: ANOMALY_LLM_BASE_URL is unset` on stderr, `exit=1`, +and no LLM call attempted. + +- [ ] **Step 7: Verify a real LLM call end to end** + +This exercises stages 5 and 6 for the first time, against the OpenRouter endpoint +described in "The LLM endpoint used for verification". Pass the key through the +environment so it never lands in a file or in shell history on the node: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +ORKEY=$(grep -oE 'sk-or-[A-Za-z0-9._-]+' open_router | head -1) +scp -q imageroot/bin/anomaly-detector ${NODE}:/tmp/ +ssh ${NODE} "ANOMALY_LLM_API_KEY='${ORKEY}' runagent -m loki1 env \ + ANOMALY_LLM_BASE_URL=https://openrouter.ai/api/v1 \ + ANOMALY_LLM_MODEL=google/gemma-4-26b-a4b-it:free \ + ANOMALY_LLM_API_KEY=\"\$ANOMALY_LLM_API_KEY\" \ + python3 /tmp/anomaly-detector --since 1h --pretty --no-webhook" +``` + +Expected: exit 0, and on **stdout** either one or more `[SEVERITY] title` blocks +followed by a `window …: , N finding(s)` line, or just that summary +line if the model found nothing. Diagnostics, if any, appear on stderr only. + +Then prove the machine-readable path, which is what systemd actually captures: + +```bash +ssh ${NODE} "ANOMALY_LLM_API_KEY='${ORKEY}' runagent -m loki1 env \ + ANOMALY_LLM_BASE_URL=https://openrouter.ai/api/v1 \ + ANOMALY_LLM_MODEL=google/gemma-4-26b-a4b-it:free \ + ANOMALY_LLM_API_KEY=\"\$ANOMALY_LLM_API_KEY\" \ + python3 /tmp/anomaly-detector --since 1h --no-webhook" \ + | while read -r line; do echo "$line" | python3 -m json.tool >/dev/null \ + && echo "valid JSON: $(echo "$line" | head -c 80)" \ + || echo "NOT JSON: $line"; done +``` + +Expected: **every** stdout line parses as JSON — one object per finding plus the +summary object. A single `NOT JSON` line means diagnostics leaked into stdout, +which would poison the Loki recall query in stage 3; treat it as a failure. + +Finally confirm the schema-rejection path is real rather than theoretical, by +pointing at a model that ignores `response_format`: + +```bash +ssh ${NODE} "ANOMALY_LLM_API_KEY='${ORKEY}' runagent -m loki1 env \ + ANOMALY_LLM_BASE_URL=https://openrouter.ai/api/v1 \ + ANOMALY_LLM_MODEL=google/gemma-3-4b-it \ + ANOMALY_LLM_API_KEY=\"\$ANOMALY_LLM_API_KEY\" \ + python3 /tmp/anomaly-detector --since 1h --no-webhook"; echo "exit=$?" +``` + +Expected: either a clean run (if that model happens to comply) or `exit=1` with a +message on stderr containing the body truncated to 500 characters and **no partial +finding on stdout**. What must never happen is a malformed finding being emitted. + +Report the observed token count and the model's actual findings in your report — +prompt quality is a judgement call that only real logs can settle. + +- [ ] **Step 8: Commit** + +```bash +git add imageroot/bin/anomaly-detector tests/unit/test_anomaly_detector.py +git commit -m "feat(anomaly-detector): add config precedence, LLM call, emit and CLI" +``` + +--- + +### Task 6: Systemd units, unit refresh and backup inclusion + +**Files:** +- Create: `imageroot/systemd/user/anomaly-detector.service` +- Create: `imageroot/systemd/user/anomaly-detector.timer` +- Create: `imageroot/update-module.d/15systemd` +- Modify: `imageroot/etc/state-include.conf` + +**Interfaces:** +- Consumes: `imageroot/bin/anomaly-detector` (Task 5). +- Produces: the unit names `anomaly-detector.service` and `anomaly-detector.timer`, which Task 7 enables and Tasks 8–9 query. + +- [ ] **Step 1: Write the unit files** + +Create `imageroot/systemd/user/anomaly-detector.service` (mode 644): + +``` +[Unit] +Description=Loki anomaly detector +Requires=loki-server.service +After=loki-server.service + +[Service] +Type=oneshot +EnvironmentFile=%E/state/environment +EnvironmentFile=-%E/state/secrets.env +ExecStart=runagent %E/bin/anomaly-detector +SyslogIdentifier=%u/%N +``` + +There is deliberately no `[Install]` section: the timer is what gets enabled. + +`SyslogIdentifier=%u/%N` is mandatory, not cosmetic. Alloy (configured by +`core/imageroot/var/lib/nethserver/node/bin/generate-promtail-config` in ns8-core) +assigns the `module_id` label only when the module name appears in `_SYSTEMD_UNIT`, +`SYSLOG_IDENTIFIER` or `CONTAINER_NAME`. A rootless user unit reports +`_SYSTEMD_UNIT=user@.service`, which contains no module name, so without this +the detector's own findings would be ingested unlabeled and the recall query in +`recall_findings()` would never match them. `%u/%N` expands to `loki1/anomaly-detector`. + +`EnvironmentFile=-%E/state/secrets.env` carries the leading `-` because the file +does not exist until `set-anomaly-detector` runs; `runagent` loads `state/environment` +but not `secrets.env`. + +Create `imageroot/systemd/user/anomaly-detector.timer` (mode 644): + +``` +[Unit] +Description=Loki anomaly detector timer + +[Timer] +OnCalendar=hourly +Persistent=true +RandomizedDelaySec=5m +FixedRandomDelay=true + +[Install] +WantedBy=timers.target +``` + +`Persistent=true` recovers a single window missed across a reboot. +`RandomizedDelaySec=5m` with `FixedRandomDelay=true` keeps the fire off the exact +top of the hour so the journal has settled and Loki has ingested it; the analysed +window is derived from the wall clock, so the delay never shifts it. This matches +`ns8-nethvoice/imageroot/systemd/user/nethvoice-cdr-cleanup.timer`. + +- [ ] **Step 2: Add the update hook** + +Create `imageroot/update-module.d/15systemd` (mode 755): + +```bash +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e + +# Redirect any output to the journal (stderr) +exec 1>&2 + +# Pick up unit files added by this update (anomaly-detector.service/.timer) +systemctl --user daemon-reload +``` + +This is idempotent and does not enable anything: the detector stays disabled until +`set-anomaly-detector` is called. It runs before `20restart`, which restarts +`loki.service`. + +- [ ] **Step 3: Include the secrets file in backups** + +Modify `imageroot/etc/state-include.conf`. Current content is the single line +`volumes/loki-server-data`. Result: + +``` +state/secrets.env +volumes/loki-server-data +``` + +Paths are relative to the module install root. The Restic repository is encrypted, +so the API key is protected at rest; including it means restore keeps a configured +detector working. + +- [ ] **Step 4: Verify the units parse** + +```bash +chmod 644 imageroot/systemd/user/anomaly-detector.service imageroot/systemd/user/anomaly-detector.timer +chmod 755 imageroot/update-module.d/15systemd +systemd-analyze verify --user-unit imageroot/systemd/user/anomaly-detector.timer 2>&1 | grep -v 'loki-server.service' || true +``` + +Expected: no syntax errors reported for `anomaly-detector.timer` or +`anomaly-detector.service`. Complaints about the missing `loki-server.service` +dependency are expected outside a real module and are filtered out above. + +- [ ] **Step 5: Verify the units load on the test node** + +Sync `imageroot/` to the node as described in "Deploying to the node between +tasks", then: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +ssh ${NODE} 'runagent -m loki1 systemctl --user show --property=LoadState anomaly-detector.service +runagent -m loki1 systemctl --user show --property=LoadState anomaly-detector.timer +runagent -m loki1 systemctl --user is-enabled anomaly-detector.timer +runagent -m loki1 systemctl --user show anomaly-detector.service -p SyslogIdentifier --value' +``` + +Expected: `LoadState=loaded` twice; `disabled` for `is-enabled` (the detector is +off until configured); `SyslogIdentifier` prints `loki1/anomaly-detector` — this +is the check that the `%u/%N` expansion actually produces the string the recall +query and the `module_id` label rule depend on. Any other value here breaks +stage 3. + +- [ ] **Step 6: Commit** + +```bash +git add imageroot/systemd/user/anomaly-detector.service \ + imageroot/systemd/user/anomaly-detector.timer \ + imageroot/update-module.d/15systemd \ + imageroot/etc/state-include.conf +git commit -m "feat(anomaly-detector): add oneshot service, hourly timer and state include" +``` + +--- + +### Task 7: The `set-anomaly-detector` action + +**Files:** +- Create: `imageroot/actions/set-anomaly-detector/validate-input.json` +- Create: `imageroot/actions/set-anomaly-detector/10set` + +**Interfaces:** +- Consumes: the unit name `anomaly-detector.timer` (Task 6); the env var names in `CONFIG_KEYS` (Task 5). +- Produces: `state/environment` entries `ANOMALY_LLM_BASE_URL`, `ANOMALY_LLM_MODEL`, `ANOMALY_MAX_LINES`, `ANOMALY_WEBHOOK_URL`; `state/secrets.env` entries `ANOMALY_LLM_API_KEY`, `ANOMALY_WEBHOOK_TOKEN`. Task 8 reads all of these back. + +- [ ] **Step 1: Write the input schema** + +Create `imageroot/actions/set-anomaly-detector/validate-input.json` (mode 644): + +```json +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://schema.nethserver.org/loki/set-anomaly-detector.json", + "title": "Configure Loki anomaly detector", + "description": "Configure the hourly LLM-based journal anomaly detector.", + "type": "object", + "properties": { + "active": { + "type": "boolean", + "description": "Enable or disable the hourly detector timer." + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "OpenAI-compatible API base URL, without the /chat/completions suffix." + }, + "model": { + "type": "string", + "minLength": 1, + "description": "Model name passed to the completions endpoint." + }, + "api_key": { + "type": "string", + "minLength": 1, + "description": "API key for the completions endpoint. Stored outside Redis and never returned." + }, + "max_lines": { + "type": "integer", + "minimum": 1, + "maximum": 5000, + "description": "Cap on prefiltered log lines sent per window." + }, + "webhook_url": { + "type": "string", + "description": "Optional URL receiving a copy of each report. Empty string clears it." + }, + "webhook_token": { + "type": "string", + "description": "Optional bearer token for the webhook. Empty string clears it." + } + }, + "oneOf": [ + { + "properties": { + "active": {"enum": [true]} + }, + "required": [ + "active", + "base_url", + "model", + "api_key" + ] + }, + { + "properties": { + "active": {"enum": [false]} + }, + "required": ["active"] + } + ] +} +``` + +- [ ] **Step 2: Write the action step** + +Create `imageroot/actions/set-anomaly-detector/10set` (mode 755): + +```python +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import os +import subprocess +import sys + +import agent + +# Secrets never go through agent.set_env: that writes state/environment, +# which is mirrored into the Redis hash module//environment. +SECRETS_FILE = "secrets.env" +SECRET_KEYS = ("ANOMALY_LLM_API_KEY", "ANOMALY_WEBHOOK_TOKEN") +PUBLIC_KEYS = ("ANOMALY_LLM_BASE_URL", "ANOMALY_LLM_MODEL", + "ANOMALY_MAX_LINES", "ANOMALY_WEBHOOK_URL") + +DEFAULT_MAX_LINES = 500 + + +def read_secrets(): + """Read secrets.env, merging rather than overwriting unrelated keys.""" + try: + return agent.read_envfile(SECRETS_FILE) + except FileNotFoundError: + return {} + + +def write_secrets(secrets): + """Write secrets.env and force 0600. + + safe_open() preserves the mode of an existing file, but a freshly + created one would otherwise depend on the umask. + """ + agent.write_envfile(SECRETS_FILE, secrets) + os.chmod(SECRETS_FILE, 0o600) + + +request = json.load(sys.stdin) +secrets = read_secrets() + +if request['active']: + agent.set_env('ANOMALY_LLM_BASE_URL', request['base_url'].rstrip('/')) + agent.set_env('ANOMALY_LLM_MODEL', request['model']) + agent.set_env('ANOMALY_MAX_LINES', str(request.get('max_lines', DEFAULT_MAX_LINES))) + + if request.get('webhook_url'): + agent.set_env('ANOMALY_WEBHOOK_URL', request['webhook_url']) + else: + agent.unset_env('ANOMALY_WEBHOOK_URL') + + secrets['ANOMALY_LLM_API_KEY'] = request['api_key'] + if request.get('webhook_token'): + secrets['ANOMALY_WEBHOOK_TOKEN'] = request['webhook_token'] + else: + secrets.pop('ANOMALY_WEBHOOK_TOKEN', None) + + action = 'enable' +else: + agent.munset_env(list(PUBLIC_KEYS)) + for key in SECRET_KEYS: + secrets.pop(key, None) + action = 'disable' + +write_secrets(secrets) + +# The unit files may be new to this installation +subprocess.run(["systemctl", "--user", "daemon-reload"], + stdout=sys.stderr, + stderr=sys.stderr, + text=True, + check=True) + +# Enable or disable the timer, never the oneshot service. Re-running while +# active only rewrites the configuration: the oneshot reads its environment +# at each fire, so the timer needs no restart. +subprocess.run(["systemctl", "--user", action, "--now", "anomaly-detector.timer"], + stdout=sys.stderr, + stderr=sys.stderr, + text=True, + check=True) +``` + +- [ ] **Step 3: Verify the schema is valid draft-04 and the script compiles** + +```bash +chmod 755 imageroot/actions/set-anomaly-detector/10set +chmod 644 imageroot/actions/set-anomaly-detector/validate-input.json +python3 -c "import json; json.load(open('imageroot/actions/set-anomaly-detector/validate-input.json'))" +python3 -m py_compile imageroot/actions/set-anomaly-detector/10set && echo OK +``` + +Expected: no output from the `json.load`, then `OK`. + +- [ ] **Step 4: Verify both `oneOf` branches against the schema** + +```bash +python3 - <<'PY' +import json, jsonschema # pip install jsonschema if missing +schema = json.load(open('imageroot/actions/set-anomaly-detector/validate-input.json')) +validator = jsonschema.Draft4Validator(schema) +ok = [ + {"active": False}, + {"active": True, "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", "api_key": "sk-x"}, + {"active": True, "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", "api_key": "sk-x", "max_lines": 100, + "webhook_url": "https://example.org/hook", "webhook_token": "t"}, +] +bad = [ + {"active": True}, + {"active": True, "base_url": "https://x/v1", "model": "m"}, + {"active": True, "base_url": "https://x/v1", "api_key": "k"}, + {}, + {"active": True, "base_url": "https://x/v1", "model": "m", + "api_key": "k", "max_lines": 0}, +] +for payload in ok: + validator.validate(payload) +for payload in bad: + assert not validator.is_valid(payload), payload +print("schema OK") +PY +``` + +Expected: `schema OK`. + +- [ ] **Step 5: Verify the action on the test node** + +Sync `imageroot/` to the node, then run the action for real. Use a deliberately +unreachable `base_url` so nothing is ever sent anywhere: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{ + \"active\": true, + \"base_url\": \"http://127.0.0.1:9\", + \"model\": \"probe-model\", + \"api_key\": \"sk-probe-key-do-not-use\", + \"max_lines\": 50 +}"' + +ssh ${NODE} 'echo "== timer:"; runagent -m loki1 systemctl --user is-active anomaly-detector.timer +echo "== mode:"; runagent -m loki1 stat -c %a state/secrets.env +echo "== secrets keys:"; runagent -m loki1 sed "s/=.*/=/" state/secrets.env +echo "== public env:"; runagent -m loki1 grep ^ANOMALY_ state/environment +echo "== redis (must be empty):"; redis-cli hgetall module/loki1/environment | grep -i -e api_key -e webhook_token || echo NONE' +``` + +Expected: timer `active`; mode `600`; `secrets.env` lists +`ANOMALY_LLM_API_KEY=` and no `ANOMALY_WEBHOOK_TOKEN`; `state/environment` +carries `ANOMALY_LLM_BASE_URL`, `ANOMALY_LLM_MODEL`, `ANOMALY_MAX_LINES`; +the Redis grep prints `NONE`. **`NONE` is the load-bearing assertion of this +task** — the whole reason secrets bypass `agent.set_env`. + +Then verify idempotence and teardown: + +```bash +# re-running while active must succeed and leave the timer active +ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{ + \"active\": true, \"base_url\": \"http://127.0.0.1:9\", + \"model\": \"probe-model-2\", \"api_key\": \"sk-probe-key-do-not-use\"}" +runagent -m loki1 systemctl --user is-active anomaly-detector.timer +runagent -m loki1 grep ANOMALY_LLM_MODEL state/environment' + +# disabling must clear both the timer and the stored key +ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{\"active\": false}" +runagent -m loki1 systemctl --user is-active anomaly-detector.timer || true +runagent -m loki1 cat state/secrets.env +runagent -m loki1 grep ^ANOMALY_ state/environment || echo "no ANOMALY_ vars left"' +``` + +Expected: still `active` with `ANOMALY_LLM_MODEL=probe-model-2` after the second +enable; then `inactive`, an empty `secrets.env`, and `no ANOMALY_ vars left`. + +Leave the detector disabled at the end of this task. + +- [ ] **Step 6: Commit** + +```bash +git add imageroot/actions/set-anomaly-detector +git commit -m "feat(anomaly-detector): add set-anomaly-detector action" +``` + +--- + +### Task 8: Expose detector state through `get-configuration` + +**Files:** +- Modify: `imageroot/actions/get-configuration/10get` +- Modify: `imageroot/actions/get-configuration/validate-output.json` + +**Interfaces:** +- Consumes: the env vars written by Task 7; the unit names from Task 6. +- Produces: the `anomaly_detector` object in the `get-configuration` output, asserted by Task 9. + +- [ ] **Step 1: Add the anomaly detector block to `10get`** + +In `imageroot/actions/get-configuration/10get`, insert the following **after** the +Syslog block (after the `syslog["last_timestamp"] = ""` `except` clause) and +**before** the `# General` comment. It reuses the file's existing `match`/`case` +status idiom and its `os.getenv` style. + +```python +# Anomaly detector + +anomaly_detector = {} + +ad_status = subprocess.run(['systemctl', '--user', 'is-active', 'anomaly-detector.timer'], capture_output=True, text=True) +match ad_status.stdout.strip(): + case 'active': + anomaly_detector["status"] = 'active' + case 'failed': + anomaly_detector["status"] = 'failed' + case _: + anomaly_detector["status"] = 'inactive' + +anomaly_detector["base_url"] = os.getenv('ANOMALY_LLM_BASE_URL', '') +anomaly_detector["model"] = os.getenv('ANOMALY_LLM_MODEL', '') +anomaly_detector["max_lines"] = int(os.getenv('ANOMALY_MAX_LINES', '500')) +anomaly_detector["webhook_url"] = os.getenv('ANOMALY_WEBHOOK_URL', '') + +# The API key value is never returned, only its presence. +try: + anomaly_detector["api_key_configured"] = bool(agent.read_envfile('secrets.env').get('ANOMALY_LLM_API_KEY')) +except FileNotFoundError: + anomaly_detector["api_key_configured"] = False + +ad_last_run = subprocess.run(['systemctl', '--user', 'show', 'anomaly-detector.service', '-p', 'ExecMainExitTimestamp', '--value'], capture_output=True, text=True) +anomaly_detector["last_run"] = ad_last_run.stdout.strip() +``` + +Then add the object to the response dict, so it reads: + +```python +response = { + "retention_days": int(os.getenv('LOKI_RETENTION_PERIOD')), + "active_from": os.getenv('LOKI_ACTIVE_FROM'), + "cloud_log_manager": cloud_log_manager, + "syslog": syslog, + "anomaly_detector": anomaly_detector +} +``` + +- [ ] **Step 2: Declare it in `validate-output.json`** + +Add `"anomaly_detector"` to the top-level `required` array, and this property +alongside the existing `cloud_log_manager` and `syslog` properties: + +```json + "anomaly_detector": { + "type": "object", + "title": "Anomaly detector", + "description": "State of the hourly LLM-based journal anomaly detector.", + "required": ["status", "api_key_configured"], + "properties": { + "status": { + "type": "string", + "enum": ["active", "failed", "inactive"], + "description": "State of anomaly-detector.timer." + }, + "base_url": { + "type": "string", + "description": "OpenAI-compatible API base URL." + }, + "model": { + "type": "string", + "description": "Model name." + }, + "max_lines": { + "type": "integer", + "description": "Cap on prefiltered log lines per window." + }, + "webhook_url": { + "type": "string", + "description": "Optional report delivery URL." + }, + "api_key_configured": { + "type": "boolean", + "description": "True when an API key is stored. The key value is never returned." + }, + "last_run": { + "type": "string", + "description": "ExecMainExitTimestamp of anomaly-detector.service, empty if never run." + } + } + } +``` + +- [ ] **Step 3: Verify the schema is still valid and the script compiles** + +```bash +python3 -c "import json; json.load(open('imageroot/actions/get-configuration/validate-output.json'))" +python3 -m py_compile imageroot/actions/get-configuration/10get && echo OK +``` + +Expected: no output, then `OK`. + +- [ ] **Step 4: Verify a representative response validates** + +```bash +python3 - <<'PY' +import json, jsonschema +schema = json.load(open('imageroot/actions/get-configuration/validate-output.json')) +response = { + "retention_days": 7, + "active_from": "2026-07-29T13:00:00+00:00", + "cloud_log_manager": {"status": "inactive"}, + "syslog": {"status": "inactive"}, + "anomaly_detector": { + "status": "active", "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", "max_lines": 500, "webhook_url": "", + "api_key_configured": True, "last_run": "Wed 2026-07-29 14:00:11 UTC", + }, +} +jsonschema.Draft4Validator(schema).validate(response) +missing = dict(response); del missing["anomaly_detector"] +assert not jsonschema.Draft4Validator(schema).is_valid(missing) +print("output schema OK") +PY +``` + +Expected: `output schema OK`. + +- [ ] **Step 5: Verify on the test node, both configured and not** + +Sync `imageroot/` to the node. First with the detector off: + +```bash +NODE=root@rl1.leader.default.gs.nethserver.net +ssh ${NODE} 'api-cli run module/loki1/get-configuration | python3 -m json.tool' +``` + +Expected: an `anomaly_detector` object with `"status": "inactive"`, +`"api_key_configured": false`, empty `base_url`/`model`/`webhook_url`, +`"max_lines": 500`, and a `last_run` string. The action must exit 0 — a non-zero +exit here means the output failed its own `validate-output.json`. + +Then with it on: + +```bash +ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{ + \"active\": true, \"base_url\": \"http://127.0.0.1:9\", + \"model\": \"probe-model\", \"api_key\": \"sk-probe-key-do-not-use\"}" +api-cli run module/loki1/get-configuration | python3 -m json.tool' +``` + +Expected: `"status": "active"`, `"api_key_configured": true`, +`"model": "probe-model"`, and **the string `sk-probe-key-do-not-use` appears +nowhere in the output**. Confirm that explicitly: + +```bash +ssh ${NODE} 'api-cli run module/loki1/get-configuration | grep -c sk-probe-key-do-not-use || echo "key not leaked"' +ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{\"active\": false}"' +``` + +Expected: `key not leaked`, then the detector is left disabled. + +- [ ] **Step 6: Commit** + +```bash +git add imageroot/actions/get-configuration +git commit -m "feat(anomaly-detector): expose detector state in get-configuration" +``` + +--- + +### Task 9: Robot test against a stub LLM + +**Files:** +- Create: `tests/llm-stub.py` +- Create: `tests/20__anomaly_detector.robot` + +**Interfaces:** +- Consumes: `set-anomaly-detector` (Task 7), `get-configuration` (Task 8), both units (Task 6), the script's `--since/--pretty/--no-webhook` flags (Task 5). +- Produces: nothing consumed downstream. + +The suite inherits `Connect to the node`, `Wait until boot completes` and the +journal collection from `tests/__init__.robot`, which applies its `Suite Setup` +and `Suite Teardown` to every file in `tests/`. `${MID}` is `loki1`, matching +`tests/10__check_services.robot`. + +- [ ] **Step 1: Write the stub LLM server** + +Create `tests/llm-stub.py` (mode 644 — it is copied to the node and run with +`python3 `, never executed in place): + +```python +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# +# Canned OpenAI-compatible chat completions server for the Robot suite. +# No real LLM in CI: no network egress, no cost, deterministic. +# +# python3 llm-stub.py [PORT] +# + +import json +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +CANNED_REPORT = { + "window_assessment": "degraded", + "findings": [ + { + "severity": "high", + "title": "ROBOTSTUBFINDING synthetic error burst", + "summary": "Synthetic errors injected by the Robot test suite.", + "evidence": ["<3> [1:loki1:robot-noise] robot synthetic error"], + "modules": ["loki1"], + "suggested_action": "None: this finding is produced by the test stub.", + } + ], +} + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get('Content-Length') or 0) + self.rfile.read(length) + payload = { + "id": "chatcmpl-robotstub", + "object": "chat.completion", + "model": "robot-stub", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": { + "role": "assistant", + "content": json.dumps(CANNED_REPORT), + }, + } + ], + } + body = json.dumps(payload).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + self.send_response(200) + self.send_header('Content-Length', '2') + self.end_headers() + self.wfile.write(b'ok') + + def log_message(self, fmt, *args): + sys.stderr.write("llm-stub: " + (fmt % args) + "\n") + + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 9099 + HTTPServer(('127.0.0.1', port), Handler).serve_forever() +``` + +- [ ] **Step 2: Write the Robot suite** + +Create `tests/20__anomaly_detector.robot` (mode 644): + +```robotframework +*** Settings *** +Library SSHLibrary +Library String +Suite Setup Start the stub LLM server +Suite Teardown Tear down the anomaly detector + +*** Variables *** +${MID} loki1 +${STUB_PORT} 9099 +${STUB_URL} http://127.0.0.1:${STUB_PORT}/v1 +${STUB_TITLE} ROBOTSTUBFINDING +${NOISE_TAG} robot-noise + +*** Keywords *** +Start the stub LLM server + Put File ${CURDIR}/llm-stub.py /tmp/llm-stub.py + Execute Command setsid nohup python3 /tmp/llm-stub.py ${STUB_PORT} /tmp/llm-stub.log 2>&1 & + Wait Until Keyword Succeeds 30s 2s The stub LLM server answers + +The stub LLM server answers + ${output} ${rc} = Execute Command + ... curl -sf http://127.0.0.1:${STUB_PORT}/ return_rc=${True} + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${output} ok + +Tear down the anomaly detector + Execute Command api-cli run module/${MID}/set-anomaly-detector --data '{"active":false}' + Execute Command pkill -f llm-stub.py + +Run module action + [Arguments] ${action} ${data}=${EMPTY} + IF '${data}' == '${EMPTY}' + ${output} ${rc} = Execute Command + ... api-cli run module/${MID}/${action} return_rc=${True} + ELSE + ${output} ${rc} = Execute Command + ... api-cli run module/${MID}/${action} --data '${data}' return_rc=${True} + END + Should Be Equal As Integers ${rc} 0 action ${action} failed: ${output} + RETURN ${output} + +Query Loki for the detector output + ${command} = Catenate + ... runagent -m ${MID} bash -c + ... 'LOKI_ADDR=http://127.0.0.1:$LOKI_HTTP_PORT + ... LOKI_USERNAME=$LOKI_API_AUTH_USERNAME + ... LOKI_PASSWORD=$LOKI_API_AUTH_PASSWORD + ... logcli query --limit 50 --since 20m --forward --no-labels -q -o raw + ... "{module_id=\\"${MID}\\"} | json identifier=\\"SYSLOG_IDENTIFIER\\", message=\\"MESSAGE\\" + ... | identifier=\\"${MID}/anomaly-detector\\" | line_format \\"{{.message}}\\""' + ${output} ${rc} = Execute Command ${command} return_rc=${True} + Should Be Equal As Integers ${rc} 0 logcli failed: ${output} + RETURN ${output} + +The detector output is in Loki + ${output} = Query Loki for the detector output + Should Contain ${output} window_assessment + +*** Test Cases *** +Configure the anomaly detector against the stub + ${data} = Catenate SEPARATOR= + ... {"active":true, + ... "base_url":"${STUB_URL}", + ... "model":"robot-stub", + ... "api_key":"sk-robot-stub-key", + ... "max_lines":50} + Run module action set-anomaly-detector ${data} + ${output} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user is-active anomaly-detector.timer + ... return_rc=${True} + Should Be Equal As Strings ${output} active + +The secrets file is not world readable + ${output} ${rc} = Execute Command + ... runagent -m ${MID} stat -c %a state/secrets.env return_rc=${True} + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${output} 600 + +The API key never reaches Redis + ${output} = Run module action get-configuration + Should Not Contain ${output} sk-robot-stub-key + Should Contain ${output} "api_key_configured": true + +The oneshot service runs and lands in the journal and in Loki + ${output} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user start anomaly-detector.service + ... return_rc=${True} + Should Be Equal As Integers ${rc} 0 service failed to run: ${output} + ${result} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user show anomaly-detector.service -p Result --value + ... return_rc=${True} + Should Be Equal As Strings ${result} success + # The summary line is emitted for every window, nominal or not + ${journal} ${rc} = Execute Command + ... journalctl --no-pager -o cat SYSLOG_IDENTIFIER=${MID}/anomaly-detector + ... return_rc=${True} + Should Be Equal As Integers ${rc} 0 + Should Contain ${journal} window_assessment + # Proves the SyslogIdentifier setting produced the module_id label + Wait Until Keyword Succeeds 90s 10s The detector output is in Loki + +The LLM path produces the canned finding + # Inject errors into the current hour, then analyse a window that + # contains them, so the run cannot take the nominal early-exit. + FOR ${i} IN RANGE 5 + Execute Command logger -p daemon.err -t ${NOISE_TAG} robot synthetic error ${i} + END + Sleep 20s let the collector ship the noise to Loki + ${command} = Catenate + ... runagent -m ${MID} env ANOMALY_LLM_BASE_URL=${STUB_URL} + ... ANOMALY_LLM_MODEL=robot-stub ANOMALY_LLM_API_KEY=sk-robot-stub-key + ... python3 bin/../bin/anomaly-detector --since 30m --pretty --no-webhook + ${output} ${rc} = Execute Command ${command} return_rc=${True} + Should Be Equal As Integers ${rc} 0 manual run failed: ${output} + Should Contain ${output} ${STUB_TITLE} + +The dry run makes no LLM call + ${before} ${rc} = Execute Command wc -l < /tmp/llm-stub.log return_rc=${True} + ${command} = Catenate + ... runagent -m ${MID} python3 bin/../bin/anomaly-detector --dry-run --since 30m + ${output} ${rc} = Execute Command ${command} return_rc=${True} + Should Be Equal As Integers ${rc} 0 dry run failed: ${output} + Should Contain ${output} WINDOW + Should Contain ${output} no LLM call + ${after} ${rc2} = Execute Command wc -l < /tmp/llm-stub.log return_rc=${True} + Should Be Equal As Strings ${before} ${after} the dry run contacted the stub + +Disabling the detector clears the secrets + Run module action set-anomaly-detector {"active":false} + ${output} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user is-active anomaly-detector.timer + ... return_rc=${True} + Should Not Be Equal As Strings ${output} active + ${secrets} ${rc} = Execute Command + ... runagent -m ${MID} cat state/secrets.env return_rc=${True} + Should Not Contain ${secrets} ANOMALY_LLM_API_KEY + ${config} = Run module action get-configuration + Should Contain ${config} "api_key_configured": false +``` + +Two details that will bite if changed: the manual-run tests invoke the script with +`python3 bin/../bin/anomaly-detector` because `runagent` chdirs to +`AGENT_STATE_DIR` (`%E/state`), so the script is one level up at `%E/bin/`; and the +`env VAR=...` prefix is needed because `runagent` loads `state/environment` but not +`state/secrets.env`, so the API key is not in the inherited environment. + +- [ ] **Step 3: Run the suite against the test node** + +Sync `imageroot/` to the node first (see "Deploying to the node between tasks"), +because `test-module.sh` with the default `SCENARIO=install` does not reinstall +the module — it tests whatever code is already on the node. + +```bash +SSH_KEYFILE=~/.ssh/id_ecdsa ./test-module.sh \ + rl1.leader.default.gs.nethserver.net ghcr.io/nethserver/loki:latest \ + --suite '*anomaly*' +``` + +Expected: 7 tests PASS. Two to watch: + +- `The LLM path produces the canned finding` — if it fails because `is_nominal` + took the early exit, the injected `daemon.err` lines have not reached Loki yet; + raise the `Sleep 20s`. +- `The oneshot service runs and lands in the journal and in Loki` — the + `Wait Until Keyword Succeeds 90s` covers collector latency. If it still fails, + check `SyslogIdentifier` on the node before suspecting the query: + `ssh root@rl1.leader.default.gs.nethserver.net journalctl -o cat SYSLOG_IDENTIFIER=loki1/anomaly-detector` + +Leave the detector disabled afterwards — the suite teardown does this, but +confirm: + +```bash +ssh root@rl1.leader.default.gs.nethserver.net \ + 'runagent -m loki1 systemctl --user is-active anomaly-detector.timer || true; pkill -f llm-stub.py || true' +``` + +- [ ] **Step 4: Commit** + +```bash +git add tests/llm-stub.py tests/20__anomaly_detector.robot +git commit -m "test(anomaly-detector): end-to-end suite against a stub LLM server" +``` + +--- + +### Task 10: Documentation + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Consumes: the action name and parameters (Task 7), the `get-configuration` shape (Task 8), the CLI flags (Task 5). +- Produces: nothing. + +The README currently has this outline: `# Loki` → `## Install` → `## Usage` → +`## APIs` → `### configure-module` (`#### Parameters`, `#### Example`) → +`### get-configuration` (`#### Example`) → `## Uninstall`. It documents neither +`set-clm-forwarder` nor `set-syslog-forwarder`, so there is no prior section to +copy — the style to follow is `### \`action-name\`` heading, a one-line +description, a `#### Parameters` bullet list, and a `#### Example` with a +` ```bash ` fenced `api-cli run ...` command. + +- [ ] **Step 1: Add the API section** + +Insert into `README.md` after the `### \`get-configuration\`` section and before +`## Uninstall`: + +````markdown +### `set-anomaly-detector` + +Configure the hourly anomaly detector. It sends a scrubbed digest of the +cluster journal to an OpenAI-compatible LLM and writes the findings back to the +journal. Disabled by default. + +#### Parameters + +- `active`: enable or disable the hourly timer. Required. +- `base_url`: OpenAI-compatible API base URL, without the `/chat/completions` + suffix. Required when `active` is `true`. +- `model`: model name. Required when `active` is `true`. +- `api_key`: API key. Required when `active` is `true`. Stored in + `state/secrets.env` with mode `0600`, kept out of the Redis environment hash, + and never returned by `get-configuration`. +- `max_lines`: cap on prefiltered log lines sent per window. Optional, default + `500`. +- `webhook_url`: optional URL receiving a copy of each report. Pass an empty + string to clear it. +- `webhook_token`: optional bearer token for the webhook. Pass an empty string + to clear it. + +#### Example + +```bash +api-cli run module/loki1/set-anomaly-detector --data '{ + "active": true, + "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", + "api_key": "sk-...", + "max_lines": 500 +}' +``` + +Disable it again, which also removes the stored key and token: + +```bash +api-cli run module/loki1/set-anomaly-detector --data '{"active": false}' +``` + +#### Findings + +Findings are written to the journal as one JSON object per line under +`SYSLOG_IDENTIFIER=loki1/anomaly-detector`, plus one summary line per window +carrying `window_assessment` (`nominal`, `degraded` or `incident`). The log +collector ships them to Loki, so they are queryable and graphable like any other +log: + +```bash +runagent -m loki1 bash -c 'LOKI_ADDR=http://127.0.0.1:$LOKI_HTTP_PORT \ + LOKI_USERNAME=$LOKI_API_AUTH_USERNAME LOKI_PASSWORD=$LOKI_API_AUTH_PASSWORD \ + logcli query --since 24h -o raw \ + "{module_id=\"loki1\"} | json identifier=\"SYSLOG_IDENTIFIER\", message=\"MESSAGE\" | identifier=\"loki1/anomaly-detector\" | line_format \"{{.message}}\""' +``` + +An empty findings list is the normal outcome. The detector reads its own last 10 +findings back out of Loki each run and is instructed not to repeat them, so its +memory needs no state file and nothing extra to back up. + +#### Manual execution + +The script is also a CLI, with every flag optional, so systemd invokes it with +none. It can be run on any node with an installed Loki module — no unit, no +configuration, nothing written to module state: + +```bash +# see exactly what would be sent, with a character and token count. +# no key needed, no cost. +runagent -m loki1 python3 ../bin/anomaly-detector --dry-run --since 2h + +# real call, findings on the terminal, no webhook delivery +runagent -m loki1 env \ + ANOMALY_LLM_BASE_URL=https://api.openai.com/v1 \ + ANOMALY_LLM_MODEL=gpt-4o-mini \ + ANOMALY_LLM_API_KEY=sk-... \ + python3 ../bin/anomaly-detector --since 2h --pretty --no-webhook +``` + +`runagent` changes directory to the module state directory, hence the +`../bin/` prefix. + +| Flag | Effect | +|------|--------| +| `--dry-run` | collect, digest, scrub and render the prompt, print it with a character and approximate token count, make no LLM call, emit no findings | +| `--since 2h` | window becomes `[now-2h, now]`; accepts `30m`, `6h`, `2d` | +| `--config FILE` | read `ANOMALY_*` from a plain env file instead of `environment` and `secrets.env` | +| `--pretty` | render findings as indented text instead of JSON lines | +| `--no-webhook` | skip webhook delivery | +| `--max-lines N` | override the line cap for one run | +| `--print-prompt` | print the prompt to stderr alongside a real LLM call | + +Precedence: CLI flag, then `--config` file, then shell environment, then +`state/secrets.env`, then `state/environment`. + +#### Privacy + +**Enabling the anomaly detector sends log text from your cluster to a +third-party API.** This is the real privacy boundary, and it is the reason the +feature is disabled by default and requires an explicit API key. + +Before each request the detector removes likely secrets — `password=`, +`token=`, `api_key=`, `secret=` and similar assignments, `Authorization` +headers, base64 or hex runs of 32 characters or more, and email addresses. +IP addresses, hostnames, module IDs and usernames are deliberately **kept**, +because they carry the anomaly signal. This scrubbing is defence in depth, not +a guarantee. + +Point `base_url` at a self-hosted gateway (vLLM, Ollama, or any +OpenAI-compatible endpoint) if log text must not leave your infrastructure. +The API key is stored in `state/secrets.env` with mode `0600` and is included in +module backups; the Restic repository is encrypted. +```` + +- [ ] **Step 2: Extend the `get-configuration` example** + +In the existing `### \`get-configuration\`` section, extend the example JSON +response so it shows the new object: + +```json +{ + "retention_days": 7, + "active_from": "2021-05-28T15:49:27Z+00:00", + "active_to": "2021-05-28T15:49:27Z+00:00", + "anomaly_detector": { + "status": "active", + "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", + "max_lines": 500, + "webhook_url": "", + "api_key_configured": true, + "last_run": "Wed 2026-07-29 14:00:11 UTC" + } +} +``` + +- [ ] **Step 3: Verify the rendered Markdown** + +```bash +grep -n '^#' README.md +``` + +Expected: the outline now reads `# Loki`, `## Install`, `## Usage`, `## APIs`, +`### configure-module`, `#### Parameters`, `#### Example`, `### get-configuration`, +`#### Example`, `### set-anomaly-detector`, `#### Parameters`, `#### Example`, +`#### Findings`, `#### Manual execution`, `#### Privacy`, `## Uninstall`. + +- [ ] **Step 4: Commit** + +```bash +git add README.md +git commit -m "docs(anomaly-detector): document the action, manual runs and privacy boundary" +``` + +--- + +## Final verification + +- [ ] `./test-unit.sh` — every pytest test passes. +- [ ] `SSH_KEYFILE=~/.ssh/id_ecdsa ./test-module.sh rl1.leader.default.gs.nethserver.net ghcr.io/nethserver/loki:latest` — both Robot suites pass. +- [ ] `git ls-files -s imageroot/bin/anomaly-detector imageroot/actions/set-anomaly-detector/10set imageroot/update-module.d/15systemd test-unit.sh` — all four are mode `100755`. +- [ ] `git ls-files -s imageroot/systemd/user/anomaly-detector.service imageroot/systemd/user/anomaly-detector.timer imageroot/actions/set-anomaly-detector/validate-input.json` — all three are mode `100644`. +- [ ] `grep -rn '%S' imageroot/systemd/` returns nothing. +- [ ] Full timer-driven run against the test node, using the OpenRouter endpoint — + the last thing the Robot stub cannot prove, because it exercises the real unit, + the real secret store and the real journal round-trip together: + ```bash + NODE=root@rl1.leader.default.gs.nethserver.net + ORKEY=$(grep -oE 'sk-or-[A-Za-z0-9._-]+' open_router | head -1) + ssh ${NODE} "api-cli run module/loki1/set-anomaly-detector --data '{ + \"active\": true, + \"base_url\": \"https://openrouter.ai/api/v1\", + \"model\": \"google/gemma-4-26b-a4b-it:free\", + \"api_key\": \"${ORKEY}\" + }'" + ssh ${NODE} 'runagent -m loki1 systemctl --user start anomaly-detector.service + runagent -m loki1 systemctl --user show anomaly-detector.service -p Result --value + journalctl --no-pager -o cat SYSLOG_IDENTIFIER=loki1/anomaly-detector | tail -5' + ``` + Expected: `Result=success`, and the journal tail shows JSON findings written by + the service itself — proving `EnvironmentFile=-%E/state/secrets.env` delivered the + key and `SyslogIdentifier` labelled the output. Then confirm the round-trip into + Loki, which is the detector's own memory: + ```bash + ssh ${NODE} 'runagent -m loki1 bash -c "LOKI_ADDR=http://127.0.0.1:\$LOKI_HTTP_PORT \ + LOKI_USERNAME=\$LOKI_API_AUTH_USERNAME LOKI_PASSWORD=\$LOKI_API_AUTH_PASSWORD \ + logcli query --since 30m -o raw --limit 20 -q \ + \"{module_id=\\\"loki1\\\"} | json identifier=\\\"SYSLOG_IDENTIFIER\\\", message=\\\"MESSAGE\\\" | identifier=\\\"loki1/anomaly-detector\\\" | line_format \\\"{{.message}}\\\"\""' + ``` + Then **disable it again and confirm the key is gone**, so the test node is not + left holding a live credential: + ```bash + ssh ${NODE} 'api-cli run module/loki1/set-anomaly-detector --data "{\"active\": false}" + runagent -m loki1 cat state/secrets.env + runagent -m loki1 systemctl --user is-active anomaly-detector.timer || true' + ``` + Expected: an empty `secrets.env` and an inactive timer. This step is mandatory, + not tidiness — leaving a real OpenRouter key in a shared test node's module state + is a credential leak. +- [ ] Report the measured prompt size rather than tuning `max_lines` silently. This + node produces ~9k tokens per hour for the `LINES` block alone, against the spec's + 4–6k target; the default of 500 is fixed by the spec, and lowering it is an + operator decision. +- [ ] Confirm the key never entered the repo: `git log -p --all | grep -c 'sk-or-'` + must print `0`, and `git check-ignore open_router test.sh` must list both. + +## Spec coverage + +| Spec section | Task | +|---|---| +| New files: `bin/anomaly-detector` | 1–5 | +| New files: service + timer | 6 | +| New files: `set-anomaly-detector/*` | 7 | +| Modified: `get-configuration/10get`, `validate-output.json` | 8 | +| Modified: `etc/state-include.conf` | 6 | +| Modified: unit installation on update | 6 (`update-module.d/15systemd`, not `10config`) | +| Modified: `README.md` | 10 | +| Stage 1 Window | 2 | +| Stage 2 Collect (digest, baseline, lines, own-identifier exclusion) | 4 | +| Stage 3 Recall own findings | 4 | +| Stage 4 Scrub | 1 | +| Stage 5 Ask | 3 (prompt, schema), 5 (HTTP) | +| Stage 6 Emit (journal, webhook, stderr-only diagnostics, truncation notice) | 5 | +| Configuration + secrets placement | 7 | +| Enabling and disabling | 7 | +| Manual execution flags and precedence | 5, 10 | +| Error handling table | 4 (`run_logcli`), 5 (`ask_llm`, `parse_findings`, `main` guard) | +| Unit tests | 1–5 | +| Robot test | 9 | +| Manual verification | Final verification | +| Out of scope: no UI, no Redis history, no email, no re-hydration, fixed interval | respected — nothing in this plan adds them | diff --git a/docs/superpowers/plans/2026-08-05-nethesis-insights.md b/docs/superpowers/plans/2026-08-05-nethesis-insights.md new file mode 100644 index 0000000..d53c505 --- /dev/null +++ b/docs/superpowers/plans/2026-08-05-nethesis-insights.md @@ -0,0 +1,4617 @@ +# Nethesis Insights Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Build `nethesis-insights`, a Go server that receives deduplicated log bundles from NethServer nodes, gates them against novelty and deviation before spending any LLM call, and stores fingerprinted findings that never repeat. + +**Architecture:** One container running Redpanda, a static Go binary and SQLite under `s6-overlay`. HTTP ingest authenticates by forwarding Basic credentials to an external validator, then produces to a Redpanda topic. A consumer reads bundles, gates them, calls an OpenAI-compatible LLM only when warranted, and upserts findings keyed by a server-computed fingerprint. All storage goes through a `Store` interface so the SQLite backend can be swapped for Postgres. + +**Tech Stack:** Go 1.23, `uptrace/bun` (SQLite + Postgres), `modernc.org/sqlite` (CGO-free), `twmb/franz-go` (Kafka client), `golang-migrate/migrate`, `oklog/ulid`, `golang.org/x/time/rate`, `stretchr/testify`, `s6-overlay`, Redpanda. + +**Spec:** [2026-08-05 Nethesis Insights Design](../specs/2026-08-05-nethesis-insights-design.md) + +**Repository:** `https://github.com/nethesis/nethesis-insights` — already created, public, **empty**, no license, no default branch. Task 1 makes the first commit. + +**License:** GPL-3.0-or-later, matching `ns8-loki`. + +## Development environment + +**All work happens on the dev machine, not locally.** The operator has a local bandwidth limit, and this project pulls a Go module cache, a Redpanda image and a Postgres image. + +``` +host: root@rl1.leader.default.gs.nethserver.net +os: Rocky Linux 9.8 (Blue Onyx) +disk: 154 GB free on / +``` + +Verified present: `podman`. Verified **missing**: `go`, `git`, `make`, `gh`, `golangci-lint` — Task 1 installs all five. + +`rl1` is a **shared live cluster** running other NS8 modules (`nethvoice2`, `crowdsec1`, `samba2`, `metrics1`, `nethvoice-proxy1`, `traefik1`). Work only inside `/root/nethesis-insights`. Never restart another module's services. The containers this project runs bind to high ports (see Task 17) specifically to avoid colliding with them. + +Work checked out at `/root/nethesis-insights`. Every `Run:` step in this plan executes on that host inside that directory unless stated otherwise. + +## Global Constraints + +Every task's requirements implicitly include this section. Values are copied verbatim from the spec. + +**License header — required on every file created by this plan.** Go files: + +```go +// Copyright (C) 2026 Nethesis S.r.l. +// SPDX-License-Identifier: GPL-3.0-or-later +``` + +SQL, YAML, Makefile and shell files use the `#` form, matching `ns8-loki`: + +``` +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +``` + +In Go files the header goes **above** the `package` clause, separated by a blank line so it is not mistaken for a package doc comment. Code blocks in later tasks omit the header to stay readable; prepend it to every file regardless. Task 1 adds a CI step that fails the build on any missing header, so this is enforced rather than trusted. + +**Commit conventions** (NethServer contribution process): +- [Conventional Commits](https://www.conventionalcommits.org/) for every commit. +- **Never put an issue reference in an individual commit message.** Issue references belong in the merge or squash commit body only — this keeps the GitHub reference graph clean. +- Work on a branch, never commit directly to `main`. + +**Schema portability** (spec §3.4) — enforced by the dual-dialect migration test in Task 3: +- IDs generated in Go as ULID. Never `AUTOINCREMENT` or `SERIAL`. +- Timestamps stored as `INTEGER` unix-millis. Never native date types. +- `ON CONFLICT … DO UPDATE` only. Never `INSERT OR REPLACE`. +- JSON held as `TEXT` and parsed in Go. No `jsonb` operators, no SQLite `json1` functions. +- Migrations via `golang-migrate`, one dialect-agnostic SQL directory. + +**SQLite runtime** (spec §3.4): WAL mode, `busy_timeout=5000`, all writes serialized. + +**LLM** (spec §8.2): strict `response_format: json_schema` with `strict: true`. **No `temperature` field** — some models reject any non-default value. `PROMPT_VERSION` is a code constant, never configuration. + +**Secrets** (spec §10): `LLM_API_KEY` and `AUTH_PEPPER` come from the environment only. Never written to the database. Never logged. The auth cache stores `HMAC(pepper, credential)`, never the credential. + +**Data protection** (spec §10): raw `samples` are never written to the database. They exist only in the `bundles` topic. + +**Auth** (spec §4): fail **closed**. Validator unreachable with no cache hit → `503`. + +**Determinism** (spec §8.2): identical bundle input must produce byte-identical prompts. Templates sorted `(module_id, priority, template)`; digest sorted `(module_id, priority)`. + +**Findings ordering** (spec §5.6): severity-descending, then `last_seen` descending. Severity rank: `critical` > `high` > `medium` > `low`. + +--- + +## Deviations from the spec + +Five deliberate refinements. Each is a decision a reviewer should be able to accept or reject on its own. + +1. **`internal/prompt` is its own package.** Spec §3.3 folds prompt assembly into `internal/analyzer`, but §13 requires golden-file tests proving byte-identical output — that is a unit with its own contract. Splitting it keeps `analyzer` free of string building. + +2. **Fingerprint takes `modules []string`, not `module_id`.** Spec §6.2 writes `module_id` singular while the `findings` table in §6 stores `modules` plural. Resolved toward the table: modules are sorted and joined, same as evidence. `category` is derived server-side as `"security"` if any cited template carried `category=security`, else `""` — propagating the edge's classification without the server classifying anything (§8.1). + +3. **SQLite write serialization uses a mutex, not a dedicated goroutine.** Spec §3.4 says "a single writer goroutine owning all writes". A mutex gives the identical serialization guarantee with no channel lifecycle to leak and no shutdown ordering to get wrong. + +4. **Two config keys added:** `LLM_PRICE_INPUT_PER_MTOK` and `LLM_PRICE_OUTPUT_PER_MTOK`. The spec's cost ledger (§6) and daily spend cap (§9.3) compute `cost_micros`, which is impossible without prices. They are configuration rather than constants because provider pricing changes independently of releases. + +5. **The container's final stage is the official Fedora-based Redpanda image, not Alpine.** Project container conventions prefer Alpine but name `*-slim`/glibc images as the documented fallback on musl incompatibility. Redpanda requires glibc. Justified in Task 17. + +--- + +## File structure + +``` +nethesis-insights/ +├── go.mod go.sum Makefile renovate.json README.md +├── .golangci.yml +├── .github/workflows/ci.yml +├── Containerfile +├── compose.yaml # local dev: Redpanda + server +├── s6/ # s6-overlay service definitions +│ ├── redpanda/{type,run,notification-fd} +│ └── insightsd/{type,run,dependencies.d/redpanda} +├── cmd/insightsd/main.go # wiring, config, graceful shutdown +└── internal/ + ├── model/ bundle.go finding.go severity.go # shared types, no deps + ├── store/ store.go sqlite.go postgres.go + │ migrations/*.sql systems.go findings.go prune.go + ├── fingerprint/ fingerprint.go # pure + ├── gate/ gate.go # pure + ├── prompt/ prompt.go schema.go testdata/*.golden # pure + ├── llm/ llm.go openai.go stub.go + ├── budget/ budget.go # concurrency + spend cap + ├── queue/ queue.go franz.go fake.go + ├── auth/ auth.go forwarder.go cache.go + ├── ingest/ ingest.go validate.go ratelimit.go + ├── api/ api.go + ├── analyzer/ analyzer.go + └── maint/ maint.go +``` + +**Responsibility boundaries.** `model` has no dependencies and is imported by everything. `fingerprint`, `gate` and `prompt` are pure — no network, no disk, no clock beyond an injected `now` — so they carry the correctness and cost logic in the most testable form available. `store`, `queue`, `llm` and `auth` are interfaces with a real and a fake implementation each, which is what lets `analyzer` be tested end-to-end with no container running. + +--- + +## Task 1: Provision the dev machine, scaffold the repository, CI and lint + +**Files:** +- Create: `LICENSE`, `go.mod`, `Makefile`, `.golangci.yml`, `.github/workflows/ci.yml`, `renovate.json`, `README.md`, `.gitignore` +- Create: `hack/check-license-headers.sh` +- Create: `internal/version/version.go` +- Test: `internal/version/version_test.go` + +**Interfaces:** +- Consumes: nothing. +- Produces: `version.Version` (string var), `make test`, `make lint`, `make build`, `make license-check`. Every later task runs `make test`. + +- [ ] **Step 1: Install the toolchain on the dev machine** + +All five tools are missing. Versions are pinned rather than taken from `dnf`, because Rocky's `golang` package floats and this project needs a known 1.23. + +Run on `root@rl1.leader.default.gs.nethserver.net`: + +```bash +dnf install -y git make tar gzip + +# Go 1.23 from upstream +curl -fsSL https://go.dev/dl/go1.23.6.linux-amd64.tar.gz -o /tmp/go.tgz +rm -rf /usr/local/go && tar -C /usr/local -xzf /tmp/go.tgz && rm -f /tmp/go.tgz + +# gh CLI +curl -fsSL https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_amd64.tar.gz \ + -o /tmp/gh.tgz +tar -C /tmp -xzf /tmp/gh.tgz +install -m0755 /tmp/gh_2.63.2_linux_amd64/bin/gh /usr/local/bin/gh +rm -rf /tmp/gh.tgz /tmp/gh_2.63.2_linux_amd64 + +cat >/etc/profile.d/go.sh <<'EOF' +export PATH=$PATH:/usr/local/go/bin:/root/go/bin +EOF +. /etc/profile.d/go.sh + +# golangci-lint, matching the version CI uses +curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh \ + | sh -s -- -b /root/go/bin v1.62.2 +``` + +- [ ] **Step 2: Verify the toolchain** + +Run: + +```bash +. /etc/profile.d/go.sh +go version && git --version && make --version | head -1 \ + && gh --version | head -1 && golangci-lint --version && podman --version +``` + +Expected: `go version go1.23.6 linux/amd64`, and a version line from each of the other five. If `go version` reports anything below 1.23, stop — later tasks use `for range int` and structured `log/slog` handling that need it. + +- [ ] **Step 3: Authenticate `gh` (operator step)** + +`gh auth login` is interactive and cannot be scripted here. The operator runs it on the dev machine and selects HTTPS + a token with `repo` and `workflow` scopes. + +Verify afterwards: + +```bash +gh auth status && gh repo view nethesis/nethesis-insights --json isEmpty +``` + +Expected: authenticated, and `{"isEmpty":true}`. + +- [ ] **Step 4: Clone the empty repository and create the working branch** + +The repository already exists and is empty — clone it rather than `git init`, so the remote is correct from the first commit. + +```bash +cd /root +git clone https://github.com/nethesis/nethesis-insights.git +cd nethesis-insights +git config user.name "Giacomo Sanchietti" +git config user.email "giacomo.sanchietti@nethesis.it" +git checkout -b feat/server-scaffold +go mod init github.com/nethesis/nethesis-insights +mkdir -p cmd/insightsd internal/version hack +``` + +Cloning an empty repository warns `you appear to have cloned an empty repository` and leaves no branch checked out; `git checkout -b` is what establishes the first one. The default branch becomes `main` on first push (Step 12). + +- [ ] **Step 5: Add the GPLv3 license and the header checker** + +Fetch the canonical text rather than hand-writing it: + +```bash +curl -fsSL https://www.gnu.org/licenses/gpl-3.0.txt -o LICENSE +``` + +Create `hack/check-license-headers.sh`: + +```bash +#!/usr/bin/env bash +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +# Fails if any tracked source file is missing its SPDX identifier. + +set -euo pipefail + +missing=0 +while IFS= read -r f; do + case "$f" in + LICENSE|*.md|*.json|*.golden|go.sum|go.mod|.gitignore) continue ;; + esac + if ! grep -q 'SPDX-License-Identifier: GPL-3.0-or-later' "$f"; then + echo "missing SPDX header: $f" >&2 + missing=1 + fi +done < <(git ls-files) + +if [ "$missing" -ne 0 ]; then + echo "run: add the GPL-3.0-or-later header to the files above" >&2 + exit 1 +fi +echo "license headers OK" +``` + +```bash +chmod +x hack/check-license-headers.sh +``` + +Create `.gitignore`: + +``` +bin/ +*.db +*.db-shm +*.db-wal +coverage.out +``` + +- [ ] **Step 6: Write the failing test** + +Create `internal/version/version_test.go` (with the Go license header from Global Constraints): + +```go +package version + +import "testing" + +func TestVersionIsSet(t *testing.T) { + if Version == "" { + t.Fatal("Version must not be empty") + } +} +``` + +- [ ] **Step 7: Run it to make sure it fails** + +Run: `go test ./internal/version/ -v` +Expected: FAIL — `undefined: Version` + +- [ ] **Step 8: Write the minimal implementation** + +Create `internal/version/version.go`: + +```go +// Package version records the build identity of the server. +package version + +// Version is overridden at build time with -ldflags "-X .../version.Version=x.y.z". +var Version = "0.0.0-dev" +``` + +- [ ] **Step 9: Run the tests and make sure they pass** + +Run: `go test ./internal/version/ -v` +Expected: PASS + +- [ ] **Step 10: Add the Makefile** + +Create `Makefile`: + +```makefile +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +BINARY := insightsd +PKG := github.com/nethesis/nethesis-insights +VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo 0.0.0-dev) + +.PHONY: build test lint tidy license-check check + +build: + CGO_ENABLED=0 go build -trimpath \ + -ldflags "-s -w -X $(PKG)/internal/version.Version=$(VERSION)" \ + -o bin/$(BINARY) ./cmd/insightsd + +test: + go test ./... -race -count=1 + +lint: + golangci-lint run + +license-check: + ./hack/check-license-headers.sh + +check: license-check lint test + +tidy: + go mod tidy +``` + +`CGO_ENABLED=0` is required: the binary must be static so it can be copied into the Redpanda base image in Task 17 without dragging a libc dependency. + +- [ ] **Step 11: Add the lint configuration** + +Create `.golangci.yml`: + +```yaml +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +version: "2" +linters: + enable: + - errcheck + - govet + - staticcheck + - ineffassign + - unused + - bodyclose + - sqlclosecheck + - rowserrcheck + - gosec +linters-settings: + gosec: + excludes: + - G404 # math/rand is never used for security here +``` + +`bodyclose`, `sqlclosecheck` and `rowserrcheck` are the ones that matter for this codebase: it is mostly HTTP clients and database rows, and leaking either is the most likely real bug. + +- [ ] **Step 12: Add CI** + +Create `.github/workflows/ci.yml`: + +```yaml +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +name: ci +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: test + POSTGRES_DB: insights_test + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + env: + TEST_POSTGRES_DSN: postgres://postgres:test@localhost:5432/insights_test?sslmode=disable + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.23" + - run: make license-check + - run: make test + - uses: golangci/golangci-lint-action@v6 + with: + version: v1.62 + - run: make build +``` + +`make license-check` runs **first** and before the test suite, so a missing GPL header fails fast rather than after several minutes of tests. + +The Postgres service exists so Task 3's dual-dialect migration test actually runs in CI. Without it, the portability rules in Global Constraints are enforced by memory rather than by the build. + +- [ ] **Step 13: Add Renovate and README** + +Create `renovate.json`: + +```json +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["github>NethServer/.github"] +} +``` + +Create `README.md`: + +```markdown +# nethesis-insights + +Central anomaly analysis for NethServer fleets. Receives deduplicated log +bundles from nodes, gates them against novelty and deviation, calls an LLM only +when warranted, and stores fingerprinted findings that do not repeat. + +Design: `docs/superpowers/specs/2026-08-05-nethesis-insights-design.md` in the +`ns8-loki` repository. + +## Development + + make check # license headers, lint, tests + make test # unit + integration, race detector on + make build # static binary into bin/insightsd + +## License + +GPL-3.0-or-later. See `LICENSE`. +``` + +- [ ] **Step 14: Add the placeholder entrypoint and verify the whole toolchain** + +`cmd/insightsd` is not wired until Task 16, but `make build` must succeed from Task 1 onward so CI is meaningful throughout. + +Create `cmd/insightsd/main.go` (with the Go license header): + +```go +// Command insightsd is the Nethesis Insights server. Wiring lands in Task 16. +package main + +func main() {} +``` + +Run: `make check && make build` +Expected: `license headers OK`, lint clean, tests PASS, `bin/insightsd` produced. + +- [ ] **Step 15: Commit and make the repository's first push** + +This is the first commit in an empty repository, so it also establishes `main`. + +```bash +git add . +git commit -m "chore: scaffold module, license, CI, lint and build" + +# Establish main from this branch, then push the working branch. +git branch -M feat/server-scaffold +git push -u origin feat/server-scaffold +gh api -X PATCH repos/nethesis/nethesis-insights -f default_branch=main 2>/dev/null || true +``` + +`git add .` is safe here and only here: the repository was empty, `.gitignore` is already in place from Step 5, and nothing untracked exists that should not be committed. Every later task stages explicit paths. + +Note the default branch cannot be set to `main` until a `main` ref exists. Create it from the scaffold once the branch is pushed: + +```bash +git push origin feat/server-scaffold:refs/heads/main +gh api -X PATCH repos/nethesis/nethesis-insights -f default_branch=main +gh repo view nethesis/nethesis-insights --json defaultBranchRef +``` + +Expected: `{"defaultBranchRef":{"name":"main"}}`. Subsequent work continues on `feat/server-scaffold`, and Task 19 opens the draft PR against `main`. + +--- + +## Task 2: `internal/model` — shared types + +**Files:** +- Create: `internal/model/bundle.go`, `internal/model/finding.go`, `internal/model/severity.go` +- Test: `internal/model/bundle_test.go`, `internal/model/severity_test.go` + +**Interfaces:** +- Consumes: nothing. +- Produces: `model.Bundle`, `model.Window`, `model.DigestEntry`, `model.Template`, `model.Budget`, `model.TruncatedModule`, `model.Finding`, `model.SeverityRank(string) int`, `model.SortFindings([]Finding)`. Every later task imports this package. + +- [ ] **Step 1: Write the failing test for bundle decoding** + +Create `internal/model/bundle_test.go`: + +```go +package model + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" +) + +const sampleBundle = `{ + "schema_version": 1, + "system_id": "abc123", + "collector_version": "2.0.0", + "masking_version": 1, + "window": { "start": 1754380800000, "end": 1754381700000 }, + "digest": [ + { "module_id": "traefik1", "priority": 3, + "observed": 42, "expected": 3.2, "ratio": 13.1 } + ], + "templates": [ + { "template": "<3> [n1:traefik1:traefik] connection refused to :", + "count": 37, "module_id": "traefik1", "priority": 3, + "category": "security", + "first_seen": 1754380811000, "last_seen": 1754381690000, + "samples": ["<3> raw line"] } + ], + "budget": { + "max_lines": 500, "lines_seen": 4210, "lines_kept": 500, + "truncated_modules": [ { "module_id": "traefik1", "dropped": 3200 } ] + } +}` + +func TestBundleDecodesEveryProtocolField(t *testing.T) { + var b Bundle + require.NoError(t, json.Unmarshal([]byte(sampleBundle), &b)) + + require.Equal(t, 1, b.SchemaVersion) + require.Equal(t, "abc123", b.SystemID) + require.Equal(t, "2.0.0", b.CollectorVersion) + require.Equal(t, 1, b.MaskingVersion) + require.Equal(t, int64(1754380800000), b.Window.Start) + require.Equal(t, int64(1754381700000), b.Window.End) + + require.Len(t, b.Digest, 1) + require.Equal(t, "traefik1", b.Digest[0].ModuleID) + require.Equal(t, 3, b.Digest[0].Priority) + require.Equal(t, int64(42), b.Digest[0].Observed) + require.NotNil(t, b.Digest[0].Expected) + require.InDelta(t, 3.2, *b.Digest[0].Expected, 0.001) + + require.Len(t, b.Templates, 1) + require.Equal(t, "security", b.Templates[0].Category) + require.Equal(t, int64(37), b.Templates[0].Count) + require.Equal(t, []string{"<3> raw line"}, b.Templates[0].Samples) + + require.Equal(t, 500, b.Budget.MaxLines) + require.Len(t, b.Budget.TruncatedModules, 1) + require.Equal(t, int64(3200), b.Budget.TruncatedModules[0].Dropped) +} + +func TestExpectedIsNilWhenEdgeDegraded(t *testing.T) { + // The edge omits `expected` when its Loki metric query fails. + var b Bundle + require.NoError(t, json.Unmarshal([]byte( + `{"digest":[{"module_id":"m","priority":3,"observed":9}]}`), &b)) + require.Nil(t, b.Digest[0].Expected) + require.Equal(t, int64(9), b.Digest[0].Observed) +} +``` + +`Expected` must be a pointer, not a `float64`. When the edge's Loki metric query fails it omits the field entirely, and a non-pointer would decode to `0.0`, which is indistinguishable from a genuine zero rate and would make the gate divide by zero. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/model/ -run TestBundle -v` +Expected: FAIL — `undefined: Bundle` + +- [ ] **Step 3: Write the minimal implementation** + +Create `internal/model/bundle.go`: + +```go +// Package model holds the types shared across the server. It has no +// dependencies on any other internal package. +package model + +// SchemaVersion is the only bundle schema version this server accepts. +const SchemaVersion = 1 + +// Window is the closed time range a bundle covers, in unix milliseconds. +type Window struct { + Start int64 `json:"start"` + End int64 `json:"end"` +} + +// DigestEntry is one (module, priority) count for the window. Expected and +// Ratio are pointers because the edge omits them when its metric query fails. +type DigestEntry struct { + ModuleID string `json:"module_id"` + Priority int `json:"priority"` + Observed int64 `json:"observed"` + Expected *float64 `json:"expected,omitempty"` + Ratio *float64 `json:"ratio,omitempty"` +} + +// Template is one masked log line pattern plus how often it occurred. +// Samples are representative raw lines and are never persisted (spec §10). +type Template struct { + Template string `json:"template"` + Count int64 `json:"count"` + ModuleID string `json:"module_id"` + Priority int `json:"priority"` + Category string `json:"category,omitempty"` + FirstSeen int64 `json:"first_seen"` + LastSeen int64 `json:"last_seen"` + Samples []string `json:"samples,omitempty"` +} + +// TruncatedModule reports lines the edge dropped for one module. +type TruncatedModule struct { + ModuleID string `json:"module_id"` + Dropped int64 `json:"dropped"` +} + +// Budget reports what the edge's line cap did to this window. +type Budget struct { + MaxLines int `json:"max_lines"` + LinesSeen int64 `json:"lines_seen"` + LinesKept int64 `json:"lines_kept"` + TruncatedModules []TruncatedModule `json:"truncated_modules,omitempty"` +} + +// Bundle is one 15-minute analysis payload from one node. +type Bundle struct { + SchemaVersion int `json:"schema_version"` + SystemID string `json:"system_id"` + CollectorVersion string `json:"collector_version"` + MaskingVersion int `json:"masking_version"` + Window Window `json:"window"` + Digest []DigestEntry `json:"digest"` + Templates []Template `json:"templates"` + Budget Budget `json:"budget"` +} + +// CategoryOf returns the edge-assigned category for a template text, or "". +func (b Bundle) CategoryOf(template string) string { + for _, t := range b.Templates { + if t.Template == template { + return t.Category + } + } + return "" +} +``` + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/model/ -run TestBundle -v && go test ./internal/model/ -run TestExpected -v` +Expected: PASS + +- [ ] **Step 5: Write the failing test for findings and severity ordering** + +Create `internal/model/severity_test.go`: + +```go +package model + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSeverityRankIsHighestFirst(t *testing.T) { + require.Less(t, SeverityRank("critical"), SeverityRank("high")) + require.Less(t, SeverityRank("high"), SeverityRank("medium")) + require.Less(t, SeverityRank("medium"), SeverityRank("low")) +} + +func TestUnknownSeveritySortsLast(t *testing.T) { + require.Greater(t, SeverityRank("banana"), SeverityRank("low")) +} + +func TestSortFindingsSeverityThenLastSeen(t *testing.T) { + in := []Finding{ + {Severity: "low", LastSeen: 100}, + {Severity: "critical", LastSeen: 50}, + {Severity: "high", LastSeen: 10}, + {Severity: "critical", LastSeen: 90}, + } + SortFindings(in) + require.Equal(t, []string{"critical", "critical", "high", "low"}, + []string{in[0].Severity, in[1].Severity, in[2].Severity, in[3].Severity}) + // Within equal severity, most recent first. + require.Equal(t, int64(90), in[0].LastSeen) + require.Equal(t, int64(50), in[1].LastSeen) +} + +func TestValidSeverity(t *testing.T) { + require.True(t, ValidSeverity("critical")) + require.False(t, ValidSeverity("CRITICAL")) + require.False(t, ValidSeverity("")) +} +``` + +- [ ] **Step 6: Run it to make sure it fails** + +Run: `go test ./internal/model/ -run TestSeverity -v` +Expected: FAIL — `undefined: SeverityRank` + +- [ ] **Step 7: Write the minimal implementation** + +Create `internal/model/severity.go`: + +```go +package model + +import "sort" + +// Severities are ordered highest-first; index is the sort rank. +var Severities = []string{"critical", "high", "medium", "low"} + +// Assessments are the permitted window-level verdicts. +var Assessments = []string{"nominal", "degraded", "incident"} + +// Finding statuses. +const ( + StatusOpen = "open" + StatusStale = "stale" +) + +// SeverityRank returns the sort rank of a severity, lowest number being most +// severe. Unknown severities sort after every known one. +func SeverityRank(s string) int { + for i, known := range Severities { + if known == s { + return i + } + } + return len(Severities) +} + +// ValidSeverity reports whether s is one of the permitted severities. +func ValidSeverity(s string) bool { + return SeverityRank(s) < len(Severities) +} + +// ValidAssessment reports whether s is one of the permitted assessments. +func ValidAssessment(s string) bool { + for _, known := range Assessments { + if known == s { + return true + } + } + return false +} + +// SortFindings orders findings severity-descending, then last_seen descending. +func SortFindings(f []Finding) { + sort.SliceStable(f, func(i, j int) bool { + ri, rj := SeverityRank(f[i].Severity), SeverityRank(f[j].Severity) + if ri != rj { + return ri < rj + } + return f[i].LastSeen > f[j].LastSeen + }) +} +``` + +Create `internal/model/finding.go`: + +```go +package model + +// Finding is one insight about one system. Fingerprint is computed server-side +// from the cited evidence and is the dedup key; the model never supplies it. +type Finding struct { + ID string `json:"id"` + SystemID string `json:"system_id"` + Fingerprint string `json:"fingerprint"` + Severity string `json:"severity"` + Title string `json:"title"` + Summary string `json:"summary"` + SuggestedAction string `json:"suggested_action"` + Modules []string `json:"modules"` + Evidence []string `json:"evidence"` + Status string `json:"status"` + OccurrenceCount int `json:"occurrence_count"` + FirstSeen int64 `json:"first_seen"` + LastSeen int64 `json:"last_seen"` + ReopenedAt *int64 `json:"reopened_at,omitempty"` + LLMModel string `json:"llm_model"` + PromptVersion string `json:"prompt_version"` +} +``` + +- [ ] **Step 8: Run the whole package and make sure it passes** + +Run: `go test ./internal/model/ -v` +Expected: PASS, all tests + +- [ ] **Step 9: Commit** + +```bash +go get github.com/stretchr/testify@latest +go mod tidy +git add internal/model go.mod go.sum +git commit -m "feat(model): bundle protocol, finding and severity ordering types" +``` + +--- + +## Task 3: `internal/store` — schema, migrations, systems, templates, baselines + +**Files:** +- Create: `internal/store/store.go`, `internal/store/sqlite.go`, `internal/store/postgres.go`, `internal/store/systems.go` +- Create: `internal/store/migrations/0001_init.up.sql`, `internal/store/migrations/0001_init.down.sql` +- Test: `internal/store/store_test.go`, `internal/store/migrate_test.go` + +**Interfaces:** +- Consumes: `model.Bundle`, `model.Template`, `model.DigestEntry` (Task 2). +- Produces: + - `store.Store` interface (extended in Task 4) + - `store.Open(driver, dsn string) (Store, error)` — driver is `"sqlite"` or `"postgres"` + - `store.BaselineKey{ModuleID string; Priority int}` + - `store.System{SystemID, TenantID, CollectorVersion string; FirstSeen, LastSeen int64}` + - `Store.Migrate(ctx) error` + - `Store.UpsertSystem(ctx, System) error` + - `Store.KnownTemplates(ctx, systemID string) (map[string]bool, error)` + - `Store.UpsertTemplates(ctx, systemID string, ts []model.Template, now int64) error` + - `Store.Baselines(ctx, systemID string) (map[BaselineKey]float64, error)` + - `Store.UpsertBaselines(ctx, systemID string, d []model.DigestEntry, alpha float64) error` + - `Store.Close() error` + +- [ ] **Step 1: Write the failing migration test** + +Create `internal/store/migrate_test.go`: + +```go +package store + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// openSQLite returns a Store backed by a throwaway file. A file, not +// :memory:, because WAL mode and busy_timeout only mean anything on a file. +func openSQLite(t *testing.T) Store { + t.Helper() + dsn := filepath.Join(t.TempDir(), "test.db") + s, err := Open("sqlite", dsn) + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + require.NoError(t, s.Migrate(context.Background())) + return s +} + +func TestMigrateSQLiteIsIdempotent(t *testing.T) { + s := openSQLite(t) + // Migrate again on an already-migrated database. + require.NoError(t, s.Migrate(context.Background())) +} + +// TestMigratePostgres enforces the portability rules in Global Constraints. +// It is the reason CI runs a Postgres service. +func TestMigratePostgres(t *testing.T) { + dsn := os.Getenv("TEST_POSTGRES_DSN") + if dsn == "" { + t.Skip("TEST_POSTGRES_DSN not set") + } + s, err := Open("postgres", dsn) + require.NoError(t, err) + defer func() { _ = s.Close() }() + require.NoError(t, s.Migrate(context.Background())) + require.NoError(t, s.Migrate(context.Background())) +} +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/store/ -run TestMigrate -v` +Expected: FAIL — `undefined: Open` + +- [ ] **Step 3: Write the migration SQL** + +Create `internal/store/migrations/0001_init.up.sql`: + +```sql +CREATE TABLE IF NOT EXISTS systems ( + system_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + collector_version TEXT NOT NULL DEFAULT '', + first_seen BIGINT NOT NULL, + last_seen BIGINT NOT NULL +); + +CREATE TABLE IF NOT EXISTS system_templates ( + system_id TEXT NOT NULL, + template TEXT NOT NULL, + module_id TEXT NOT NULL DEFAULT '', + priority INT NOT NULL DEFAULT 0, + category TEXT NOT NULL DEFAULT '', + first_seen BIGINT NOT NULL, + last_seen BIGINT NOT NULL, + total_count BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (system_id, template) +); +CREATE INDEX IF NOT EXISTS idx_templates_last_seen + ON system_templates (last_seen); + +CREATE TABLE IF NOT EXISTS module_baselines ( + system_id TEXT NOT NULL, + module_id TEXT NOT NULL, + priority INT NOT NULL, + ewma_rate DOUBLE PRECISION NOT NULL, + updated_at BIGINT NOT NULL, + PRIMARY KEY (system_id, module_id, priority) +); + +CREATE TABLE IF NOT EXISTS findings ( + id TEXT PRIMARY KEY, + system_id TEXT NOT NULL, + fingerprint TEXT NOT NULL, + severity TEXT NOT NULL, + title TEXT NOT NULL, + summary TEXT NOT NULL, + suggested_action TEXT NOT NULL DEFAULT '', + modules TEXT NOT NULL DEFAULT '[]', + evidence TEXT NOT NULL DEFAULT '[]', + status TEXT NOT NULL, + occurrence_count INT NOT NULL DEFAULT 1, + first_seen BIGINT NOT NULL, + last_seen BIGINT NOT NULL, + reopened_at BIGINT, + llm_model TEXT NOT NULL DEFAULT '', + prompt_version TEXT NOT NULL DEFAULT '' +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_findings_fingerprint + ON findings (system_id, fingerprint); +CREATE INDEX IF NOT EXISTS idx_findings_lookup + ON findings (system_id, status, last_seen); + +CREATE TABLE IF NOT EXISTS analyses ( + id TEXT PRIMARY KEY, + system_id TEXT NOT NULL, + window_start BIGINT NOT NULL, + window_end BIGINT NOT NULL, + gated INT NOT NULL DEFAULT 0, + gate_reasons TEXT NOT NULL DEFAULT '[]', + llm_called INT NOT NULL DEFAULT 0, + input_tokens INT NOT NULL DEFAULT 0, + output_tokens INT NOT NULL DEFAULT 0, + cost_micros BIGINT NOT NULL DEFAULT 0, + model TEXT NOT NULL DEFAULT '', + duration_ms INT NOT NULL DEFAULT 0, + error TEXT NOT NULL DEFAULT '', + created_at BIGINT NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS idx_analyses_window + ON analyses (system_id, window_start); +CREATE INDEX IF NOT EXISTS idx_analyses_created + ON analyses (created_at); +``` + +`BIGINT`, `INT`, `DOUBLE PRECISION` and `TEXT` are the four types that mean the same thing in both dialects. Booleans are `INT` because SQLite has no boolean type — a `BOOLEAN` column would work in SQLite by aliasing but produce a genuine type mismatch when `bun` scans a Postgres `boolean` into an `int`. + +Create `internal/store/migrations/0001_init.down.sql`: + +```sql +DROP TABLE IF EXISTS analyses; +DROP TABLE IF EXISTS findings; +DROP TABLE IF EXISTS module_baselines; +DROP TABLE IF EXISTS system_templates; +DROP TABLE IF EXISTS systems; +``` + +- [ ] **Step 4: Write the store interface and the SQLite implementation** + +Create `internal/store/store.go`: + +```go +// Package store owns the database schema and every query against it. No SQL +// exists outside this package. +package store + +import ( + "context" + "embed" + "errors" + "fmt" + + "github.com/nethesis/nethesis-insights/internal/model" +) + +//go:embed migrations/*.sql +var migrationsFS embed.FS + +// BaselineKey identifies one rate baseline. +type BaselineKey struct { + ModuleID string + Priority int +} + +// System is a registered node. +type System struct { + SystemID string + TenantID string + CollectorVersion string + FirstSeen int64 + LastSeen int64 +} + +// Store is the whole persistence surface. Implementations must be safe for +// concurrent use. +type Store interface { + Migrate(ctx context.Context) error + Close() error + + UpsertSystem(ctx context.Context, s System) error + KnownTemplates(ctx context.Context, systemID string) (map[string]bool, error) + UpsertTemplates(ctx context.Context, systemID string, ts []model.Template, now int64) error + Baselines(ctx context.Context, systemID string) (map[BaselineKey]float64, error) + UpsertBaselines(ctx context.Context, systemID string, d []model.DigestEntry, alpha float64) error +} + +// ErrUnknownDriver is returned by Open for an unsupported driver name. +var ErrUnknownDriver = errors.New("store: unknown driver") + +// Open returns a Store for driver "sqlite" or "postgres". +func Open(driver, dsn string) (Store, error) { + switch driver { + case "sqlite": + return openSQLite(dsn) + case "postgres": + return openPostgres(dsn) + default: + return nil, fmt.Errorf("%w: %q", ErrUnknownDriver, driver) + } +} +``` + +Create `internal/store/sqlite.go`: + +```go +package store + +import ( + "context" + "database/sql" + "fmt" + "sync" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + _ "modernc.org/sqlite" // CGO-free driver +) + +type sqliteStore struct { + db *bun.DB + // writeMu serializes writes. SQLite permits one writer at a time; taking + // a mutex gives the same guarantee as a dedicated writer goroutine with + // no channel lifecycle to leak (see plan Deviation 3). + writeMu sync.Mutex + dsn string +} + +func openSQLite(dsn string) (Store, error) { + // WAL lets the read API run concurrently with the analyzer's writes; + // busy_timeout absorbs the brief contention that remains. + conn := fmt.Sprintf("file:%s?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=foreign_keys(1)", dsn) + sqldb, err := sql.Open("sqlite", conn) + if err != nil { + return nil, fmt.Errorf("store: open sqlite: %w", err) + } + // One connection removes any chance of "database is locked" from + // concurrent writers inside this process. + sqldb.SetMaxOpenConns(1) + return &sqliteStore{db: bun.NewDB(sqldb, sqlitedialect.New()), dsn: dsn}, nil +} + +func (s *sqliteStore) Migrate(ctx context.Context) error { + src, err := iofs.New(migrationsFS, "migrations") + if err != nil { + return fmt.Errorf("store: migration source: %w", err) + } + drv, err := sqlite.WithInstance(s.db.DB, &sqlite.Config{}) + if err != nil { + return fmt.Errorf("store: migration driver: %w", err) + } + m, err := migrate.NewWithInstance("iofs", src, "sqlite", drv) + if err != nil { + return fmt.Errorf("store: migrator: %w", err) + } + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + return fmt.Errorf("store: migrate: %w", err) + } + return nil +} + +func (s *sqliteStore) Close() error { return s.db.Close() } + +func (s *sqliteStore) bun() *bun.DB { return s.db } +func (s *sqliteStore) lock() *sync.Mutex { return &s.writeMu } +``` + +Create `internal/store/postgres.go`: + +```go +package store + +import ( + "context" + "database/sql" + "fmt" + "sync" + + "github.com/golang-migrate/migrate/v4" + migratepg "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/golang-migrate/migrate/v4/source/iofs" + _ "github.com/jackc/pgx/v5/stdlib" // database/sql driver + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/pgdialect" +) + +type pgStore struct { + db *bun.DB + // Postgres handles concurrent writers; the mutex is never contended and + // exists only so pgStore satisfies the same internal helper interface. + writeMu sync.Mutex +} + +func openPostgres(dsn string) (Store, error) { + sqldb, err := sql.Open("pgx", dsn) + if err != nil { + return nil, fmt.Errorf("store: open postgres: %w", err) + } + return &pgStore{db: bun.NewDB(sqldb, pgdialect.New())}, nil +} + +func (s *pgStore) Migrate(ctx context.Context) error { + src, err := iofs.New(migrationsFS, "migrations") + if err != nil { + return fmt.Errorf("store: migration source: %w", err) + } + drv, err := migratepg.WithInstance(s.db.DB, &migratepg.Config{}) + if err != nil { + return fmt.Errorf("store: migration driver: %w", err) + } + m, err := migrate.NewWithInstance("iofs", src, "pgx5", drv) + if err != nil { + return fmt.Errorf("store: migrator: %w", err) + } + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + return fmt.Errorf("store: migrate: %w", err) + } + return nil +} + +func (s *pgStore) Close() error { return s.db.Close() } + +func (s *pgStore) bun() *bun.DB { return s.db } +func (s *pgStore) lock() *sync.Mutex { return &s.writeMu } +``` + +- [ ] **Step 5: Run the migration tests** + +```bash +go get github.com/uptrace/bun github.com/uptrace/bun/dialect/sqlitedialect \ + github.com/uptrace/bun/dialect/pgdialect \ + github.com/golang-migrate/migrate/v4 github.com/jackc/pgx/v5 \ + modernc.org/sqlite +go mod tidy +go test ./internal/store/ -run TestMigrate -v +``` + +Expected: `TestMigrateSQLiteIsIdempotent` PASS, `TestMigratePostgres` SKIP. + +Then prove the Postgres path on the dev machine too, rather than only in CI. Port `55432` avoids colliding with anything the shared cluster runs: + +```bash +podman run -d --name insights-pg-test \ + -e POSTGRES_PASSWORD=test -e POSTGRES_DB=insights_test \ + -p 127.0.0.1:55432:5432 postgres:16-alpine + +until podman exec insights-pg-test pg_isready -q; do sleep 1; done + +TEST_POSTGRES_DSN='postgres://postgres:test@127.0.0.1:55432/insights_test?sslmode=disable' \ + go test ./internal/store/ -run TestMigratePostgres -v + +podman rm -f insights-pg-test +``` + +Expected: PASS. A failure here means one of the portability rules in Global Constraints was broken — most likely a type or an `ON CONFLICT` form that only SQLite accepts. + +- [ ] **Step 6: Write the failing tests for systems, templates and baselines** + +Create `internal/store/store_test.go`: + +```go +package store + +import ( + "context" + "testing" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/stretchr/testify/require" +) + +func TestUpsertSystemPreservesFirstSeen(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + require.NoError(t, s.UpsertSystem(ctx, System{ + SystemID: "sys1", TenantID: "t1", CollectorVersion: "2.0.0", + FirstSeen: 100, LastSeen: 100, + })) + require.NoError(t, s.UpsertSystem(ctx, System{ + SystemID: "sys1", TenantID: "t1", CollectorVersion: "2.1.0", + FirstSeen: 500, LastSeen: 500, + })) + + // first_seen must not move on a repeat visit; last_seen and version must. + // Scanned into a tagged local struct rather than store.System, which + // carries no bun tags — it is a plain API type, not a table mapping. + var got struct { + FirstSeen int64 `bun:"first_seen"` + LastSeen int64 `bun:"last_seen"` + CollectorVersion string `bun:"collector_version"` + } + require.NoError(t, s.(*sqliteStore).db.NewSelect(). + Table("systems"). + Column("first_seen", "last_seen", "collector_version"). + Where("system_id = ?", "sys1").Scan(ctx, &got)) + require.Equal(t, int64(100), got.FirstSeen) + require.Equal(t, int64(500), got.LastSeen) + require.Equal(t, "2.1.0", got.CollectorVersion) +} + +func TestKnownTemplatesIsEmptyForNewSystem(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + known, err := s.KnownTemplates(ctx, "sys1") + require.NoError(t, err) + require.Empty(t, known) +} + +func TestUpsertTemplatesAccumulatesCounts(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + ts := []model.Template{{ + Template: "conn refused to ", Count: 5, ModuleID: "m1", + Priority: 3, Category: "security", FirstSeen: 10, LastSeen: 20, + }} + require.NoError(t, s.UpsertTemplates(ctx, "sys1", ts, 100)) + + ts[0].Count = 7 + require.NoError(t, s.UpsertTemplates(ctx, "sys1", ts, 200)) + + known, err := s.KnownTemplates(ctx, "sys1") + require.NoError(t, err) + require.True(t, known["conn refused to "]) + + var row struct { + TotalCount int64 `bun:"total_count"` + FirstSeen int64 `bun:"first_seen"` + LastSeen int64 `bun:"last_seen"` + } + require.NoError(t, s.(*sqliteStore).db.NewSelect(). + Table("system_templates"). + Column("total_count", "first_seen", "last_seen"). + Where("system_id = ?", "sys1").Scan(ctx, &row)) + require.Equal(t, int64(12), row.TotalCount) + require.Equal(t, int64(100), row.FirstSeen) // unchanged + require.Equal(t, int64(200), row.LastSeen) // advanced +} + +func TestTemplatesAreScopedPerSystem(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + ts := []model.Template{{Template: "same text", Count: 1, ModuleID: "m"}} + require.NoError(t, s.UpsertTemplates(ctx, "sys1", ts, 100)) + + // The same template text on another system must still look novel there. + known, err := s.KnownTemplates(ctx, "sys2") + require.NoError(t, err) + require.False(t, known["same text"]) +} + +func TestUpsertTemplatesNeverStoresSamples(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + require.NoError(t, s.UpsertTemplates(ctx, "sys1", []model.Template{{ + Template: "masked ", Count: 1, ModuleID: "m", + Samples: []string{"SECRET raw line 10.0.0.4"}, + }}, 100)) + + // Spec §10: raw samples must never reach the database. Assert no column + // anywhere in the row contains the sample text. + rows, err := s.(*sqliteStore).db.QueryContext(ctx, + `SELECT * FROM system_templates`) + require.NoError(t, err) + defer func() { _ = rows.Close() }() + cols, err := rows.Columns() + require.NoError(t, err) + require.True(t, rows.Next()) + cells := make([]any, len(cols)) + for i := range cells { + var v any + cells[i] = &v + } + require.NoError(t, rows.Scan(cells...)) + require.NoError(t, rows.Err()) + for i, c := range cells { + v := *(c.(*any)) + if str, ok := v.(string); ok { + require.NotContains(t, str, "SECRET", "column %s leaked a sample", cols[i]) + } + } +} + +func TestBaselinesEWMA(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + d := []model.DigestEntry{{ModuleID: "m1", Priority: 3, Observed: 100}} + + // First observation seeds the baseline at the observed value. + require.NoError(t, s.UpsertBaselines(ctx, "sys1", d, 0.3)) + b, err := s.Baselines(ctx, "sys1") + require.NoError(t, err) + require.InDelta(t, 100.0, b[BaselineKey{"m1", 3}], 0.001) + + // Second observation blends: 0.3*200 + 0.7*100 = 130 + d[0].Observed = 200 + require.NoError(t, s.UpsertBaselines(ctx, "sys1", d, 0.3)) + b, err = s.Baselines(ctx, "sys1") + require.NoError(t, err) + require.InDelta(t, 130.0, b[BaselineKey{"m1", 3}], 0.001) +} +``` + +The samples test is worth its awkwardness: it is the only automated check that the §10 data-protection boundary holds, and it fails loudly if someone later adds a `samples` column for convenience. + +- [ ] **Step 7: Run them to make sure they fail** + +Run: `go test ./internal/store/ -run 'TestUpsert|TestKnown|TestBaselines|TestTemplates' -v` +Expected: FAIL — `UpsertSystem` etc. not implemented + +- [ ] **Step 8: Write the implementation** + +Create `internal/store/systems.go`: + +```go +package store + +import ( + "context" + "fmt" + "sync" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/uptrace/bun" +) + +// writer is satisfied by both sqliteStore and pgStore so the query bodies are +// written once. This is what makes the two dialects share one query codebase. +type writer interface { + bun() *bun.DB + lock() *sync.Mutex +} + +func upsertSystem(ctx context.Context, w writer, s System) error { + w.lock().Lock() + defer w.lock().Unlock() + _, err := w.bun().NewRaw(` + INSERT INTO systems + (system_id, tenant_id, collector_version, first_seen, last_seen) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (system_id) DO UPDATE SET + tenant_id = excluded.tenant_id, + collector_version = excluded.collector_version, + last_seen = excluded.last_seen`, + s.SystemID, s.TenantID, s.CollectorVersion, s.FirstSeen, s.LastSeen, + ).Exec(ctx) + if err != nil { + return fmt.Errorf("store: upsert system: %w", err) + } + return nil +} + +func knownTemplates(ctx context.Context, w writer, systemID string) (map[string]bool, error) { + var texts []string + err := w.bun().NewRaw( + `SELECT template FROM system_templates WHERE system_id = ?`, systemID, + ).Scan(ctx, &texts) + if err != nil { + return nil, fmt.Errorf("store: known templates: %w", err) + } + known := make(map[string]bool, len(texts)) + for _, t := range texts { + known[t] = true + } + return known, nil +} + +func upsertTemplates(ctx context.Context, w writer, systemID string, + ts []model.Template, now int64) error { + if len(ts) == 0 { + return nil + } + w.lock().Lock() + defer w.lock().Unlock() + tx, err := w.bun().BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("store: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + + for _, t := range ts { + // Samples are deliberately not referenced here (spec §10). + if _, err := tx.NewRaw(` + INSERT INTO system_templates + (system_id, template, module_id, priority, category, + first_seen, last_seen, total_count) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (system_id, template) DO UPDATE SET + last_seen = excluded.last_seen, + total_count = system_templates.total_count + excluded.total_count, + category = excluded.category`, + systemID, t.Template, t.ModuleID, t.Priority, t.Category, + now, now, t.Count, + ).Exec(ctx); err != nil { + return fmt.Errorf("store: upsert template: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("store: commit templates: %w", err) + } + return nil +} + +func baselines(ctx context.Context, w writer, systemID string) (map[BaselineKey]float64, error) { + var rows []struct { + ModuleID string `bun:"module_id"` + Priority int `bun:"priority"` + Rate float64 `bun:"ewma_rate"` + } + err := w.bun().NewRaw(` + SELECT module_id, priority, ewma_rate + FROM module_baselines WHERE system_id = ?`, systemID, + ).Scan(ctx, &rows) + if err != nil { + return nil, fmt.Errorf("store: baselines: %w", err) + } + out := make(map[BaselineKey]float64, len(rows)) + for _, r := range rows { + out[BaselineKey{r.ModuleID, r.Priority}] = r.Rate + } + return out, nil +} + +func upsertBaselines(ctx context.Context, w writer, systemID string, + d []model.DigestEntry, alpha float64) error { + if len(d) == 0 { + return nil + } + w.lock().Lock() + defer w.lock().Unlock() + tx, err := w.bun().BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("store: begin: %w", err) + } + defer func() { _ = tx.Rollback() }() + + for _, e := range d { + // First observation seeds at the observed value; later ones blend. + // Expressed in SQL so the read-modify-write is atomic per row. + if _, err := tx.NewRaw(` + INSERT INTO module_baselines + (system_id, module_id, priority, ewma_rate, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (system_id, module_id, priority) DO UPDATE SET + ewma_rate = (? * excluded.ewma_rate) + + ((1 - ?) * module_baselines.ewma_rate), + updated_at = excluded.updated_at`, + systemID, e.ModuleID, e.Priority, float64(e.Observed), e.Observed, + alpha, alpha, + ).Exec(ctx); err != nil { + return fmt.Errorf("store: upsert baseline: %w", err) + } + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("store: commit baselines: %w", err) + } + return nil +} +``` + +Now bind these to both implementations. Append to `internal/store/sqlite.go`: + +```go +func (s *sqliteStore) UpsertSystem(ctx context.Context, sys System) error { + return upsertSystem(ctx, s, sys) +} + +func (s *sqliteStore) KnownTemplates(ctx context.Context, systemID string) (map[string]bool, error) { + return knownTemplates(ctx, s, systemID) +} + +func (s *sqliteStore) UpsertTemplates(ctx context.Context, systemID string, + ts []model.Template, now int64) error { + return upsertTemplates(ctx, s, systemID, ts, now) +} + +func (s *sqliteStore) Baselines(ctx context.Context, systemID string) (map[BaselineKey]float64, error) { + return baselines(ctx, s, systemID) +} + +func (s *sqliteStore) UpsertBaselines(ctx context.Context, systemID string, + d []model.DigestEntry, alpha float64) error { + return upsertBaselines(ctx, s, systemID, d, alpha) +} +``` + +Add the identical five methods to `internal/store/postgres.go` with receiver `(s *pgStore)`, delegating to the same package-level functions. Add the required imports (`context`, `github.com/nethesis/nethesis-insights/internal/model`) to both files. + +- [ ] **Step 9: Run the tests and make sure they pass** + +Run: `go test ./internal/store/ -v` +Expected: PASS (Postgres test skipped locally) + +- [ ] **Step 10: Commit** + +```bash +git add internal/store go.mod go.sum +git commit -m "feat(store): schema, dual-dialect migrations, systems and baselines" +``` + +--- + +## Task 4: `internal/store` — findings, analyses ledger, pruning + +**Files:** +- Create: `internal/store/findings.go`, `internal/store/prune.go` +- Modify: `internal/store/store.go` (extend `Store`), `internal/store/sqlite.go`, `internal/store/postgres.go` (bind methods) +- Test: `internal/store/findings_test.go`, `internal/store/analyses_test.go`, `internal/store/prune_test.go` + +**Interfaces:** +- Consumes: `store.Store`, the `writer` helper interface, `openSQLite` test helper (Task 3); `model.Finding`, `model.SortFindings`, `model.StatusOpen`, `model.StatusStale` (Task 2). +- Produces, added to `Store`: + - `BeginAnalysis(ctx, systemID string, windowStart, windowEnd, now int64) (bool, error)` — `false` means duplicate window + - `FinalizeAnalysis(ctx, a Analysis) error` + - `SpendSince(ctx, sinceMs int64) (int64, error)` — summed `cost_micros` + - `OpenFindings(ctx, systemID string) ([]model.Finding, error)` + - `UpsertFinding(ctx, f model.Finding, now int64) (Outcome, error)` + - `MarkStale(ctx, systemID string, olderThan int64) (int, error)` + - `ListFindings(ctx, systemID string, since int64, status string) ([]model.Finding, error)` + - `PruneTemplates/PruneFindings/PruneAnalyses(ctx, olderThan int64) (int, error)` + - types `store.Analysis`, `store.Outcome` with `OutcomeInserted`, `OutcomeBumped`, `OutcomeReopened` + +- [ ] **Step 1: Write the failing finding-lifecycle tests** + +Create `internal/store/findings_test.go`: + +```go +package store + +import ( + "context" + "testing" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/stretchr/testify/require" +) + +func finding(fp, severity string, seen int64) model.Finding { + return model.Finding{ + SystemID: "sys1", Fingerprint: fp, Severity: severity, + Title: "t", Summary: "s", SuggestedAction: "a", + Modules: []string{"m1"}, Evidence: []string{"tmpl "}, + FirstSeen: seen, LastSeen: seen, + LLMModel: "gpt-4o-mini", PromptVersion: "v1", + } +} + +func TestNewFindingIsInserted(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + out, err := s.UpsertFinding(ctx, finding("fp1", "high", 100), 100) + require.NoError(t, err) + require.Equal(t, OutcomeInserted, out) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1) + require.Equal(t, model.StatusOpen, open[0].Status) + require.Equal(t, 1, open[0].OccurrenceCount) + require.NotEmpty(t, open[0].ID, "ID must be a generated ULID") + require.Equal(t, []string{"m1"}, open[0].Modules) + require.Equal(t, []string{"tmpl "}, open[0].Evidence) +} + +func TestRecurrenceBumpsInsteadOfInserting(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp1", "high", 100), 100) + require.NoError(t, err) + + out, err := s.UpsertFinding(ctx, finding("fp1", "high", 200), 200) + require.NoError(t, err) + require.Equal(t, OutcomeBumped, out) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1, "recurrence must never insert a second row") + require.Equal(t, 2, open[0].OccurrenceCount) + require.Equal(t, int64(100), open[0].FirstSeen, "first_seen must not move") + require.Equal(t, int64(200), open[0].LastSeen) + require.Nil(t, open[0].ReopenedAt, "a bump is not a reopen") +} + +func TestFingerprintIsScopedPerSystem(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + f := finding("fp1", "high", 100) + _, err := s.UpsertFinding(ctx, f, 100) + require.NoError(t, err) + + f.SystemID = "sys2" + out, err := s.UpsertFinding(ctx, f, 100) + require.NoError(t, err) + require.Equal(t, OutcomeInserted, out, + "the same fingerprint on another system is a different finding") +} + +func TestMarkStaleThenRecurrenceReopens(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp1", "high", 100), 100) + require.NoError(t, err) + + n, err := s.MarkStale(ctx, "sys1", 500) + require.NoError(t, err) + require.Equal(t, 1, n) + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Empty(t, open) + + out, err := s.UpsertFinding(ctx, finding("fp1", "high", 900), 900) + require.NoError(t, err) + require.Equal(t, OutcomeReopened, out) + + open, err = s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1) + require.NotNil(t, open[0].ReopenedAt) + require.Equal(t, int64(900), *open[0].ReopenedAt) +} + +func TestMarkStaleSparesRecentFindings(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp1", "high", 1000), 1000) + require.NoError(t, err) + n, err := s.MarkStale(ctx, "sys1", 500) + require.NoError(t, err) + require.Equal(t, 0, n, "last_seen newer than the cutoff must stay open") +} + +func TestListFindingsIsSeverityDescending(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + for _, f := range []model.Finding{ + finding("fp-low", "low", 400), + finding("fp-crit", "critical", 100), + finding("fp-med", "medium", 300), + } { + _, err := s.UpsertFinding(ctx, f, f.LastSeen) + require.NoError(t, err) + } + got, err := s.ListFindings(ctx, "sys1", 0, "") + require.NoError(t, err) + require.Equal(t, []string{"critical", "medium", "low"}, + []string{got[0].Severity, got[1].Severity, got[2].Severity}) +} + +func TestListFindingsFiltersBySinceAndStatus(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp1", "high", 100), 100) + require.NoError(t, err) + _, err = s.UpsertFinding(ctx, finding("fp2", "high", 900), 900) + require.NoError(t, err) + + got, err := s.ListFindings(ctx, "sys1", 500, "") + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "fp2", got[0].Fingerprint) + + _, err = s.MarkStale(ctx, "sys1", 500) + require.NoError(t, err) + got, err = s.ListFindings(ctx, "sys1", 0, model.StatusStale) + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, "fp1", got[0].Fingerprint) +} + +func TestSeverityEscalatesOnRecurrence(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp1", "medium", 100), 100) + require.NoError(t, err) + _, err = s.UpsertFinding(ctx, finding("fp1", "critical", 200), 200) + require.NoError(t, err) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Equal(t, "critical", open[0].Severity, + "an escalating condition must not stay pinned at its first severity") +} +``` + +That last test encodes a real judgement call: a recurring finding whose severity rises must escalate. Freezing severity at first sight would hide a `medium` turning into a `critical` behind a silent occurrence bump. + +- [ ] **Step 2: Write the failing ledger tests** + +Create `internal/store/analyses_test.go`: + +```go +package store + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestBeginAnalysisIsIdempotentPerWindow(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + fresh, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 2000) + require.NoError(t, err) + require.True(t, fresh) + + // An edge retry of the same window must be recognised, not reprocessed. + fresh, err = s.BeginAnalysis(ctx, "sys1", 1000, 1900, 2000) + require.NoError(t, err) + require.False(t, fresh) +} + +func TestBeginAnalysisSeparatesSystems(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 2000) + require.NoError(t, err) + fresh, err := s.BeginAnalysis(ctx, "sys2", 1000, 1900, 2000) + require.NoError(t, err) + require.True(t, fresh, "same window on another system is a new analysis") +} + +func TestFinalizeAnalysisRecordsTheLedger(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 2000) + require.NoError(t, err) + require.NoError(t, s.FinalizeAnalysis(ctx, Analysis{ + SystemID: "sys1", WindowStart: 1000, + Gated: false, GateReasons: []string{"new_templates=2"}, + LLMCalled: true, InputTokens: 12400, OutputTokens: 300, + CostMicros: 2040, Model: "gpt-4o-mini", DurationMs: 3100, + })) + + spend, err := s.SpendSince(ctx, 0) + require.NoError(t, err) + require.Equal(t, int64(2040), spend) +} + +func TestSpendSinceIgnoresOlderRows(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 1000) + require.NoError(t, err) + require.NoError(t, s.FinalizeAnalysis(ctx, Analysis{ + SystemID: "sys1", WindowStart: 1000, CostMicros: 500})) + _, err = s.BeginAnalysis(ctx, "sys1", 9000, 9900, 9000) + require.NoError(t, err) + require.NoError(t, s.FinalizeAnalysis(ctx, Analysis{ + SystemID: "sys1", WindowStart: 9000, CostMicros: 700})) + + spend, err := s.SpendSince(ctx, 5000) + require.NoError(t, err) + require.Equal(t, int64(700), spend, "only rows created after the cutoff count") +} + +func TestSpendSinceIsZeroOnEmptyLedger(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + spend, err := s.SpendSince(ctx, 0) + require.NoError(t, err) + require.Zero(t, spend, "SUM over no rows is NULL and must scan as 0") +} + +func TestGatedAnalysisCostsNothing(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 2000) + require.NoError(t, err) + require.NoError(t, s.FinalizeAnalysis(ctx, Analysis{ + SystemID: "sys1", WindowStart: 1000, + Gated: true, GateReasons: []string{}, LLMCalled: false})) + spend, err := s.SpendSince(ctx, 0) + require.NoError(t, err) + require.Zero(t, spend) +} +``` + +`TestSpendSinceIsZeroOnEmptyLedger` guards a specific trap: `SUM()` over zero rows returns `NULL` in both dialects, and scanning that into a plain `int64` errors. The implementation must scan into `sql.NullInt64`. This is the first query the spend cap runs on a fresh deployment, so getting it wrong breaks the cost ceiling on day one. + +- [ ] **Step 3: Run both files to make sure they fail** + +Run: `go test ./internal/store/ -run 'TestNewFinding|TestBeginAnalysis' -v` +Expected: FAIL — `UpsertFinding`, `BeginAnalysis` undefined + +- [ ] **Step 4: Extend the `Store` interface** + +In `internal/store/store.go`, add these lines inside the `Store` interface, after `UpsertBaselines`: + +```go + BeginAnalysis(ctx context.Context, systemID string, windowStart, windowEnd, now int64) (bool, error) + FinalizeAnalysis(ctx context.Context, a Analysis) error + SpendSince(ctx context.Context, sinceMs int64) (int64, error) + + OpenFindings(ctx context.Context, systemID string) ([]model.Finding, error) + UpsertFinding(ctx context.Context, f model.Finding, now int64) (Outcome, error) + MarkStale(ctx context.Context, systemID string, olderThan int64) (int, error) + ListFindings(ctx context.Context, systemID string, since int64, status string) ([]model.Finding, error) + + PruneTemplates(ctx context.Context, olderThan int64) (int, error) + PruneFindings(ctx context.Context, olderThan int64) (int, error) + PruneAnalyses(ctx context.Context, olderThan int64) (int, error) +``` + +Append the supporting types to the same file: + +```go +// Outcome reports what UpsertFinding did, so callers can tell an alert-worthy +// event from a silent recurrence. +type Outcome string + +const ( + OutcomeInserted Outcome = "inserted" + OutcomeBumped Outcome = "bumped" + OutcomeReopened Outcome = "reopened" +) + +// Analysis is one row of the cost and audit ledger. +type Analysis struct { + SystemID string + WindowStart int64 + Gated bool + GateReasons []string + LLMCalled bool + InputTokens int + OutputTokens int + CostMicros int64 + Model string + DurationMs int + Error string +} +``` + +- [ ] **Step 5: Write `internal/store/findings.go`** + +```go +package store + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/oklog/ulid/v2" +) + +func newID() string { return ulid.Make().String() } + +func boolToInt(b bool) int { + if b { + return 1 + } + return 0 +} + +func beginAnalysis(ctx context.Context, w writer, systemID string, + windowStart, windowEnd, now int64) (bool, error) { + w.lock().Lock() + defer w.lock().Unlock() + // DO NOTHING makes a duplicate window a zero-row result rather than an + // error, so an edge retry is ordinary data instead of an exception. + res, err := w.bun().NewRaw(` + INSERT INTO analyses (id, system_id, window_start, window_end, created_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (system_id, window_start) DO NOTHING`, + newID(), systemID, windowStart, windowEnd, now, + ).Exec(ctx) + if err != nil { + return false, fmt.Errorf("store: begin analysis: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return false, fmt.Errorf("store: begin analysis rows: %w", err) + } + return n > 0, nil +} + +func finalizeAnalysis(ctx context.Context, w writer, a Analysis) error { + reasons, err := json.Marshal(a.GateReasons) + if err != nil { + return fmt.Errorf("store: marshal gate reasons: %w", err) + } + w.lock().Lock() + defer w.lock().Unlock() + _, err = w.bun().NewRaw(` + UPDATE analyses SET + gated = ?, gate_reasons = ?, llm_called = ?, + input_tokens = ?, output_tokens = ?, cost_micros = ?, + model = ?, duration_ms = ?, error = ? + WHERE system_id = ? AND window_start = ?`, + boolToInt(a.Gated), string(reasons), boolToInt(a.LLMCalled), + a.InputTokens, a.OutputTokens, a.CostMicros, + a.Model, a.DurationMs, a.Error, a.SystemID, a.WindowStart, + ).Exec(ctx) + if err != nil { + return fmt.Errorf("store: finalize analysis: %w", err) + } + return nil +} + +func spendSince(ctx context.Context, w writer, sinceMs int64) (int64, error) { + // NullInt64 because SUM() over zero rows is NULL in both dialects. + var total sql.NullInt64 + err := w.bun().NewRaw( + `SELECT SUM(cost_micros) FROM analyses WHERE created_at >= ?`, sinceMs, + ).Scan(ctx, &total) + if err != nil { + return 0, fmt.Errorf("store: spend since: %w", err) + } + return total.Int64, nil +} + +// findingRow is the on-disk shape. Modules and Evidence are JSON TEXT per the +// portability rules, so they are marshalled in Go, never by the database. +type findingRow struct { + ID string `bun:"id"` + SystemID string `bun:"system_id"` + Fingerprint string `bun:"fingerprint"` + Severity string `bun:"severity"` + Title string `bun:"title"` + Summary string `bun:"summary"` + SuggestedAction string `bun:"suggested_action"` + Modules string `bun:"modules"` + Evidence string `bun:"evidence"` + Status string `bun:"status"` + OccurrenceCount int `bun:"occurrence_count"` + FirstSeen int64 `bun:"first_seen"` + LastSeen int64 `bun:"last_seen"` + ReopenedAt sql.NullInt64 `bun:"reopened_at"` + LLMModel string `bun:"llm_model"` + PromptVersion string `bun:"prompt_version"` +} + +func (r findingRow) toModel() (model.Finding, error) { + f := model.Finding{ + ID: r.ID, SystemID: r.SystemID, Fingerprint: r.Fingerprint, + Severity: r.Severity, Title: r.Title, Summary: r.Summary, + SuggestedAction: r.SuggestedAction, Status: r.Status, + OccurrenceCount: r.OccurrenceCount, + FirstSeen: r.FirstSeen, LastSeen: r.LastSeen, + LLMModel: r.LLMModel, PromptVersion: r.PromptVersion, + } + if err := json.Unmarshal([]byte(r.Modules), &f.Modules); err != nil { + return f, fmt.Errorf("store: unmarshal modules: %w", err) + } + if err := json.Unmarshal([]byte(r.Evidence), &f.Evidence); err != nil { + return f, fmt.Errorf("store: unmarshal evidence: %w", err) + } + if r.ReopenedAt.Valid { + v := r.ReopenedAt.Int64 + f.ReopenedAt = &v + } + return f, nil +} + +func upsertFinding(ctx context.Context, w writer, f model.Finding, + now int64) (Outcome, error) { + modules, err := json.Marshal(f.Modules) + if err != nil { + return "", fmt.Errorf("store: marshal modules: %w", err) + } + evidence, err := json.Marshal(f.Evidence) + if err != nil { + return "", fmt.Errorf("store: marshal evidence: %w", err) + } + + w.lock().Lock() + defer w.lock().Unlock() + + // Read the prior status first: it is the only thing that distinguishes a + // bump from a reopen, and the upsert destroys it. + var prior string + err = w.bun().NewRaw( + `SELECT status FROM findings WHERE system_id = ? AND fingerprint = ?`, + f.SystemID, f.Fingerprint, + ).Scan(ctx, &prior) + switch { + case errors.Is(err, sql.ErrNoRows): + prior = "" + case err != nil: + return "", fmt.Errorf("store: read prior status: %w", err) + } + + outcome := OutcomeInserted + switch prior { + case model.StatusOpen: + outcome = OutcomeBumped + case model.StatusStale: + outcome = OutcomeReopened + } + + // reopened_at is stamped only on the stale->open transition, keeping a + // plain recurrence distinguishable from a genuine re-occurrence. + var reopenedAt any + if outcome == OutcomeReopened { + reopenedAt = now + } + + _, err = w.bun().NewRaw(` + INSERT INTO findings + (id, system_id, fingerprint, severity, title, summary, + suggested_action, modules, evidence, status, occurrence_count, + first_seen, last_seen, reopened_at, llm_model, prompt_version) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, ?, ?, NULL, ?, ?) + ON CONFLICT (system_id, fingerprint) DO UPDATE SET + severity = excluded.severity, + title = excluded.title, + summary = excluded.summary, + suggested_action = excluded.suggested_action, + modules = excluded.modules, + evidence = excluded.evidence, + status = excluded.status, + occurrence_count = findings.occurrence_count + 1, + last_seen = excluded.last_seen, + reopened_at = ?, + llm_model = excluded.llm_model, + prompt_version = excluded.prompt_version`, + newID(), f.SystemID, f.Fingerprint, f.Severity, f.Title, f.Summary, + f.SuggestedAction, string(modules), string(evidence), + model.StatusOpen, now, now, f.LLMModel, f.PromptVersion, + reopenedAt, + ).Exec(ctx) + if err != nil { + return "", fmt.Errorf("store: upsert finding: %w", err) + } + return outcome, nil +} + +func listFindings(ctx context.Context, w writer, systemID string, + since int64, status string) ([]model.Finding, error) { + q := w.bun().NewSelect().Table("findings"). + Where("system_id = ?", systemID). + Where("last_seen >= ?", since) + if status != "" { + q = q.Where("status = ?", status) + } + var rows []findingRow + if err := q.Scan(ctx, &rows); err != nil { + return nil, fmt.Errorf("store: list findings: %w", err) + } + out := make([]model.Finding, 0, len(rows)) + for _, r := range rows { + f, err := r.toModel() + if err != nil { + return nil, err + } + out = append(out, f) + } + // Ordered in Go rather than SQL: severity rank is a Go concept, and a CASE + // expression would duplicate model.Severities into the schema. + model.SortFindings(out) + return out, nil +} + +func openFindings(ctx context.Context, w writer, systemID string) ([]model.Finding, error) { + return listFindings(ctx, w, systemID, 0, model.StatusOpen) +} + +func markStale(ctx context.Context, w writer, systemID string, + olderThan int64) (int, error) { + w.lock().Lock() + defer w.lock().Unlock() + res, err := w.bun().NewRaw(` + UPDATE findings SET status = ? + WHERE system_id = ? AND status = ? AND last_seen < ?`, + model.StatusStale, systemID, model.StatusOpen, olderThan, + ).Exec(ctx) + if err != nil { + return 0, fmt.Errorf("store: mark stale: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: mark stale rows: %w", err) + } + return int(n), nil +} +``` + +- [ ] **Step 6: Write `internal/store/prune.go`** + +```go +package store + +import ( + "context" + "fmt" + + "github.com/nethesis/nethesis-insights/internal/model" +) + +func pruneTemplates(ctx context.Context, w writer, olderThan int64) (int, error) { + return execCount(ctx, w, + `DELETE FROM system_templates WHERE last_seen < ?`, olderThan) +} + +func pruneFindings(ctx context.Context, w writer, olderThan int64) (int, error) { + // Only stale findings are pruned. An open finding is current by + // definition, however old its first_seen is. + return execCount(ctx, w, + `DELETE FROM findings WHERE status = ? AND last_seen < ?`, + model.StatusStale, olderThan) +} + +func pruneAnalyses(ctx context.Context, w writer, olderThan int64) (int, error) { + return execCount(ctx, w, + `DELETE FROM analyses WHERE created_at < ?`, olderThan) +} + +func execCount(ctx context.Context, w writer, query string, args ...any) (int, error) { + w.lock().Lock() + defer w.lock().Unlock() + res, err := w.bun().NewRaw(query, args...).Exec(ctx) + if err != nil { + return 0, fmt.Errorf("store: prune: %w", err) + } + n, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("store: prune rows: %w", err) + } + return int(n), nil +} +``` + +- [ ] **Step 7: Bind the ten new methods to both implementations** + +Append to `internal/store/sqlite.go`: + +```go +func (s *sqliteStore) BeginAnalysis(ctx context.Context, systemID string, + windowStart, windowEnd, now int64) (bool, error) { + return beginAnalysis(ctx, s, systemID, windowStart, windowEnd, now) +} + +func (s *sqliteStore) FinalizeAnalysis(ctx context.Context, a Analysis) error { + return finalizeAnalysis(ctx, s, a) +} + +func (s *sqliteStore) SpendSince(ctx context.Context, sinceMs int64) (int64, error) { + return spendSince(ctx, s, sinceMs) +} + +func (s *sqliteStore) OpenFindings(ctx context.Context, systemID string) ([]model.Finding, error) { + return openFindings(ctx, s, systemID) +} + +func (s *sqliteStore) UpsertFinding(ctx context.Context, f model.Finding, + now int64) (Outcome, error) { + return upsertFinding(ctx, s, f, now) +} + +func (s *sqliteStore) MarkStale(ctx context.Context, systemID string, + olderThan int64) (int, error) { + return markStale(ctx, s, systemID, olderThan) +} + +func (s *sqliteStore) ListFindings(ctx context.Context, systemID string, + since int64, status string) ([]model.Finding, error) { + return listFindings(ctx, s, systemID, since, status) +} + +func (s *sqliteStore) PruneTemplates(ctx context.Context, olderThan int64) (int, error) { + return pruneTemplates(ctx, s, olderThan) +} + +func (s *sqliteStore) PruneFindings(ctx context.Context, olderThan int64) (int, error) { + return pruneFindings(ctx, s, olderThan) +} + +func (s *sqliteStore) PruneAnalyses(ctx context.Context, olderThan int64) (int, error) { + return pruneAnalyses(ctx, s, olderThan) +} +``` + +Add the identical ten methods to `internal/store/postgres.go` with receiver `(s *pgStore)`, delegating to the same package-level functions. + +- [ ] **Step 8: Write the failing pruning tests** + +Create `internal/store/prune_test.go`: + +```go +package store + +import ( + "context" + "testing" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/stretchr/testify/require" +) + +func TestPruneTemplatesByLastSeen(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + require.NoError(t, s.UpsertTemplates(ctx, "sys1", + []model.Template{{Template: "old", Count: 1}}, 100)) + require.NoError(t, s.UpsertTemplates(ctx, "sys1", + []model.Template{{Template: "new", Count: 1}}, 900)) + + n, err := s.PruneTemplates(ctx, 500) + require.NoError(t, err) + require.Equal(t, 1, n) + + known, err := s.KnownTemplates(ctx, "sys1") + require.NoError(t, err) + require.False(t, known["old"]) + require.True(t, known["new"]) +} + +func TestPruneFindingsSparesOpenOnes(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.UpsertFinding(ctx, finding("fp-open", "high", 100), 100) + require.NoError(t, err) + _, err = s.UpsertFinding(ctx, finding("fp-stale", "high", 100), 100) + require.NoError(t, err) + + // Stale both, then bring fp-open back so only fp-stale is prunable. + _, err = s.MarkStale(ctx, "sys1", 200) + require.NoError(t, err) + _, err = s.UpsertFinding(ctx, finding("fp-open", "high", 300), 300) + require.NoError(t, err) + + n, err := s.PruneFindings(ctx, 250) + require.NoError(t, err) + require.Equal(t, 1, n, "only the stale finding is deleted") + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1) + require.Equal(t, "fp-open", open[0].Fingerprint) +} + +func TestPruneAnalysesByCreatedAt(t *testing.T) { + ctx, s := context.Background(), openSQLite(t) + _, err := s.BeginAnalysis(ctx, "sys1", 1000, 1900, 100) + require.NoError(t, err) + _, err = s.BeginAnalysis(ctx, "sys1", 2000, 2900, 900) + require.NoError(t, err) + + n, err := s.PruneAnalyses(ctx, 500) + require.NoError(t, err) + require.Equal(t, 1, n) +} +``` + +- [ ] **Step 9: Run the whole store package** + +```bash +go get github.com/oklog/ulid/v2 && go mod tidy +go test ./internal/store/ -v +``` + +Expected: PASS, every test. + +- [ ] **Step 10: Re-verify the Postgres dialect** + +This task added `DO NOTHING`, `SUM()`, `UPDATE … WHERE` and `DELETE`. Confirm every one is portable: + +```bash +podman run -d --name insights-pg-test \ + -e POSTGRES_PASSWORD=test -e POSTGRES_DB=insights_test \ + -p 127.0.0.1:55432:5432 postgres:16-alpine +until podman exec insights-pg-test pg_isready -q; do sleep 1; done +TEST_POSTGRES_DSN='postgres://postgres:test@127.0.0.1:55432/insights_test?sslmode=disable' \ + go test ./internal/store/ -run TestMigratePostgres -v +podman rm -f insights-pg-test +``` + +Expected: PASS + +- [ ] **Step 11: Commit** + +```bash +git add internal/store go.mod go.sum +git commit -m "feat(store): finding lifecycle, analysis ledger and pruning" +``` + +--- + +## Task 5: `internal/fingerprint` — server-owned finding identity + +**Files:** +- Create: `internal/fingerprint/fingerprint.go` +- Test: `internal/fingerprint/fingerprint_test.go` + +**Interfaces:** +- Consumes: nothing. +- Produces: + - `fingerprint.Version` — const `"v1"` + - `fingerprint.Compute(systemID string, modules, evidence []string, category string) string` — returns 64-char lowercase hex + +- [ ] **Step 1: Write the failing test** + +Create `internal/fingerprint/fingerprint_test.go`: + +```go +package fingerprint + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestStableAcrossEvidenceOrder(t *testing.T) { + a := Compute("sys1", []string{"m1"}, []string{"tmpl A", "tmpl B"}, "") + b := Compute("sys1", []string{"m1"}, []string{"tmpl B", "tmpl A"}, "") + require.Equal(t, a, b, + "the model may cite evidence in any order; identity must not depend on it") +} + +func TestStableAcrossModuleOrder(t *testing.T) { + require.Equal(t, + Compute("sys1", []string{"m1", "m2"}, []string{"t"}, ""), + Compute("sys1", []string{"m2", "m1"}, []string{"t"}, "")) +} + +func TestDistinctPerSystem(t *testing.T) { + require.NotEqual(t, + Compute("sys1", []string{"m1"}, []string{"t"}, ""), + Compute("sys2", []string{"m1"}, []string{"t"}, ""), + "one customer's finding must never dedup against another's") +} + +func TestDistinctPerModuleSet(t *testing.T) { + require.NotEqual(t, + Compute("s", []string{"m1"}, []string{"t"}, ""), + Compute("s", []string{"m2"}, []string{"t"}, "")) +} + +func TestDistinctPerCategory(t *testing.T) { + require.NotEqual(t, + Compute("s", []string{"m"}, []string{"t"}, "security"), + Compute("s", []string{"m"}, []string{"t"}, "")) +} + +func TestDistinctPerEvidence(t *testing.T) { + require.NotEqual(t, + Compute("s", []string{"m"}, []string{"t1"}, ""), + Compute("s", []string{"m"}, []string{"t1", "t2"}, "")) +} + +func TestDuplicateEvidenceIsCollapsed(t *testing.T) { + require.Equal(t, + Compute("s", []string{"m"}, []string{"t1"}, ""), + Compute("s", []string{"m"}, []string{"t1", "t1"}, ""), + "a model repeating one template must not mint a second identity") +} + +func TestFieldBoundariesCannotBeForged(t *testing.T) { + // Two evidence items must not collide with one item containing whatever + // separator the implementation uses. A plain strings.Join would collide. + require.NotEqual(t, + Compute("s", []string{"m"}, []string{"x", "y"}, ""), + Compute("s", []string{"m"}, []string{"x\x1fy"}, "")) + // The same hazard across adjacent fields. + require.NotEqual(t, + Compute("ab", []string{"m"}, []string{"t"}, "c"), + Compute("a", []string{"m"}, []string{"t"}, "bc")) +} + +func TestEmptyInputsAreHandled(t *testing.T) { + got := Compute("s", nil, nil, "") + require.Regexp(t, "^[0-9a-f]{64}$", got) +} + +func TestIsHexSHA256(t *testing.T) { + got := Compute("s", []string{"m"}, []string{"t"}, "") + require.Len(t, got, 64) + require.Regexp(t, "^[0-9a-f]{64}$", got) +} + +func TestGoldenValueIsPinned(t *testing.T) { + // Pinned in Step 5 so an accidental formula change fails loudly here + // rather than silently re-raising every open finding on the fleet. + got := Compute("sys1", []string{"traefik1"}, + []string{"<3> [n1:traefik1:traefik] connection refused to :"}, + "security") + t.Logf("pin this value in Step 5: %s", got) + require.Len(t, got, 64) +} +``` + +`TestFieldBoundariesCannotBeForged` is the test that earns its keep. Without length-prefixed fields, a template containing the separator byte collapses two distinct findings into one identity — a silent dedup bug that would be nearly impossible to diagnose from production symptoms. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/fingerprint/ -v` +Expected: FAIL — `undefined: Compute` + +- [ ] **Step 3: Write the implementation** + +Create `internal/fingerprint/fingerprint.go`: + +```go +// Package fingerprint computes the stable identity of a finding. It is pure: +// no I/O, no clock, no randomness. +package fingerprint + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "hash" + "sort" +) + +// Version prefixes every fingerprint. Changing it changes the identity of +// every finding in existence, so it moves only as a deliberate migration. +const Version = "v1" + +// Compute returns the identity of a finding: its system, category, module set +// and the set of evidence templates it cites. +// +// Lists are sorted and de-duplicated so the model's presentation order never +// affects identity, and every field is length-prefixed so no input value can +// imitate a field boundary. +func Compute(systemID string, modules, evidence []string, category string) string { + h := sha256.New() + writeField(h, Version) + writeField(h, systemID) + writeField(h, category) + writeList(h, modules) + writeList(h, evidence) + return hex.EncodeToString(h.Sum(nil)) +} + +// writeField length-prefixes a value so "ab"+"c" cannot collide with "a"+"bc". +func writeField(h hash.Hash, s string) { + var n [8]byte + binary.BigEndian.PutUint64(n[:], uint64(len(s))) + _, _ = h.Write(n[:]) + _, _ = h.Write([]byte(s)) +} + +// writeList length-prefixes the element count, then each element. +func writeList(h hash.Hash, items []string) { + uniq := dedupeSorted(items) + var n [8]byte + binary.BigEndian.PutUint64(n[:], uint64(len(uniq))) + _, _ = h.Write(n[:]) + for _, it := range uniq { + writeField(h, it) + } +} + +func dedupeSorted(items []string) []string { + if len(items) == 0 { + return nil + } + cp := append([]string(nil), items...) + sort.Strings(cp) + out := cp[:1] + for _, it := range cp[1:] { + if it != out[len(out)-1] { + out = append(out, it) + } + } + return out +} +``` + +`sha256.New()` returns a `hash.Hash` whose `Write` never returns an error, which is why the returns are discarded — `errcheck` accepts the explicit `_ =` form. + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/fingerprint/ -v` +Expected: PASS. Record the value logged by `TestGoldenValueIsPinned`. + +- [ ] **Step 5: Pin the golden value** + +Replace `TestGoldenValueIsPinned` with the observed hash substituted for the placeholder: + +```go +func TestGoldenValueIsPinned(t *testing.T) { + got := Compute("sys1", []string{"traefik1"}, + []string{"<3> [n1:traefik1:traefik] connection refused to :"}, + "security") + require.Equal(t, "PASTE_THE_64_HEX_VALUE_FROM_STEP_4", got, + "the fingerprint formula changed; that is a fleet-wide identity "+ + "migration, not a refactor") +} +``` + +Run: `go test ./internal/fingerprint/ -v` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add internal/fingerprint +git commit -m "feat(fingerprint): stable server-computed finding identity" +``` + +--- + +## Task 6: `internal/gate` — the cost control + +**Files:** +- Create: `internal/gate/gate.go` +- Test: `internal/gate/gate_test.go` + +**Interfaces:** +- Consumes: `model.Bundle`, `model.DigestEntry`, `model.Template` (Task 2); `store.BaselineKey` (Task 3). +- Produces: + - `gate.SystemState{KnownTemplates map[string]bool; Baselines map[store.BaselineKey]float64; SecurityOnly bool}` + - `gate.Decision{Call bool; Reasons []string}` + - `gate.Evaluate(b model.Bundle, s SystemState, tolerance float64) Decision` + - reason constants `gate.ReasonNewTemplates`, `ReasonDeviation`, `ReasonSecurity`, `ReasonTruncatedDeviating` + +- [ ] **Step 1: Write the failing test** + +Create `internal/gate/gate_test.go`: + +```go +package gate + +import ( + "strings" + "testing" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/nethesis/nethesis-insights/internal/store" + "github.com/stretchr/testify/require" +) + +func f64(v float64) *float64 { return &v } + +// steady is a bundle a healthy, unchanged system produces: every template +// already known, every rate at baseline, nothing security-flagged. +func steady() model.Bundle { + return model.Bundle{ + SystemID: "sys1", + Digest: []model.DigestEntry{ + {ModuleID: "m1", Priority: 4, Observed: 10, Expected: f64(10)}, + }, + Templates: []model.Template{ + {Template: "known line ", Count: 10, ModuleID: "m1", Priority: 4}, + }, + } +} + +func steadyState() SystemState { + return SystemState{ + KnownTemplates: map[string]bool{"known line ": true}, + Baselines: map[store.BaselineKey]float64{{ModuleID: "m1", Priority: 4}: 10}, + } +} + +func TestSteadyStateCostsNothing(t *testing.T) { + d := Evaluate(steady(), steadyState(), 3.0) + require.False(t, d.Call, "an unchanged system must not spend an LLM call") + require.Empty(t, d.Reasons) +} + +func TestNewTemplateTriggers(t *testing.T) { + b := steady() + b.Templates = append(b.Templates, model.Template{ + Template: "never seen before ", Count: 1, ModuleID: "m1", Priority: 4, + }) + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call) + require.Contains(t, strings.Join(d.Reasons, ","), ReasonNewTemplates) +} + +func TestDeviationUsesEdgeExpected(t *testing.T) { + b := steady() + b.Digest[0].Observed = 40 // 40/10 = 4.0 > 3.0 + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call) + require.Contains(t, strings.Join(d.Reasons, ","), ReasonDeviation) +} + +func TestDeviationJustUnderToleranceDoesNotTrigger(t *testing.T) { + b := steady() + b.Digest[0].Observed = 29 // 2.9 < 3.0 + require.False(t, Evaluate(b, steadyState(), 3.0).Call) +} + +func TestDeviationFallsBackToServerEWMAWhenEdgeDegraded(t *testing.T) { + // The edge's Loki metric query failed, so Expected is absent. The server + // baseline must keep the deviation gate working. + b := steady() + b.Digest[0].Expected = nil + b.Digest[0].Observed = 40 + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call, "server EWMA must cover for a degraded edge") + require.Contains(t, strings.Join(d.Reasons, ","), ReasonDeviation) +} + +func TestNoExpectedAndNoBaselineDoesNotTrigger(t *testing.T) { + // Nothing to compare against. Novelty already covers a first observation, + // so inventing a deviation here would double-charge for the same signal. + b := steady() + b.Digest[0].Expected = nil + b.Digest[0].Observed = 9999 + s := steadyState() + s.Baselines = map[store.BaselineKey]float64{} + require.False(t, Evaluate(b, s, 3.0).Call) +} + +func TestZeroExpectedNeverDividesByZero(t *testing.T) { + b := steady() + b.Digest[0].Expected = f64(0) + b.Digest[0].Observed = 500 + s := steadyState() + s.Baselines = map[store.BaselineKey]float64{} + require.NotPanics(t, func() { Evaluate(b, s, 3.0) }) + require.False(t, Evaluate(b, s, 3.0).Call) +} + +func TestSecurityCategoryAlwaysTriggers(t *testing.T) { + b := steady() + b.Templates[0].Category = "security" + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call, "a security line is never gated out") + require.Contains(t, strings.Join(d.Reasons, ","), ReasonSecurity) +} + +func TestTruncationAloneDoesNotTrigger(t *testing.T) { + b := steady() + b.Budget.TruncatedModules = []model.TruncatedModule{ + {ModuleID: "m1", Dropped: 3000}, + } + require.False(t, Evaluate(b, steadyState(), 3.0).Call, + "a chatty module at its normal rate is noise, not signal") +} + +func TestTruncationPlusDeviationTriggers(t *testing.T) { + b := steady() + b.Digest[0].Observed = 40 + b.Budget.TruncatedModules = []model.TruncatedModule{ + {ModuleID: "m1", Dropped: 3000}, + } + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call) + joined := strings.Join(d.Reasons, ",") + require.Contains(t, joined, ReasonTruncatedDeviating) +} + +func TestTruncationOfANonDeviatingModuleDoesNotTrigger(t *testing.T) { + b := steady() + b.Digest = append(b.Digest, model.DigestEntry{ + ModuleID: "m2", Priority: 4, Observed: 40, Expected: f64(10), + }) + b.Budget.TruncatedModules = []model.TruncatedModule{ + {ModuleID: "m1", Dropped: 3000}, // m1 is fine; m2 is the one deviating + } + d := Evaluate(b, steadyState(), 3.0) + require.True(t, d.Call, "m2 deviates, so the call happens") + require.NotContains(t, strings.Join(d.Reasons, ","), ReasonTruncatedDeviating, + "truncation of a healthy module is not itself a reason") +} + +func TestSecurityOnlyModeSuppressesEverythingElse(t *testing.T) { + // Spend cap breached: only security may still spend. + s := steadyState() + s.SecurityOnly = true + + b := steady() + b.Templates = append(b.Templates, model.Template{ + Template: "brand new ", Count: 1, ModuleID: "m1", Priority: 4}) + b.Digest[0].Observed = 999 + require.False(t, Evaluate(b, s, 3.0).Call, + "novelty and deviation must be suppressed under the spend cap") + + b.Templates[0].Category = "security" + d := Evaluate(b, s, 3.0) + require.True(t, d.Call, "security must still get through") + require.Contains(t, strings.Join(d.Reasons, ","), ReasonSecurity) +} + +func TestReasonsAreDeterministic(t *testing.T) { + b := steady() + b.Digest[0].Observed = 40 + b.Templates = append(b.Templates, model.Template{ + Template: "new ", Count: 1, ModuleID: "m1", Priority: 4}) + + first := Evaluate(b, steadyState(), 3.0).Reasons + for range 20 { + require.Equal(t, first, Evaluate(b, steadyState(), 3.0).Reasons, + "reasons are persisted and compared; map iteration must not leak in") + } +} +``` + +`TestReasonsAreDeterministic` guards against the most likely bug in this package: iterating `KnownTemplates` or `Baselines` directly and letting Go's randomized map order into the persisted `gate_reasons`. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/gate/ -v` +Expected: FAIL — `undefined: Evaluate` + +- [ ] **Step 3: Write the implementation** + +Create `internal/gate/gate.go`: + +```go +// Package gate decides whether a bundle is worth an LLM call. It is pure: no +// I/O, no clock. This is the primary cost control for the whole system. +package gate + +import ( + "fmt" + "sort" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/nethesis/nethesis-insights/internal/store" +) + +// Reason codes recorded in the analyses ledger. +const ( + ReasonNewTemplates = "new_templates" + ReasonDeviation = "deviation" + ReasonSecurity = "security_category" + ReasonTruncatedDeviating = "truncated_deviating" +) + +// SystemState is everything the gate needs to know about a system's history. +type SystemState struct { + KnownTemplates map[string]bool + Baselines map[store.BaselineKey]float64 + // SecurityOnly narrows the gate to security lines only. Set when the + // daily spend cap has been breached. + SecurityOnly bool +} + +// Decision is the gate's verdict plus the reasons behind it, which are +// persisted so both "why did this cost money" and "why was this missed" are +// answerable from stored data. +type Decision struct { + Call bool + Reasons []string +} + +// Evaluate returns the decision for one bundle. +func Evaluate(b model.Bundle, s SystemState, tolerance float64) Decision { + var reasons []string + + // Security is evaluated first because it is the only condition that + // survives SecurityOnly mode. + if securityPresent(b) { + reasons = append(reasons, ReasonSecurity) + } + + if !s.SecurityOnly { + if n := countNovel(b, s); n > 0 { + reasons = append(reasons, fmt.Sprintf("%s=%d", ReasonNewTemplates, n)) + } + deviating := deviatingModules(b, s, tolerance) + for _, key := range sortedKeys(deviating) { + reasons = append(reasons, fmt.Sprintf("%s:%s/%d=%.2f", + ReasonDeviation, key.ModuleID, key.Priority, deviating[key])) + } + // Truncation is only a reason when the truncated module is also + // deviating: under-sampling a module that is behaving normally tells + // us nothing. + for _, id := range truncatedAndDeviating(b, deviating) { + reasons = append(reasons, fmt.Sprintf("%s:%s", ReasonTruncatedDeviating, id)) + } + } + + return Decision{Call: len(reasons) > 0, Reasons: reasons} +} + +func securityPresent(b model.Bundle) bool { + for _, t := range b.Templates { + if t.Category == "security" { + return true + } + } + return false +} + +func countNovel(b model.Bundle, s SystemState) int { + n := 0 + for _, t := range b.Templates { + if !s.KnownTemplates[t.Template] { + n++ + } + } + return n +} + +// deviatingModules returns the observed/expected ratio for every entry above +// tolerance. Edge-supplied Expected wins; the server EWMA is the fallback for +// a degraded edge; with neither, there is nothing to compare and the entry is +// skipped — a first observation is already covered by novelty. +func deviatingModules(b model.Bundle, s SystemState, + tolerance float64) map[store.BaselineKey]float64 { + out := map[store.BaselineKey]float64{} + for _, e := range b.Digest { + key := store.BaselineKey{ModuleID: e.ModuleID, Priority: e.Priority} + expected := 0.0 + if e.Expected != nil { + expected = *e.Expected + } + if expected <= 0 { + expected = s.Baselines[key] + } + if expected <= 0 { + continue + } + if ratio := float64(e.Observed) / expected; ratio > tolerance { + out[key] = ratio + } + } + return out +} + +func truncatedAndDeviating(b model.Bundle, + deviating map[store.BaselineKey]float64) []string { + deviatingModuleIDs := map[string]bool{} + for key := range deviating { + deviatingModuleIDs[key.ModuleID] = true + } + var out []string + for _, tm := range b.Budget.TruncatedModules { + if deviatingModuleIDs[tm.ModuleID] { + out = append(out, tm.ModuleID) + } + } + sort.Strings(out) + return out +} + +// sortedKeys gives map iteration a stable order so persisted reasons are +// deterministic. +func sortedKeys(m map[store.BaselineKey]float64) []store.BaselineKey { + keys := make([]store.BaselineKey, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].ModuleID != keys[j].ModuleID { + return keys[i].ModuleID < keys[j].ModuleID + } + return keys[i].Priority < keys[j].Priority + }) + return keys +} +``` + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/gate/ -v` +Expected: PASS, all fourteen tests + +- [ ] **Step 5: Commit** + +```bash +git add internal/gate +git commit -m "feat(gate): novelty and deviation gating before any LLM spend" +``` + +--- + +## Task 7: `internal/prompt` — the LLM wire contract + +This package owns both directions of the LLM contract: what is sent (system prompt, user prompt, JSON schema) and what is accepted back (parse and validate). Keeping both in one package means the schema and the parser cannot drift apart. + +**Files:** +- Create: `internal/prompt/prompt.go`, `internal/prompt/schema.go` +- Test: `internal/prompt/prompt_test.go`, `internal/prompt/parse_test.go`, `internal/prompt/testdata/render.golden` + +**Interfaces:** +- Consumes: `model.Bundle`, `model.Finding`, `model.ValidSeverity`, `model.ValidAssessment` (Task 2). +- Produces: + - `prompt.Version` — const `"v1"`, the value stamped on findings + - `prompt.System` — const, the system prompt + - `prompt.Schema` — `map[string]any`, the strict JSON schema + - `prompt.Render(b model.Bundle, open []model.Finding) string` + - `prompt.Parse(body string) (findings []ParsedFinding, assessment string, err error)` + - `prompt.ParsedFinding{Severity, Title, Summary, SuggestedAction string; Modules, Evidence []string}` + +- [ ] **Step 1: Write the failing determinism test** + +Create `internal/prompt/prompt_test.go`: + +```go +package prompt + +import ( + "os" + "path/filepath" + "testing" + + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/stretchr/testify/require" +) + +func f64(v float64) *float64 { return &v } + +func fixture() model.Bundle { + return model.Bundle{ + SchemaVersion: 1, SystemID: "sys1", CollectorVersion: "2.0.0", + MaskingVersion: 1, + Window: model.Window{Start: 1754380800000, End: 1754381700000}, + Digest: []model.DigestEntry{ + {ModuleID: "traefik1", Priority: 3, Observed: 42, Expected: f64(3.2)}, + {ModuleID: "samba2", Priority: 4, Observed: 5, Expected: f64(4.0)}, + }, + Templates: []model.Template{ + {Template: "<4> [n1:samba2:smbd] slow oplock break ms", + Count: 5, ModuleID: "samba2", Priority: 4}, + {Template: "<3> [n1:traefik1:traefik] connection refused to :", + Count: 37, ModuleID: "traefik1", Priority: 3, Category: "security"}, + }, + Budget: model.Budget{ + MaxLines: 500, LinesSeen: 4210, LinesKept: 500, + TruncatedModules: []model.TruncatedModule{ + {ModuleID: "traefik1", Dropped: 3200}}, + }, + } +} + +func TestRenderIsByteIdenticalAcrossInputOrder(t *testing.T) { + a := Render(fixture(), nil) + + // Same content, reversed input order. Identical prompts are what make + // LLM output reproducible and what let a caching layer ever work. + shuffled := fixture() + shuffled.Digest[0], shuffled.Digest[1] = shuffled.Digest[1], shuffled.Digest[0] + shuffled.Templates[0], shuffled.Templates[1] = + shuffled.Templates[1], shuffled.Templates[0] + b := Render(shuffled, nil) + + require.Equal(t, a, b) +} + +func TestRenderIsStableAcrossCalls(t *testing.T) { + first := Render(fixture(), nil) + for range 20 { + require.Equal(t, first, Render(fixture(), nil)) + } +} + +func TestRenderNeverIncludesSamples(t *testing.T) { + b := fixture() + b.Templates[0].Samples = []string{"RAW 10.0.0.4 secret"} + require.NotContains(t, Render(b, nil), "RAW 10.0.0.4 secret", + "the prompt carries templates and counts, not raw lines") +} + +func TestRenderIncludesTruncationDetail(t *testing.T) { + out := Render(fixture(), nil) + require.Contains(t, out, "traefik1") + require.Contains(t, out, "3200", + "the model must see which module was under-sampled and by how much") +} + +func TestRenderIncludesOpenFindingsAndTheInstruction(t *testing.T) { + out := Render(fixture(), []model.Finding{{ + Severity: "high", Title: "Traefik backend unreachable", + Fingerprint: "abc", LastSeen: 1754380000000, + }}) + require.Contains(t, out, "Traefik backend unreachable") + require.Contains(t, out, "ALREADY KNOWN") +} + +func TestRenderMatchesGolden(t *testing.T) { + got := Render(fixture(), nil) + path := filepath.Join("testdata", "render.golden") + + if os.Getenv("UPDATE_GOLDEN") != "" { + require.NoError(t, os.MkdirAll("testdata", 0o755)) + require.NoError(t, os.WriteFile(path, []byte(got), 0o644)) + } + want, err := os.ReadFile(path) + require.NoError(t, err, "run with UPDATE_GOLDEN=1 to create the golden file") + require.Equal(t, string(want), got, + "the prompt changed; bump prompt.Version deliberately") +} +``` + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/prompt/ -run TestRender -v` +Expected: FAIL — `undefined: Render` + +- [ ] **Step 3: Write the renderer** + +Create `internal/prompt/prompt.go`: + +```go +// Package prompt owns the LLM wire contract in both directions: the system +// prompt, the user prompt and the response schema that is sent, and the parser +// that validates what comes back. It is pure: no I/O, no clock. +package prompt + +import ( + "fmt" + "sort" + "strings" + + "github.com/nethesis/nethesis-insights/internal/model" +) + +// Version is stamped on every finding. It must change whenever System, Render +// or Schema changes, so a finding always records how it was produced. +const Version = "v1" + +// System is the system prompt. It is a constant, not a template: any +// per-request variation belongs in the user prompt. +const System = "You are a log analysis assistant for NethServer systems. " + + "You are given a digest of log volumes and a set of masked log line " + + "templates with occurrence counts for one 15-minute window.\n\n" + + "Report only conditions that indicate a real problem. Report only NEW or " + + "CHANGED conditions: if a condition is listed as ALREADY KNOWN, do not " + + "report it again unless its character has materially changed.\n\n" + + "Cite evidence by copying template strings verbatim from the TEMPLATES " + + "block. Never invent a template. Never include raw hostnames, IP " + + "addresses or user names beyond what the templates already contain.\n\n" + + "If nothing warrants reporting, return an empty findings array with " + + "window_assessment \"nominal\"." + +// Render builds the user prompt. Every list is sorted, so the same bundle +// always produces byte-identical output regardless of input order. +func Render(b model.Bundle, open []model.Finding) string { + var sb strings.Builder + + fmt.Fprintf(&sb, "WINDOW\nstart_ms=%d end_ms=%d collector=%s masking=%d\n\n", + b.Window.Start, b.Window.End, b.CollectorVersion, b.MaskingVersion) + + sb.WriteString("DIGEST (module priority observed expected ratio)\n") + for _, e := range sortedDigest(b.Digest) { + if e.Expected != nil && *e.Expected > 0 { + fmt.Fprintf(&sb, "%s %d %d %.2f %.2f\n", e.ModuleID, e.Priority, + e.Observed, *e.Expected, float64(e.Observed)/*e.Expected) + } else { + fmt.Fprintf(&sb, "%s %d %d - -\n", e.ModuleID, e.Priority, e.Observed) + } + } + + sb.WriteString("\nTEMPLATES (count module priority category | template)\n") + for _, t := range sortedTemplates(b.Templates) { + category := t.Category + if category == "" { + category = "-" + } + fmt.Fprintf(&sb, "%d %s %d %s | %s\n", + t.Count, t.ModuleID, t.Priority, category, t.Template) + } + + fmt.Fprintf(&sb, "\nSAMPLING\nlines_seen=%d lines_kept=%d max_lines=%d\n", + b.Budget.LinesSeen, b.Budget.LinesKept, b.Budget.MaxLines) + if len(b.Budget.TruncatedModules) > 0 { + sb.WriteString("under_sampled (module dropped)\n") + for _, tm := range sortedTruncated(b.Budget.TruncatedModules) { + fmt.Fprintf(&sb, "%s %d\n", tm.ModuleID, tm.Dropped) + } + sb.WriteString("Lines were dropped for the modules above. Treat their " + + "counts as lower bounds.\n") + } + + sb.WriteString("\nALREADY KNOWN (do not report these again)\n") + if len(open) == 0 { + sb.WriteString("none\n") + } else { + cp := append([]model.Finding(nil), open...) + model.SortFindings(cp) + for _, f := range cp { + fmt.Fprintf(&sb, "[%s] %s\n", f.Severity, f.Title) + } + } + + return sb.String() +} + +func sortedDigest(in []model.DigestEntry) []model.DigestEntry { + out := append([]model.DigestEntry(nil), in...) + sort.Slice(out, func(i, j int) bool { + if out[i].ModuleID != out[j].ModuleID { + return out[i].ModuleID < out[j].ModuleID + } + return out[i].Priority < out[j].Priority + }) + return out +} + +func sortedTemplates(in []model.Template) []model.Template { + out := append([]model.Template(nil), in...) + sort.Slice(out, func(i, j int) bool { + if out[i].ModuleID != out[j].ModuleID { + return out[i].ModuleID < out[j].ModuleID + } + if out[i].Priority != out[j].Priority { + return out[i].Priority < out[j].Priority + } + return out[i].Template < out[j].Template + }) + return out +} + +func sortedTruncated(in []model.TruncatedModule) []model.TruncatedModule { + out := append([]model.TruncatedModule(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i].ModuleID < out[j].ModuleID }) + return out +} +``` + +- [ ] **Step 4: Create the golden file and confirm the tests pass** + +```bash +UPDATE_GOLDEN=1 go test ./internal/prompt/ -run TestRenderMatchesGolden +go test ./internal/prompt/ -run TestRender -v +cat internal/prompt/testdata/render.golden +``` + +Expected: PASS, and the golden file reads as a sorted digest, sorted templates, a sampling block naming `traefik1 3200`, and `ALREADY KNOWN\nnone`. + +The golden file needs no license header — `.golden` is excluded by `hack/check-license-headers.sh`. + +- [ ] **Step 5: Write the failing parser tests** + +Create `internal/prompt/parse_test.go`: + +```go +package prompt + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const validResponse = `{ + "window_assessment": "incident", + "findings": [ + {"severity": "high", "title": "T", "summary": "S", + "suggested_action": "A", "modules": ["m1"], "evidence": ["tmpl "]} + ] +}` + +func TestParseValidResponse(t *testing.T) { + fs, assessment, err := Parse(validResponse) + require.NoError(t, err) + require.Equal(t, "incident", assessment) + require.Len(t, fs, 1) + require.Equal(t, "high", fs[0].Severity) + require.Equal(t, []string{"tmpl "}, fs[0].Evidence) +} + +func TestParseAcceptsFencedJSON(t *testing.T) { + // Some models wrap JSON in a markdown fence despite strict schema mode. + fenced := "```json\n" + validResponse + "\n```" + _, _, err := Parse(fenced) + require.NoError(t, err) +} + +func TestParseAcceptsEmptyFindings(t *testing.T) { + fs, assessment, err := Parse(`{"window_assessment":"nominal","findings":[]}`) + require.NoError(t, err) + require.Equal(t, "nominal", assessment) + require.Empty(t, fs) +} + +func TestParseRejectsUnknownSeverity(t *testing.T) { + _, _, err := Parse(strings.Replace(validResponse, `"high"`, `"catastrophic"`, 1)) + require.Error(t, err) + require.Contains(t, err.Error(), "severity") +} + +func TestParseRejectsUnknownAssessment(t *testing.T) { + _, _, err := Parse(strings.Replace(validResponse, `"incident"`, `"vibes"`, 1)) + require.Error(t, err) + require.Contains(t, err.Error(), "assessment") +} + +func TestParseRejectsMissingTitle(t *testing.T) { + _, _, err := Parse(strings.Replace(validResponse, `"title": "T"`, `"title": ""`, 1)) + require.Error(t, err) + require.Contains(t, err.Error(), "title") +} + +func TestParseRejectsEmptyEvidence(t *testing.T) { + // Evidence is what the fingerprint is computed from. A finding with none + // has no stable identity and would dedup against every other such finding. + _, _, err := Parse(strings.Replace(validResponse, + `"evidence": ["tmpl "]`, `"evidence": []`, 1)) + require.Error(t, err) + require.Contains(t, err.Error(), "evidence") +} + +func TestParseRejectsMalformedJSON(t *testing.T) { + _, _, err := Parse(`not json at all`) + require.Error(t, err) +} + +func TestParseAllowsEmptySuggestedAction(t *testing.T) { + _, _, err := Parse(strings.Replace(validResponse, + `"suggested_action": "A"`, `"suggested_action": ""`, 1)) + require.NoError(t, err, "an action is useful but not always available") +} +``` + +`TestParseRejectsEmptyEvidence` protects the fingerprint: identity is computed from evidence, so a finding citing nothing would collapse onto every other evidence-free finding for that system. + +- [ ] **Step 6: Run them to make sure they fail** + +Run: `go test ./internal/prompt/ -run TestParse -v` +Expected: FAIL — `undefined: Parse` + +- [ ] **Step 7: Write the schema and the parser** + +Create `internal/prompt/schema.go`: + +```go +package prompt + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + + "github.com/nethesis/nethesis-insights/internal/model" +) + +// Schema is the strict JSON schema sent as response_format. additionalProperties +// is false and every property is required, which is what OpenAI's strict mode +// demands. +var Schema = map[string]any{ + "type": "object", + "properties": map[string]any{ + "window_assessment": map[string]any{ + "type": "string", "enum": model.Assessments, + }, + "findings": map[string]any{ + "type": "array", + "items": map[string]any{ + "type": "object", + "properties": map[string]any{ + "severity": map[string]any{ + "type": "string", "enum": model.Severities, + }, + "title": map[string]any{"type": "string"}, + "summary": map[string]any{"type": "string"}, + "suggested_action": map[string]any{"type": "string"}, + "modules": map[string]any{ + "type": "array", + "items": map[string]any{"type": "string"}, + }, + "evidence": map[string]any{ + "type": "array", + "items": map[string]any{"type": "string"}, + }, + }, + "required": []string{"severity", "title", "summary", + "suggested_action", "modules", "evidence"}, + "additionalProperties": false, + }, + }, + }, + "required": []string{"window_assessment", "findings"}, + "additionalProperties": false, +} + +// ParsedFinding is one finding as the model returned it. It has no fingerprint +// and no ID: identity is the server's to compute, never the model's. +type ParsedFinding struct { + Severity string `json:"severity"` + Title string `json:"title"` + Summary string `json:"summary"` + SuggestedAction string `json:"suggested_action"` + Modules []string `json:"modules"` + Evidence []string `json:"evidence"` +} + +type response struct { + WindowAssessment string `json:"window_assessment"` + Findings []ParsedFinding `json:"findings"` +} + +var fenceRE = regexp.MustCompile("(?s)^\\s*```(?:json)?\\s*\n(.*?)\n?\\s*```\\s*$") + +// Parse validates a model response. Schema mode is enforced server-side by the +// provider, but this validation is not redundant: providers vary, and a +// malformed finding that reaches the store corrupts identity permanently. +func Parse(body string) ([]ParsedFinding, string, error) { + if m := fenceRE.FindStringSubmatch(body); m != nil { + body = m[1] + } + var r response + if err := json.Unmarshal([]byte(strings.TrimSpace(body)), &r); err != nil { + return nil, "", fmt.Errorf("prompt: decode response: %w", err) + } + if !model.ValidAssessment(r.WindowAssessment) { + return nil, "", fmt.Errorf("prompt: invalid window_assessment %q", + r.WindowAssessment) + } + for i, f := range r.Findings { + if !model.ValidSeverity(f.Severity) { + return nil, "", fmt.Errorf("prompt: finding %d: invalid severity %q", + i, f.Severity) + } + if strings.TrimSpace(f.Title) == "" { + return nil, "", fmt.Errorf("prompt: finding %d: empty title", i) + } + if strings.TrimSpace(f.Summary) == "" { + return nil, "", fmt.Errorf("prompt: finding %d: empty summary", i) + } + if len(f.Evidence) == 0 { + return nil, "", fmt.Errorf( + "prompt: finding %d: empty evidence, cannot compute identity", i) + } + } + return r.Findings, r.WindowAssessment, nil +} +``` + +- [ ] **Step 8: Run the whole package** + +Run: `go test ./internal/prompt/ -v` +Expected: PASS, all tests + +- [ ] **Step 9: Commit** + +```bash +git add internal/prompt +git commit -m "feat(prompt): deterministic prompt rendering, strict schema and parser" +``` + +--- + +## Task 8: `internal/llm` — OpenAI-compatible client + +**Files:** +- Create: `internal/llm/llm.go`, `internal/llm/openai.go`, `internal/llm/stub.go` +- Test: `internal/llm/openai_test.go` + +**Interfaces:** +- Consumes: `prompt.System`, `prompt.Schema` (Task 7). +- Produces: + - `llm.Client` interface — `Complete(ctx, Request) (Response, error)` + - `llm.Request{Model, UserPrompt string}` + - `llm.Response{Content, Model string; InputTokens, OutputTokens int}` + - `llm.HTTPError{StatusCode int; Body string}` with method `Permanent() bool` + - `llm.NewOpenAI(baseURL, apiKey string, timeout time.Duration) *OpenAI` + - `llm.Stub` — a `Client` for tests, with fields `Content string`, `Err error`, `Calls int`, `LastRequest Request` + +- [ ] **Step 1: Write the failing test** + +Create `internal/llm/openai_test.go`: + +```go +package llm + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestRequestBodyOmitsTemperature(t *testing.T) { + var captured map[string]any + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + require.NoError(t, json.Unmarshal(body, &captured)) + _, _ = w.Write([]byte(`{"model":"m","choices":[{"message": + {"content":"{}"}}],"usage":{"prompt_tokens":10, + "completion_tokens":2}}`)) + })) + defer srv.Close() + + c := NewOpenAI(srv.URL, "test-key", 5*time.Second) + _, err := c.Complete(context.Background(), + Request{Model: "gpt-4o-mini", UserPrompt: "hello"}) + require.NoError(t, err) + + // Some models reject any non-default temperature outright. The field must + // be absent, not zero. + _, present := captured["temperature"] + require.False(t, present, "temperature must never be sent") + + rf, ok := captured["response_format"].(map[string]any) + require.True(t, ok) + require.Equal(t, "json_schema", rf["type"]) + js, ok := rf["json_schema"].(map[string]any) + require.True(t, ok) + require.Equal(t, true, js["strict"]) +} + +func TestCompleteReturnsContentAndUsage(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"model":"gpt-4o-mini","choices":[{"message": + {"content":"{\"window_assessment\":\"nominal\",\"findings\":[]}"}}], + "usage":{"prompt_tokens":12400,"completion_tokens":300}}`)) + })) + defer srv.Close() + + resp, err := NewOpenAI(srv.URL, "k", 5*time.Second). + Complete(context.Background(), Request{Model: "gpt-4o-mini"}) + require.NoError(t, err) + require.Contains(t, resp.Content, "nominal") + require.Equal(t, 12400, resp.InputTokens) + require.Equal(t, 300, resp.OutputTokens) + require.Equal(t, "gpt-4o-mini", resp.Model) +} + +func TestSendsBearerToken(t *testing.T) { + var auth string + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + auth = r.Header.Get("Authorization") + _, _ = w.Write([]byte(`{"choices":[{"message":{"content":"{}"}}]}`)) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "sk-secret", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + require.NoError(t, err) + require.Equal(t, "Bearer sk-secret", auth) +} + +func TestClientErrorIsPermanent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"bad schema"}}`)) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "k", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + require.Error(t, err) + + var he *HTTPError + require.ErrorAs(t, err, &he) + require.Equal(t, 400, he.StatusCode) + require.Contains(t, he.Body, "bad schema") + require.True(t, he.Permanent(), "a 400 will fail identically on retry") +} + +func TestRateLimitIsNotPermanent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "k", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + var he *HTTPError + require.ErrorAs(t, err, &he) + require.False(t, he.Permanent(), "429 is the canonical retryable case") +} + +func TestServerErrorIsNotPermanent(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusBadGateway) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "k", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + var he *HTTPError + require.ErrorAs(t, err, &he) + require.False(t, he.Permanent()) +} + +func TestErrorNeverLeaksTheAPIKey(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, _ = w.Write([]byte(`{"error":{"message":"bad key"}}`)) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "sk-super-secret", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + require.Error(t, err) + require.NotContains(t, err.Error(), "sk-super-secret", + "errors are logged; the key must never appear in one") +} + +func TestEmptyChoicesIsAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"choices":[]}`)) + })) + defer srv.Close() + + _, err := NewOpenAI(srv.URL, "k", 5*time.Second). + Complete(context.Background(), Request{Model: "m"}) + require.Error(t, err) +} +``` + +`TestErrorNeverLeaksTheAPIKey` matters because error strings end up in the `analyses.error` column and in logs. A client that formats its request into an error message would write the key to disk. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/llm/ -v` +Expected: FAIL — `undefined: NewOpenAI` + +- [ ] **Step 3: Write the implementation** + +Create `internal/llm/llm.go`: + +```go +// Package llm talks to an OpenAI-compatible chat completions endpoint. +package llm + +import ( + "context" + "fmt" + "net/http" +) + +// Request is one completion request. The system prompt and response schema are +// supplied by the llm package itself, so callers cannot vary them per call and +// accidentally break output consistency. +type Request struct { + Model string + UserPrompt string +} + +// Response is what the provider returned, plus token usage for the ledger. +type Response struct { + Content string + Model string + InputTokens int + OutputTokens int +} + +// Client is the provider abstraction. The analyzer depends on this, never on +// the concrete implementation, which is what makes it testable offline. +type Client interface { + Complete(ctx context.Context, req Request) (Response, error) +} + +// HTTPError is a non-2xx response. Body is the provider's message and never +// contains request data. +type HTTPError struct { + StatusCode int + Body string +} + +func (e *HTTPError) Error() string { + return fmt.Sprintf("llm: provider returned %d: %s", e.StatusCode, e.Body) +} + +// Permanent reports whether retrying is pointless. A 4xx other than 429 means +// the request itself is wrong and will fail identically forever, so the caller +// should dead-letter rather than retry. +func (e *HTTPError) Permanent() bool { + return e.StatusCode >= 400 && e.StatusCode < 500 && + e.StatusCode != http.StatusTooManyRequests +} +``` + +Create `internal/llm/openai.go`: + +```go +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/nethesis/nethesis-insights/internal/prompt" +) + +// maxErrorBody caps how much of a provider error is read, so a misbehaving +// endpoint cannot fill the analyses.error column or the logs. +const maxErrorBody = 4096 + +// OpenAI implements Client against any OpenAI-compatible /chat/completions +// endpoint, including OpenRouter. +type OpenAI struct { + baseURL string + apiKey string + client *http.Client +} + +// NewOpenAI returns a client. apiKey is held in memory only and never appears +// in an error or log message. +func NewOpenAI(baseURL, apiKey string, timeout time.Duration) *OpenAI { + return &OpenAI{ + baseURL: strings.TrimSuffix(baseURL, "/"), + apiKey: apiKey, + client: &http.Client{Timeout: timeout}, + } +} + +type chatRequest struct { + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + // No Temperature field at all. Some models reject any non-default value, + // and omitempty on a zero float would still send 0 when set explicitly. + ResponseFormat responseFormat `json:"response_format"` +} + +type chatMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type responseFormat struct { + Type string `json:"type"` + JSONSchema jsonSchema `json:"json_schema"` +} + +type jsonSchema struct { + Name string `json:"name"` + Strict bool `json:"strict"` + Schema map[string]any `json:"schema"` +} + +type chatResponse struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` +} + +// Complete sends one request. It does not retry: retry policy is the +// analyzer's, because only the analyzer knows whether the work is still owned. +func (c *OpenAI) Complete(ctx context.Context, req Request) (Response, error) { + body, err := json.Marshal(chatRequest{ + Model: req.Model, + Messages: []chatMessage{ + {Role: "system", Content: prompt.System}, + {Role: "user", Content: req.UserPrompt}, + }, + ResponseFormat: responseFormat{ + Type: "json_schema", + JSONSchema: jsonSchema{ + Name: "anomaly_report", Strict: true, Schema: prompt.Schema, + }, + }, + }) + if err != nil { + return Response{}, fmt.Errorf("llm: marshal request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + c.baseURL+"/chat/completions", bytes.NewReader(body)) + if err != nil { + return Response{}, fmt.Errorf("llm: build request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+c.apiKey) + + resp, err := c.client.Do(httpReq) + if err != nil { + // %w on the transport error only; the request is never formatted in, + // so the key cannot leak into a log line. + return Response{}, fmt.Errorf("llm: request failed: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBody)) + return Response{}, &HTTPError{ + StatusCode: resp.StatusCode, Body: string(msg), + } + } + + var parsed chatResponse + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return Response{}, fmt.Errorf("llm: decode response: %w", err) + } + if len(parsed.Choices) == 0 { + return Response{}, fmt.Errorf("llm: response contained no choices") + } + return Response{ + Content: parsed.Choices[0].Message.Content, + Model: parsed.Model, + InputTokens: parsed.Usage.PromptTokens, + OutputTokens: parsed.Usage.CompletionTokens, + }, nil +} +``` + +Create `internal/llm/stub.go`: + +```go +package llm + +import "context" + +// Stub is a Client for tests. It lives in the production package rather than a +// _test.go file so other packages' tests can use it. +type Stub struct { + Content string + Model string + InputTokens int + OutputTokens int + Err error + + Calls int + LastRequest Request +} + +// Complete records the call and returns the configured result. +func (s *Stub) Complete(_ context.Context, req Request) (Response, error) { + s.Calls++ + s.LastRequest = req + if s.Err != nil { + return Response{}, s.Err + } + return Response{ + Content: s.Content, Model: s.Model, + InputTokens: s.InputTokens, OutputTokens: s.OutputTokens, + }, nil +} +``` + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/llm/ -v` +Expected: PASS, all eight tests + +- [ ] **Step 5: Commit** + +```bash +git add internal/llm +git commit -m "feat(llm): OpenAI-compatible client with strict schema and no temperature" +``` + +--- + +## Task 9: `internal/budget` — concurrency cap and daily spend ceiling + +These are the thundering-herd defences from spec §9.3. A fleet-wide collector upgrade makes every template novel at once, so without these the gate opens for all 2700 systems in the same window. + +**Files:** +- Create: `internal/budget/budget.go` +- Test: `internal/budget/budget_test.go` + +**Interfaces:** +- Consumes: `store.Store` (for `SpendSince`) — Task 4. +- Produces: + - `budget.NewLimiter(n int) *Limiter` with `Acquire(ctx) error` and `Release()` + - `budget.Pricing{InputPerMTok, OutputPerMTok float64}` with `CostMicros(inputTokens, outputTokens int) int64` + - `budget.NewGuard(s SpendReader, capMicros int64, ttl time.Duration, now func() int64) *Guard` with `SecurityOnly(ctx) bool` + - `budget.SpendReader` interface — `SpendSince(ctx, sinceMs int64) (int64, error)` + +- [ ] **Step 1: Write the failing test** + +Create `internal/budget/budget_test.go`: + +```go +package budget + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLimiterCapsConcurrency(t *testing.T) { + l := NewLimiter(2) + ctx := context.Background() + require.NoError(t, l.Acquire(ctx)) + require.NoError(t, l.Acquire(ctx)) + + // The third acquire must block until a slot frees. + done := make(chan struct{}) + go func() { + _ = l.Acquire(ctx) + close(done) + }() + select { + case <-done: + t.Fatal("third acquire should have blocked") + case <-time.After(50 * time.Millisecond): + } + l.Release() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("release did not free a slot") + } +} + +func TestLimiterRespectsContextCancellation(t *testing.T) { + l := NewLimiter(1) + require.NoError(t, l.Acquire(context.Background())) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.Error(t, l.Acquire(ctx), + "a shutting-down analyzer must not block forever on a full limiter") +} + +func TestLimiterIsSafeUnderConcurrentUse(t *testing.T) { + l := NewLimiter(4) + var wg sync.WaitGroup + for range 50 { + wg.Add(1) + go func() { + defer wg.Done() + require.NoError(t, l.Acquire(context.Background())) + l.Release() + }() + } + wg.Wait() +} + +func TestCostMicros(t *testing.T) { + // gpt-4o-mini: $0.15 per 1M input, $0.60 per 1M output. + p := Pricing{InputPerMTok: 0.15, OutputPerMTok: 0.60} + // 12400 input -> $0.00186; 300 output -> $0.00018; total $0.00204 + require.Equal(t, int64(2040), p.CostMicros(12400, 300)) +} + +func TestCostMicrosIsZeroForFreeModels(t *testing.T) { + p := Pricing{} + require.Zero(t, p.CostMicros(999999, 999999), + "a free OpenRouter model must not accrue phantom spend") +} + +type fakeSpend struct { + micros int64 + err error + calls int +} + +func (f *fakeSpend) SpendSince(_ context.Context, _ int64) (int64, error) { + f.calls++ + return f.micros, f.err +} + +func TestGuardOpensBelowTheCap(t *testing.T) { + s := &fakeSpend{micros: 1000} + g := NewGuard(s, 5000, time.Minute, func() int64 { return 0 }) + require.False(t, g.SecurityOnly(context.Background())) +} + +func TestGuardClosesAtTheCap(t *testing.T) { + s := &fakeSpend{micros: 5000} + g := NewGuard(s, 5000, time.Minute, func() int64 { return 0 }) + require.True(t, g.SecurityOnly(context.Background()), + "reaching the cap must narrow the gate, not stop the service") +} + +func TestGuardCachesWithinTTL(t *testing.T) { + s := &fakeSpend{micros: 0} + now := int64(0) + g := NewGuard(s, 5000, time.Minute, func() int64 { return now }) + + require.False(t, g.SecurityOnly(context.Background())) + require.False(t, g.SecurityOnly(context.Background())) + require.Equal(t, 1, s.calls, "the ledger must not be summed on every bundle") + + now = 61_000 // past the TTL + require.False(t, g.SecurityOnly(context.Background())) + require.Equal(t, 2, s.calls) +} + +func TestGuardIsDisabledWhenCapIsZero(t *testing.T) { + s := &fakeSpend{micros: 999_999_999} + g := NewGuard(s, 0, time.Minute, func() int64 { return 0 }) + require.False(t, g.SecurityOnly(context.Background())) + require.Zero(t, s.calls, "an unset cap must not even query the ledger") +} + +func TestGuardFailsOpenOnLedgerError(t *testing.T) { + // A database hiccup must not silently narrow the gate to security-only + // and hide real incidents. Losing money is recoverable; losing detection + // without any signal is not. + s := &fakeSpend{err: errors.New("db down")} + g := NewGuard(s, 5000, time.Minute, func() int64 { return 0 }) + require.False(t, g.SecurityOnly(context.Background())) +} +``` + +`TestGuardFailsOpenOnLedgerError` is the opposite choice from the auth path, deliberately. Auth fails closed because failing open there is a security hole. The spend guard fails open because failing closed there silently suppresses detection with no operator signal, and an over-run bill is visible and recoverable. + +- [ ] **Step 2: Run it to make sure it fails** + +Run: `go test ./internal/budget/ -v` +Expected: FAIL — `undefined: NewLimiter` + +- [ ] **Step 3: Write the implementation** + +Create `internal/budget/budget.go`: + +```go +// Package budget bounds what the analyzer may spend: how many LLM calls run +// at once, and how much money may be spent per day. +package budget + +import ( + "context" + "log/slog" + "math" + "sync" + "time" +) + +// Limiter caps concurrent LLM calls. Excess work waits in the Redpanda topic, +// which is what the topic is for. +type Limiter struct { + sem chan struct{} +} + +// NewLimiter returns a limiter allowing n concurrent holders. n < 1 is treated +// as 1 so a misconfiguration cannot deadlock the analyzer. +func NewLimiter(n int) *Limiter { + if n < 1 { + n = 1 + } + return &Limiter{sem: make(chan struct{}, n)} +} + +// Acquire takes a slot, blocking until one is free or ctx is done. +func (l *Limiter) Acquire(ctx context.Context) error { + select { + case l.sem <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Release returns a slot. +func (l *Limiter) Release() { + select { + case <-l.sem: + default: + } +} + +// Pricing converts token usage into money. Values are US dollars per million +// tokens, matching how providers publish them. +type Pricing struct { + InputPerMTok float64 + OutputPerMTok float64 +} + +// CostMicros returns the cost in micro-dollars, rounded to nearest. +func (p Pricing) CostMicros(inputTokens, outputTokens int) int64 { + dollars := (float64(inputTokens)/1e6)*p.InputPerMTok + + (float64(outputTokens)/1e6)*p.OutputPerMTok + return int64(math.Round(dollars * 1e6)) +} + +// SpendReader is the slice of the store the guard needs. +type SpendReader interface { + SpendSince(ctx context.Context, sinceMs int64) (int64, error) +} + +// Guard answers whether the daily spend cap has been reached. The answer is +// cached: summing the ledger on every bundle would add a query per message for +// a value that changes slowly. +type Guard struct { + reader SpendReader + capMicros int64 + ttl time.Duration + now func() int64 + + mu sync.Mutex + cached bool + cachedAt int64 + lastValue bool +} + +// NewGuard returns a guard. capMicros <= 0 disables the cap entirely. +func NewGuard(r SpendReader, capMicros int64, ttl time.Duration, + now func() int64) *Guard { + return &Guard{reader: r, capMicros: capMicros, ttl: ttl, now: now} +} + +// SecurityOnly reports whether the gate should narrow to security lines only. +// +// It fails open. A ledger read error must not silently suppress detection: an +// over-run bill is visible and recoverable, whereas a gate quietly narrowed by +// a database hiccup hides real incidents with no operator signal. +func (g *Guard) SecurityOnly(ctx context.Context) bool { + if g.capMicros <= 0 { + return false + } + g.mu.Lock() + defer g.mu.Unlock() + + now := g.now() + if g.cached && now-g.cachedAt < g.ttl.Milliseconds() { + return g.lastValue + } + + dayStart := now - (24 * time.Hour).Milliseconds() + spent, err := g.reader.SpendSince(ctx, dayStart) + if err != nil { + slog.Error("budget: reading spend ledger failed, leaving the gate open", + "error", err) + return false + } + + breached := spent >= g.capMicros + if breached { + slog.Warn("budget: daily spend cap reached, narrowing gate to security only", + "spent_micros", spent, "cap_micros", g.capMicros) + } + g.cached, g.cachedAt, g.lastValue = true, now, breached + return breached +} +``` + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/budget/ -race -v` +Expected: PASS, all eleven tests, no race reports + +- [ ] **Step 5: Commit** + +```bash +git add internal/budget +git commit -m "feat(budget): LLM concurrency cap and daily spend ceiling" +``` + +--- + +## Task 10: `internal/analyzer` — the pipeline + +**Files:** +- Create: `internal/analyzer/analyzer.go` +- Test: `internal/analyzer/analyzer_test.go` + +**Interfaces:** +- Consumes: everything from Tasks 2–9. +- Produces: + - `analyzer.Config{Tolerance float64; StaleAfter time.Duration; EWMAAlpha float64; Model string; Pricing budget.Pricing}` + - `analyzer.New(s store.Store, c llm.Client, lim *budget.Limiter, g *budget.Guard, cfg Config, now func() int64) *Analyzer` + - `analyzer.Analyzer.Process(ctx, b model.Bundle) error` — returns `nil` on success or a gated-out bundle; a non-nil error means the caller must not commit the offset + - `analyzer.ErrPermanent` — sentinel wrapping a failure that must be dead-lettered rather than retried + +- [ ] **Step 1: Write the failing tests** + +Create `internal/analyzer/analyzer_test.go`: + +```go +package analyzer + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/nethesis/nethesis-insights/internal/budget" + "github.com/nethesis/nethesis-insights/internal/llm" + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/nethesis/nethesis-insights/internal/store" + "github.com/stretchr/testify/require" +) + +func f64(v float64) *float64 { return &v } + +const nominalReply = `{"window_assessment":"nominal","findings":[]}` + +const incidentReply = `{"window_assessment":"incident","findings":[ + {"severity":"high","title":"Traefik backend unreachable", + "summary":"Repeated connection refusals", + "suggested_action":"Check the backend service", + "modules":["traefik1"], + "evidence":["<3> [n1:traefik1:traefik] connection refused to :"]}]}` + +func openStore(t *testing.T) store.Store { + t.Helper() + s, err := store.Open("sqlite", t.TempDir()+"/a.db") + require.NoError(t, err) + t.Cleanup(func() { _ = s.Close() }) + require.NoError(t, s.Migrate(context.Background())) + return s +} + +func newAnalyzer(t *testing.T, s store.Store, c llm.Client, now int64) *Analyzer { + t.Helper() + return New(s, c, budget.NewLimiter(2), + budget.NewGuard(s, 0, time.Minute, func() int64 { return now }), + Config{ + Tolerance: 3.0, StaleAfter: 24 * time.Hour, EWMAAlpha: 0.3, + Model: "gpt-4o-mini", + Pricing: budget.Pricing{InputPerMTok: 0.15, OutputPerMTok: 0.60}, + }, + func() int64 { return now }) +} + +// novelBundle has a template no system has seen, so the gate opens. +func novelBundle() model.Bundle { + return model.Bundle{ + SchemaVersion: 1, SystemID: "sys1", CollectorVersion: "2.0.0", + Window: model.Window{Start: 1000, End: 1900}, + Digest: []model.DigestEntry{ + {ModuleID: "traefik1", Priority: 3, Observed: 37, Expected: f64(30)}, + }, + Templates: []model.Template{{ + Template: "<3> [n1:traefik1:traefik] connection refused to :", + Count: 37, ModuleID: "traefik1", Priority: 3, + }}, + } +} + +func TestGatedBundleNeverCallsTheLLM(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: nominalReply} + a := newAnalyzer(t, s, stub, 2000) + + // First pass opens the gate on novelty and records the template. + require.NoError(t, a.Process(ctx, novelBundle())) + require.Equal(t, 1, stub.Calls) + + // Second pass: same templates, same rates, nothing new. + b := novelBundle() + b.Window = model.Window{Start: 2000, End: 2900} + require.NoError(t, a.Process(ctx, b)) + require.Equal(t, 1, stub.Calls, "a steady-state bundle must cost nothing") +} + +func TestDuplicateWindowIsSkipped(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: nominalReply} + a := newAnalyzer(t, s, stub, 2000) + + require.NoError(t, a.Process(ctx, novelBundle())) + require.NoError(t, a.Process(ctx, novelBundle())) + require.Equal(t, 1, stub.Calls, "an edge retry must not be reprocessed") +} + +func TestFindingIsStoredWithServerComputedIdentity(t *testing.T) { + ctx, s := context.Background(), openStore(t) + a := newAnalyzer(t, s, &llm.Stub{Content: incidentReply, + Model: "gpt-4o-mini", InputTokens: 12400, OutputTokens: 300}, 2000) + require.NoError(t, a.Process(ctx, novelBundle())) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1) + require.Equal(t, "high", open[0].Severity) + require.Len(t, open[0].Fingerprint, 64) + require.Equal(t, "gpt-4o-mini", open[0].LLMModel) + require.NotEmpty(t, open[0].PromptVersion) +} + +func TestRepeatedIncidentBumpsRatherThanDuplicates(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: incidentReply, Model: "gpt-4o-mini"} + + // Two windows, both producing the same finding. The second only reaches + // the LLM because a fresh novel template is added. + require.NoError(t, newAnalyzer(t, s, stub, 2000).Process(ctx, novelBundle())) + + b := novelBundle() + b.Window = model.Window{Start: 2000, End: 2900} + b.Templates = append(b.Templates, model.Template{ + Template: "<4> [n1:samba2:smbd] new thing ", Count: 1, + ModuleID: "samba2", Priority: 4}) + require.NoError(t, newAnalyzer(t, s, stub, 3000).Process(ctx, b)) + require.Equal(t, 2, stub.Calls) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Len(t, open, 1, "the same insight must not be raised twice") + require.Equal(t, 2, open[0].OccurrenceCount) +} + +// This is the §7 correctness constraint, asserted directly. +func TestLLMFailureLeavesTemplatesUnrecorded(t *testing.T) { + ctx, s := context.Background(), openStore(t) + failing := &llm.Stub{Err: &llm.HTTPError{StatusCode: 503, Body: "down"}} + a := newAnalyzer(t, s, failing, 2000) + + err := a.Process(ctx, novelBundle()) + require.Error(t, err, "a retryable LLM failure must not be swallowed") + + known, err2 := s.KnownTemplates(ctx, "sys1") + require.NoError(t, err2) + require.Empty(t, known, + "recording templates before a successful analysis would make the "+ + "retry see them as known and lose the anomaly permanently") +} + +func TestPermanentLLMFailureIsNotRetryable(t *testing.T) { + ctx, s := context.Background(), openStore(t) + a := newAnalyzer(t, s, + &llm.Stub{Err: &llm.HTTPError{StatusCode: 400, Body: "bad schema"}}, 2000) + + err := a.Process(ctx, novelBundle()) + require.Error(t, err) + require.ErrorIs(t, err, ErrPermanent, + "a 400 must be dead-lettered, not retried forever") +} + +func TestUnparseableResponseIsPermanent(t *testing.T) { + ctx, s := context.Background(), openStore(t) + a := newAnalyzer(t, s, &llm.Stub{Content: "not json"}, 2000) + err := a.Process(ctx, novelBundle()) + require.Error(t, err) + require.ErrorIs(t, err, ErrPermanent) +} + +func TestLedgerRecordsGatedAndCalledRuns(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: nominalReply, InputTokens: 12400, OutputTokens: 300} + require.NoError(t, newAnalyzer(t, s, stub, 2000).Process(ctx, novelBundle())) + + spend, err := s.SpendSince(ctx, 0) + require.NoError(t, err) + require.Equal(t, int64(2040), spend, + "12400 input + 300 output at gpt-4o-mini prices is 2040 micro-dollars") +} + +func TestGatedRunSpendsNothing(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: nominalReply, InputTokens: 12400, OutputTokens: 300} + a := newAnalyzer(t, s, stub, 2000) + require.NoError(t, a.Process(ctx, novelBundle())) + + b := novelBundle() + b.Window = model.Window{Start: 2000, End: 2900} + require.NoError(t, a.Process(ctx, b)) + + spend, err := s.SpendSince(ctx, 0) + require.NoError(t, err) + require.Equal(t, int64(2040), spend, "the gated second run added nothing") +} + +func TestStaleFindingsAreMarkedAfterTheThreshold(t *testing.T) { + ctx, s := context.Background(), openStore(t) + stub := &llm.Stub{Content: incidentReply} + require.NoError(t, newAnalyzer(t, s, stub, 2000).Process(ctx, novelBundle())) + + // A later window, well past StaleAfter, in which the LLM reports nothing. + dayLater := int64(2000) + (25 * time.Hour).Milliseconds() + b := novelBundle() + b.Window = model.Window{Start: 500_000, End: 501_000} + b.Templates = append(b.Templates, model.Template{ + Template: "another new one ", Count: 1, ModuleID: "m9", Priority: 4}) + require.NoError(t, newAnalyzer(t, s, + &llm.Stub{Content: nominalReply}, dayLater).Process(ctx, b)) + + open, err := s.OpenFindings(ctx, "sys1") + require.NoError(t, err) + require.Empty(t, open, "a finding absent for over StaleAfter goes stale") + + stale, err := s.ListFindings(ctx, "sys1", 0, model.StatusStale) + require.NoError(t, err) + require.Len(t, stale, 1) +} + +func TestSystemIsRegisteredOnFirstBundle(t *testing.T) { + ctx, s := context.Background(), openStore(t) + a := newAnalyzer(t, s, &llm.Stub{Content: nominalReply}, 2000) + require.NoError(t, a.Process(ctx, novelBundle())) + // Registration is what lets an operator see a node exists before it has + // ever produced a finding. + known, err := s.KnownTemplates(ctx, "sys1") + require.NoError(t, err) + require.NotEmpty(t, known) +} + +func TestStoreErrorIsRetryable(t *testing.T) { + ctx := context.Background() + s := openStore(t) + require.NoError(t, s.Close()) // force every query to fail + a := newAnalyzer(t, s, &llm.Stub{Content: nominalReply}, 2000) + + err := a.Process(ctx, novelBundle()) + require.Error(t, err) + require.NotErrorIs(t, err, ErrPermanent, + "a closed database is transient; the message must be redelivered") +} + +func TestConcurrentProcessingIsSafe(t *testing.T) { + ctx, s := context.Background(), openStore(t) + a := newAnalyzer(t, s, &llm.Stub{Content: nominalReply}, 2000) + + errs := make(chan error, 8) + for i := range 8 { + go func(i int) { + b := novelBundle() + b.Window = model.Window{Start: int64(i) * 1000, End: int64(i)*1000 + 900} + errs <- a.Process(ctx, b) + }(i) + } + for range 8 { + require.NoError(t, <-errs) + } +} + +var _ = errors.New // keep the import if unused after edits +``` + +- [ ] **Step 2: Run them to make sure they fail** + +Run: `go test ./internal/analyzer/ -v` +Expected: FAIL — `undefined: New` + +- [ ] **Step 3: Write the implementation** + +Create `internal/analyzer/analyzer.go`: + +```go +// Package analyzer runs the bundle pipeline: gate, infer, fingerprint, store. +package analyzer + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/nethesis/nethesis-insights/internal/budget" + "github.com/nethesis/nethesis-insights/internal/fingerprint" + "github.com/nethesis/nethesis-insights/internal/gate" + "github.com/nethesis/nethesis-insights/internal/llm" + "github.com/nethesis/nethesis-insights/internal/model" + "github.com/nethesis/nethesis-insights/internal/prompt" + "github.com/nethesis/nethesis-insights/internal/store" +) + +// ErrPermanent marks a failure that will recur identically on retry. The +// consumer dead-letters these instead of redelivering forever. +var ErrPermanent = errors.New("permanent failure") + +// Config holds the analyzer's tunables. +type Config struct { + Tolerance float64 + StaleAfter time.Duration + EWMAAlpha float64 + Model string + Pricing budget.Pricing +} + +// Analyzer processes one bundle at a time and is safe for concurrent use. +type Analyzer struct { + store store.Store + llm llm.Client + lim *budget.Limiter + guard *budget.Guard + cfg Config + now func() int64 +} + +// New wires an analyzer. now is injected so tests control time. +func New(s store.Store, c llm.Client, lim *budget.Limiter, g *budget.Guard, + cfg Config, now func() int64) *Analyzer { + return &Analyzer{store: s, llm: c, lim: lim, guard: g, cfg: cfg, now: now} +} + +// Process runs the pipeline for one bundle. +// +// A nil return means the offset may be committed — including for a bundle that +// was gated out or was a duplicate window. A non-nil return means it may not. +// Wrapping ErrPermanent additionally means retrying is pointless. +func (a *Analyzer) Process(ctx context.Context, b model.Bundle) error { + now := a.now() + started := time.Now() + + // 1. Idempotency. A duplicate window is success, not an error: the edge + // retried and the work is already done. + fresh, err := a.store.BeginAnalysis(ctx, b.SystemID, + b.Window.Start, b.Window.End, now) + if err != nil { + return fmt.Errorf("analyzer: begin analysis: %w", err) + } + if !fresh { + slog.Debug("analyzer: duplicate window ignored", + "system_id", b.SystemID, "window_start", b.Window.Start) + return nil + } + + if err := a.store.UpsertSystem(ctx, store.System{ + SystemID: b.SystemID, CollectorVersion: b.CollectorVersion, + FirstSeen: now, LastSeen: now, + }); err != nil { + return fmt.Errorf("analyzer: upsert system: %w", err) + } + + // 2. Read state BEFORE recording anything, or every template looks known. + known, err := a.store.KnownTemplates(ctx, b.SystemID) + if err != nil { + return fmt.Errorf("analyzer: known templates: %w", err) + } + baselines, err := a.store.Baselines(ctx, b.SystemID) + if err != nil { + return fmt.Errorf("analyzer: baselines: %w", err) + } + + // 3. Gate. + decision := gate.Evaluate(b, gate.SystemState{ + KnownTemplates: known, + Baselines: baselines, + SecurityOnly: a.guard.SecurityOnly(ctx), + }, a.cfg.Tolerance) + + // 4. Gated out: record the decision and stop. No LLM cost. + if !decision.Call { + if err := a.record(ctx, b, store.Analysis{ + SystemID: b.SystemID, WindowStart: b.Window.Start, + Gated: true, GateReasons: decision.Reasons, + DurationMs: int(time.Since(started).Milliseconds()), + }, now); err != nil { + return err + } + return nil + } + + // 5-6. Render and infer, under the concurrency cap. + open, err := a.store.OpenFindings(ctx, b.SystemID) + if err != nil { + return fmt.Errorf("analyzer: open findings: %w", err) + } + userPrompt := prompt.Render(b, open) + + if err := a.lim.Acquire(ctx); err != nil { + return fmt.Errorf("analyzer: acquire llm slot: %w", err) + } + resp, llmErr := a.llm.Complete(ctx, llm.Request{ + Model: a.cfg.Model, UserPrompt: userPrompt, + }) + a.lim.Release() + + if llmErr != nil { + // Record the failure in the ledger before returning, so a repeatedly + // failing system is visible rather than merely absent. + _ = a.store.FinalizeAnalysis(ctx, store.Analysis{ + SystemID: b.SystemID, WindowStart: b.Window.Start, + Gated: false, GateReasons: decision.Reasons, LLMCalled: true, + Error: truncate(llmErr.Error(), 500), + DurationMs: int(time.Since(started).Milliseconds()), + }) + var he *llm.HTTPError + if errors.As(llmErr, &he) && he.Permanent() { + return fmt.Errorf("analyzer: %w: %w", ErrPermanent, llmErr) + } + return fmt.Errorf("analyzer: llm: %w", llmErr) + } + + // 7. Parse. A malformed response will parse identically on retry. + parsed, assessment, err := prompt.Parse(resp.Content) + if err != nil { + _ = a.store.FinalizeAnalysis(ctx, store.Analysis{ + SystemID: b.SystemID, WindowStart: b.Window.Start, + GateReasons: decision.Reasons, LLMCalled: true, + InputTokens: resp.InputTokens, OutputTokens: resp.OutputTokens, + CostMicros: a.cfg.Pricing.CostMicros(resp.InputTokens, resp.OutputTokens), + Model: resp.Model, + Error: truncate(err.Error(), 500), + DurationMs: int(time.Since(started).Milliseconds()), + }) + return fmt.Errorf("analyzer: %w: %w", ErrPermanent, err) + } + + // 8. Store findings under server-computed identity. + for _, pf := range parsed { + category := categoryFor(b, pf.Evidence) + fp := fingerprint.Compute(b.SystemID, pf.Modules, pf.Evidence, category) + outcome, err := a.store.UpsertFinding(ctx, model.Finding{ + SystemID: b.SystemID, Fingerprint: fp, + Severity: pf.Severity, Title: pf.Title, Summary: pf.Summary, + SuggestedAction: pf.SuggestedAction, + Modules: pf.Modules, Evidence: pf.Evidence, + LLMModel: resp.Model, PromptVersion: prompt.Version, + }, now) + if err != nil { + return fmt.Errorf("analyzer: upsert finding: %w", err) + } + if outcome != store.OutcomeBumped { + slog.Info("analyzer: finding raised", + "system_id", b.SystemID, "outcome", string(outcome), + "severity", pf.Severity, "title", pf.Title) + } + } + + // 9. Only now is it safe to record templates and baselines. Doing this + // earlier would make a retry after an LLM failure see them as known. + if err := a.record(ctx, b, store.Analysis{ + SystemID: b.SystemID, WindowStart: b.Window.Start, + Gated: false, GateReasons: decision.Reasons, LLMCalled: true, + InputTokens: resp.InputTokens, OutputTokens: resp.OutputTokens, + CostMicros: a.cfg.Pricing.CostMicros(resp.InputTokens, resp.OutputTokens), + Model: resp.Model, + DurationMs: int(time.Since(started).Milliseconds()), + }, now); err != nil { + return err + } + + slog.Debug("analyzer: window analysed", "system_id", b.SystemID, + "assessment", assessment, "findings", len(parsed)) + return nil +} + +// record commits the state a successful (or gated) pass produces: templates, +// baselines, staleness, and the ledger row. +func (a *Analyzer) record(ctx context.Context, b model.Bundle, + entry store.Analysis, now int64) error { + if err := a.store.UpsertTemplates(ctx, b.SystemID, b.Templates, now); err != nil { + return fmt.Errorf("analyzer: upsert templates: %w", err) + } + if err := a.store.UpsertBaselines(ctx, b.SystemID, b.Digest, + a.cfg.EWMAAlpha); err != nil { + return fmt.Errorf("analyzer: upsert baselines: %w", err) + } + if _, err := a.store.MarkStale(ctx, b.SystemID, + now-a.cfg.StaleAfter.Milliseconds()); err != nil { + return fmt.Errorf("analyzer: mark stale: %w", err) + } + if err := a.store.FinalizeAnalysis(ctx, entry); err != nil { + return fmt.Errorf("analyzer: finalize analysis: %w", err) + } + return nil +} + +// categoryFor propagates the edge's classification: security if any cited +// template was flagged security. The server never classifies anything itself. +func categoryFor(b model.Bundle, evidence []string) string { + for _, e := range evidence { + if b.CategoryOf(e) == "security" { + return "security" + } + } + return "" +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] +} +``` + +- [ ] **Step 4: Run the tests and make sure they pass** + +Run: `go test ./internal/analyzer/ -race -v` +Expected: PASS, all fourteen tests. Remove the `var _ = errors.New` line from the test file if `errors` is genuinely used by then. + +- [ ] **Step 5: Run the whole suite and the linter** + +Run: `make check` +Expected: `license headers OK`, lint clean, every package PASS. + +- [ ] **Step 6: Commit** + +```bash +git add internal/analyzer +git commit -m "feat(analyzer): gate, infer, fingerprint and store pipeline" +``` + +--- + +## Plan 1 complete + +At this point the entire analysis pipeline works and is tested offline: a `model.Bundle` in, findings in SQLite out, with gating, deduplication and a cost ledger. Nothing is reachable over the network yet. + +**Plan 2** ([2026-08-05-nethesis-insights-transport.md](2026-08-05-nethesis-insights-transport.md)) adds the transport and delivery layer: `internal/queue`, `internal/auth`, `internal/ingest`, `internal/api`, `internal/maint`, `cmd/insightsd` wiring, the container, the load smoke test, and the draft PR. diff --git a/docs/superpowers/specs/2026-07-29-loki-anomaly-detector-design.md b/docs/superpowers/specs/2026-07-29-loki-anomaly-detector-design.md new file mode 100644 index 0000000..bfaf8bc --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-loki-anomaly-detector-design.md @@ -0,0 +1,399 @@ +# Loki anomaly detector — design + +Date: 2026-07-29 +Repository: ns8-loki +Status: approved design, not yet implemented + +## Goal + +Detect anomalies in NethServer 8 journal logs by sending a compact, +scrubbed representation of each hour's logs to a remote LLM and recording +its findings back into the journal. + +The detector ships as an extension of the existing `ns8-loki` module, +alongside `syslog-forwarder` and `cloud-log-manager-forwarder`. It is +disabled by default and requires explicit configuration. + +## Motivation and constraints + +Measured on a live single-node cluster (`rl1`, 9 modules) over one hour: + +| Metric | Count | +|--------|-------| +| Total journal lines | 1317 (~32k/day) | +| Lines with `PRIORITY <= 4` | 82 | +| Lines with `category="security"` | 268 | +| Top talker | `nethvoice2` (656) | + +A raw hourly dump is roughly 40k input tokens per hour, near 1M tokens per +day per node. Busier nodes are worse by an order of magnitude. The design +therefore sends a digest plus a capped set of prefiltered lines, targeting +4–6k tokens per window. + +Loki labels available for selection: `node_id`, `module_id`, `category`, +`job`, `service_name`. Journal fields reachable through the `json` stage: +`PRIORITY`, `SYSLOG_IDENTIFIER`, `MESSAGE`. + +## Architecture + +### New files + +``` +imageroot/bin/anomaly-detector # the whole job, one Python script +imageroot/systemd/user/anomaly-detector.service # Type=oneshot +imageroot/systemd/user/anomaly-detector.timer # OnCalendar=hourly +imageroot/actions/set-anomaly-detector/validate-input.json +imageroot/actions/set-anomaly-detector/10set +``` + +### Modified files + +- `imageroot/actions/get-configuration/10get` — expose detector state +- `imageroot/actions/get-configuration/validate-output.json` — new object +- `imageroot/etc/state-include.conf` — add `state/secrets.env` +- `imageroot/update-module.d/10config` — install the new units +- `README.md` — configuration, manual test, privacy statement + +### Process shape + +A `systemd` timer fires hourly and starts a `Type=oneshot` service that +analyses one window and exits. The window derives from the wall clock, so +there is no cursor file and no drift. `Persistent=true` recovers a single +window missed across a reboot. A crashed run costs exactly one window; the +next fire retries. + +``` +[Unit] +Description=Loki anomaly detector +Requires=loki-server.service +After=loki-server.service + +[Service] +Type=oneshot +EnvironmentFile=%E/state/environment +EnvironmentFile=-%E/state/secrets.env +ExecStart=runagent %E/bin/anomaly-detector +SyslogIdentifier=%u/%N +``` + +Unit files use `%E`, matching the convention established by #68. + +`SyslogIdentifier=%u/%N` is mandatory, not cosmetic. The log collector +(Alloy, configured by `core/imageroot/var/lib/nethserver/node/bin/generate-promtail-config` +in ns8-core) assigns the `module_id` label only when the module name appears +in `_SYSTEMD_UNIT`, `SYSLOG_IDENTIFIER` or `CONTAINER_NAME`. A rootless user +unit reports `_SYSTEMD_UNIT=user@.service`, which contains no module +name, so without an explicit identifier the detector's own output would be +ingested unlabeled and the recall query in stage 3 would never match it. +`%u/%N` expands to `loki1/anomaly-detector`, which satisfies the label rule +and doubles as a stable selector. The same convention is already used by +five units in other NS8 modules. + +The timer, not the service, is enabled and disabled by the action. + +## Data flow + +One run performs six stages. + +### 1. Window + +`[hour_start, hour_start + 1h)` computed from the wall clock, where +`hour_start` is the start of the previous full hour. `--since` overrides +this for manual runs. + +### 2. Collect + +Three `logcli` invocations, all selecting `{node_id=~".+"}` so a single +detector on the Loki node covers the whole cluster: + +- **Digest** — `sum by (module_id, priority) (count_over_time({node_id=~".+"} | json priority="PRIORITY" [1h]))` +- **Baseline** — the same query over `[7d]`, divided by 168, giving an + expected hourly rate per `(module_id, priority)` pair +- **Lines** — `logcli query --forward -o jsonl` over the window, selecting + `PRIORITY < 5 or category="security"`, with `--limit` set to + `ANOMALY_MAX_LINES` (default 500) + +Loki stores the entire journal record as the line, so every query needs a +`| json` stage to extract `MESSAGE`, `PRIORITY` and `SYSLOG_IDENTIFIER`, +then a `line_format` to render them, exactly as +`cloud-log-manager-forwarder` does today. + +The line query must exclude the detector's own identifier: + +``` +| identifier != "/anomaly-detector" +``` + +Without this the detector feeds on itself. Diagnostics written to stderr +land in the journal at `PRIORITY=3`, which the `PRIORITY < 5` prefilter +would collect on the next run, so a single failure would be re-analysed +every hour and the evidence lines of past findings would re-enter the +prompt as fresh input. + +`logcli` requires `LOKI_ADDR`, `LOKI_USERNAME` and `LOKI_PASSWORD`, derived +from `LOKI_HTTP_PORT` and `LOKI_API_AUTH_*` exactly as +`cloud-log-manager-forwarder` does today. + +### 3. Recall own findings + +A fourth query over the last 24h selects the detector's own past output: + +``` +{module_id=""} | json identifier="SYSLOG_IDENTIFIER" + | identifier="/anomaly-detector" +``` + +The last 10 finding titles enter the prompt so the LLM can suppress +repeats instead of re-reporting them. Because findings are emitted to the +journal and the collector ships the journal to Loki, the detector's memory +is self-hosted: no state file, nothing extra to back up. + +### 4. Scrub + +Every collected line and every rendered prompt block passes through an +ordered regex list before leaving the process: + +| Pattern | Replacement | +|---------|-------------| +| `(?i)(bearer\|token\|api[-_]?key\|secret\|password\|passwd\|pwd)[=:\s"']+\S+` | `\1=` | +| `(?i)authorization:\s*\S+\s*\S*` | `authorization: ` | +| base64 or hex runs of 32 or more characters | `` | +| email addresses | `` | + +IP addresses, hostnames, module IDs and usernames appearing in message +text are deliberately preserved: they carry the anomaly signal. + +The scrub is defence in depth, not a guarantee. The real privacy boundary +is that log text is sent to a third-party API, and the README states this +plainly. + +### 5. Ask + +A single `POST` to `${ANOMALY_LLM_BASE_URL}/chat/completions` using the +OpenAI-compatible chat completions shape, so any of OpenAI, OpenRouter, +vLLM, Ollama or a self-hosted gateway works with one code path. +`temperature: 0` and `response_format: {"type": "json_schema", ...}` to +force a machine-checkable answer. + +The system message pins the role: an NS8 cluster log analyst judging this +hour against the supplied baseline, reporting only actionable deviations, +never restating a recalled finding unless it has escalated. + +The user message carries four fenced blocks: + +- `WINDOW` — start and end timestamps, and whether the line cap truncated + the window +- `RATES` — per `(module_id, priority)`: observed this hour vs expected per + hour +- `RECENT_FINDINGS` — last 10 titles with severity +- `LINES` — scrubbed, formatted ` [node:module:identifier] message` + +### 6. Emit + +Response schema: + +```json +{ + "findings": [ + { + "severity": "critical|high|medium|low", + "title": "short, stable, dedup-able", + "summary": "what happened and why it deviates from baseline", + "evidence": ["verbatim scrubbed log lines that justify it"], + "modules": ["nethvoice2"], + "suggested_action": "what an admin should check" + } + ], + "window_assessment": "nominal|degraded|incident" +} +``` + +An empty `findings` array is the normal outcome. + +Each finding is written as one JSON line to stdout, plus one summary line +carrying `window_assessment`. Under systemd, stdout is the journal, so +findings are indexed by the collector into Loki with +`SYSLOG_IDENTIFIER=/anomaly-detector` and become queryable and +graphable like any other log. When the script is run by hand the same lines appear on +the terminal — one code path, no mode divergence. + +If `ANOMALY_WEBHOOK_URL` is set, the same JSON is POSTed there, with an +optional bearer token. Journald is the source of truth; the webhook is +best-effort delivery. + +Diagnostics go exclusively to stderr, so the `identifier="anomaly-detector"` +Loki query returns findings and nothing else. + +Truncation is never silent: if the line cap trims the window, that fact +enters both the prompt and a journald notice on stderr. + +## Configuration + +One action, `set-anomaly-detector`, following the `oneOf` on `active` shape +already used by `set-clm-forwarder`. + +```json +{ + "active": true, + "base_url": "https://api.openai.com/v1", + "model": "gpt-4o-mini", + "api_key": "sk-...", + "max_lines": 500, + "webhook_url": "https://example.org/hook", + "webhook_token": "..." +} +``` + +`base_url`, `model` and `api_key` are required when `active` is `true`. +`max_lines`, `webhook_url` and `webhook_token` are optional. + +### Where each value is stored + +Secrets go to `state/secrets.env` and never to the `environment` file, +because `environment` is mirrored into the Redis hash +`module//environment` and would be readable by anything able to +read that hash. The pattern follows ns8-dependencytrack. + +| Variable | Location | +|----------|----------| +| `ANOMALY_LLM_API_KEY` | `state/secrets.env` | +| `ANOMALY_WEBHOOK_TOKEN` | `state/secrets.env` | +| `ANOMALY_LLM_BASE_URL` | `environment` | +| `ANOMALY_LLM_MODEL` | `environment` | +| `ANOMALY_WEBHOOK_URL` | `environment` | +| `ANOMALY_MAX_LINES` | `environment` | + +`secrets.env` is written with `agent.read_envfile` / `agent.write_envfile`, +merging rather than overwriting so unrelated keys survive, followed by an +explicit `os.chmod(0o600)` — `safe_open` preserves the mode of an existing +file but a freshly created one should not depend on the umask. + +`state/secrets.env` is added to `etc/state-include.conf` so restore keeps +the key. The Restic repository is encrypted. + +### Enabling and disabling + +`active: true` writes the configuration, then +`systemctl --user enable --now anomaly-detector.timer`. Re-running while +active only rewrites the configuration; the timer needs no restart because +the oneshot reads its environment at each fire. + +`active: false` runs `systemctl --user disable --now anomaly-detector.timer`, +unsets the `environment` variables, and removes both keys from +`secrets.env` rather than leaving them behind. + +### get-configuration + +A new `anomaly_detector` object: + +| Field | Source | +|-------|--------| +| `status` | `systemctl --user is-active anomaly-detector.timer` mapped to `active`/`failed`/`inactive` | +| `base_url`, `model`, `max_lines`, `webhook_url` | `environment` | +| `api_key_configured` | boolean, key presence in `secrets.env` | +| `last_run` | `systemctl --user show anomaly-detector.service -p ExecMainExitTimestamp` | + +The API key value is never returned. + +## Manual execution + +The script is also the CLI, via `argparse` with every flag optional, so +systemd invokes it with none. `runagent -m loki1` already exports +`LOKI_HTTP_PORT` and `LOKI_API_AUTH_*`, so log collection works on any +existing machine with no install, no unit, and nothing written to module +state. + +``` +scp imageroot/bin/anomaly-detector root@node:/tmp/ +ssh root@node runagent -m loki1 python3 /tmp/anomaly-detector --dry-run --since 2h +``` + +| Flag | Effect | +|------|--------| +| `--dry-run` | collect, digest, scrub and render the prompt, print it with a character and approximate token count, make no LLM call, emit no findings | +| `--since 2h` | window becomes `[now-2h, now]`; accepts `30m`, `6h`, `2d` | +| `--config FILE` | read `ANOMALY_*` from a plain env file instead of `environment` and `secrets.env` | +| `--pretty` | render findings on stdout as indented human-readable text instead of JSON lines | +| `--no-webhook` | skip webhook delivery | +| `--max-lines N` | override the line cap for one run | +| `--print-prompt` | print the prompt alongside a real LLM call | + +Precedence: CLI flag, then `--config` file, then shell environment, then +`secrets.env`, then `environment`. + +Typical sequence on an existing machine: + +``` +# 1. see what would be sent — no key, no cost +runagent -m loki1 python3 /tmp/anomaly-detector --dry-run --since 2h + +# 2. real call, findings on the terminal, no webhook delivery +ANOMALY_LLM_BASE_URL=https://api.openai.com/v1 \ +ANOMALY_LLM_MODEL=gpt-4o-mini \ +ANOMALY_LLM_API_KEY=sk-... \ + runagent -m loki1 python3 /tmp/anomaly-detector --since 2h --pretty --no-webhook +``` + +The manual flags add behaviour; they never alter the default path taken by +the systemd unit. + +## Error handling + +A oneshot failure costs one window, so the rule is to fail loudly, exit +non-zero, and let the next timer fire retry. No internal retry loops except +at the HTTP layer. + +| Failure | Behaviour | +|---------|-----------| +| `logcli` exits non-zero or exceeds its 300s timeout | log stderr, exit 1; no LLM call and no cost | +| zero prefiltered lines and rates match baseline | log `nominal, no LLM call`, exit 0 | +| LLM returns 429 or 5xx | `urllib3` `Retry(total=3, backoff_factor=2, status_forcelist=[429,500,502,503,504])`, then exit 1 | +| LLM returns 401 or 403 | log `check API key`, exit 1, no retry | +| response does not match the schema | log the body truncated to 500 characters, exit 1, emit no partial finding | +| webhook delivery fails | findings are already in the journal; log the error and exit 1 so the failure is visible | +| `secrets.env` missing while the timer is enabled | log `not configured`, exit 1 | + +Skipping the LLM call on an idle window is a meaningful saving on quiet +nodes, where most hours produce nothing worth analysing. + +## Testing + +### Unit tests + +`scrub(line)`, `build_digest(rates, baseline)`, `render_prompt(...)` and +`parse_findings(body)` are top-level functions with no I/O, tested with +`pytest` in a container in the style of `core/agent/test-agent.sh` in +ns8-core. Each scrub row gets a positive and a negative case, plus a test +asserting that an IP address and a module ID survive scrubbing. + +### Robot test + +`tests/20__anomaly_detector.robot`: + +1. Start a local stub HTTP server returning a canned findings response in + the OpenAI chat completions shape. +2. Call `set-anomaly-detector` pointing `base_url` at the stub. +3. Run `systemctl --user start anomaly-detector.service`. +4. Assert the finding appears in the journal and in Loki under + `identifier="/anomaly-detector"` — this also proves the + `SyslogIdentifier` setting produced the `module_id` label. +5. Assert `get-configuration` reports `api_key_configured: true` and never + echoes the key. + +No real LLM in CI: no network egress, no cost, deterministic. + +### Manual verification + +A one-shot run against a real node with a real key, to judge whether the +findings are useful. The stub-based test proves plumbing only; prompt +quality can only be assessed against real logs. + +## Out of scope + +- Vue UI. `ns8-loki` has no UI beyond `ui/index.html`, and findings are + queryable through Loki and deliverable by webhook. +- Redis-backed findings history. The journal round-trip through the log + collector already provides it. +- Email or cluster notification delivery. +- Local placeholder mapping and re-hydration of redacted values. +- Configurable interval. Hourly is fixed; only the line cap is tunable. diff --git a/docs/superpowers/specs/2026-08-05-nethesis-insights-design.md b/docs/superpowers/specs/2026-08-05-nethesis-insights-design.md new file mode 100644 index 0000000..84d6028 --- /dev/null +++ b/docs/superpowers/specs/2026-08-05-nethesis-insights-design.md @@ -0,0 +1,585 @@ +# Nethesis Insights — Design + +**Date:** 2026-08-05 +**Status:** Approved design, pending implementation plan +**Project:** `nethesis-insights` — new repo, Go +**Related:** [2026-07-29 Loki Anomaly Detector Design](2026-07-29-loki-anomaly-detector-design.md) + +## 1. Context + +The anomaly detector shipped in `ns8-loki` runs entirely on the edge node. One +Python job (`imageroot/bin/anomaly-detector`, 787 lines) queries Loki, builds a +digest, fetches log lines, scrubs them, calls an LLM directly, and emits findings +to journald. Every node holds an LLM API key and every node pays for its own +inference. + +That design does not survive contact with a fleet. Three problems: + +- **Credential sprawl.** An OpenAI or OpenRouter key on 2700 customer machines + is 2700 places for it to leak, and no way to rotate centrally. +- **No cross-window memory.** The edge recalls prior findings from its own + journal, so the same insight is re-raised whenever the journal rolls or the + node reboots. +- **Uncontrolled cost.** Each node decides independently to spend money. There + is no fleet-wide view, no ceiling, and no way to suppress duplicate spend. + +This spec moves analysis to a central server. The edge is reduced to collection: +filter, deduplicate, ship. The server owns inference, identity, dedup and cost. + +## 2. Scope + +**In scope:** the Go server — ingest, authentication, queueing, gating, LLM +call, finding identity and dedup, storage, read API, container packaging. + +**Out of scope, deliberately:** + +- **Edge collector v2** (filtering, template masking, per-module fairness, + digest-driven selection). Separate spec. The edge keeps calling the LLM + directly until the server is live; see §12 for the cutover. +- **Operator dashboard / UI.** The read API is the only consumer surface here. +- **Operator-wide (cross-system) queries.** The API is per-system only. +- **Finding acknowledgement / mutation.** The API is read-only. + +The edge is out of scope but not unconstrained: §4 defines the protocol the +edge must produce, and §6 explains why the edge's template masking is +load-bearing for server-side dedup. + +## 3. Architecture + +### 3.1 Topology + +One container, three processes, supervised by `s6-overlay`. + +``` + ┌──────────── container ─────────────┐ +edge (2700 nodes) │ │ + bundle/15min ──────┼─► ingest HTTP ─► topic: bundles ──┐ │ + POST /v1/bundles │ (auth) (48h retention) │ │ + │ ▼ │ + GET /v1/findings ─┼─◄─ read API ◄─ SQLite ◄── analyzer │ + │ │ │ + └────────────────────────────┼───────┘ + ▼ + OpenAI / OpenRouter +``` + +`s6-overlay` rather than a bash `&` because the container needs correct signal +forwarding, PID-1 zombie reaping, and independent restart of Redpanda without +killing the Go binary. Redpanda runs single-node: `--smp 1 --memory 1G +--overprovisioned`. + +### 3.2 Sizing + +2700 systems × 4 bundles/hour = **3 requests/second**. Redpanda is not present +for throughput — at this rate a database table would serve. It is present for +durable buffering, replay from offset, and decoupling ingest latency from +multi-second LLM calls. Sizing the deployment for a higher load would be +sizing for a load that does not exist. + +Redpanda's practical floor is ~1–2 GB RAM even idle; budget for it. + +### 3.3 Packages + +| Package | Purpose | Depends on | +|---|---|---| +| `cmd/insightsd` | wiring, config, graceful shutdown | all | +| `internal/model` | `Bundle`, `Digest`, `Template`, `Finding` types | — | +| `internal/auth` | forward-auth client + TTL cache | — | +| `internal/ingest` | HTTP handler: validate, produce | `auth`, `queue` | +| `internal/queue` | franz-go produce/consume, interface-typed | — | +| **`internal/gate`** | **pure**: `(Bundle, SystemState) → Decision` | — | +| `internal/llm` | OpenAI-compatible client, strict `json_schema` | — | +| **`internal/fingerprint`** | **pure**: `Finding → stable hash` | — | +| `internal/analyzer` | consume → gate → llm → store | all above | +| `internal/store` | `bun` queries + migrations, sole schema owner | — | +| `internal/api` | read handlers | `auth`, `store` | +| `internal/maint` | daily pruning job | `store` | + +The two packages carrying correctness and cost — `gate` and `fingerprint` — are +pure functions with no I/O, so they are table-driven tests with no fixtures. +`llm`, `queue` and `store` are interfaces, so `analyzer` is testable end-to-end +against stubs with no container running. + +### 3.4 Storage: SQLite now, external database later + +`internal/store` is an interface with a `sqliteStore` implementation today and a +`pgStore` implementation when an external database is required. Query code is +written once using **`uptrace/bun`**, which is dialect-aware and thin over +`database/sql`, so the SQL stays visible in review. + +Alternatives rejected: `sqlc` would require two query sets to keep in sync, +which is the exact portability tax being avoided; `GORM` generates opaque SQL, +making write-path behaviour hard to verify under load; `ent` is heavyweight for +six tables. + +**Portability rules the schema follows from day one** — cheap now, expensive to +retrofit: + +- IDs generated in Go (ULID). Never `AUTOINCREMENT` or `SERIAL`. +- Timestamps stored as `INTEGER` unix-millis. Never native date types. +- `ON CONFLICT … DO UPDATE` only. Never `INSERT OR REPLACE`. +- JSON held as `TEXT` and parsed in Go. No `jsonb` operators, no SQLite `json1`. +- Migrations via `golang-migrate`, one dialect-agnostic SQL directory. + +Concurrency differs per implementation, and the interface hides it: `sqliteStore` +runs WAL mode with `busy_timeout=5000` and a single writer goroutine owning all +writes; `pgStore` uses a normal connection pool. Callers see one interface. + +## 4. Authentication + +The server never stores or verifies secrets. The edge sends +`Authorization: Basic base64(system_id:secret)`, sourced from the node's +existing NethServer subscription identity (the `cluster/subscription` Redis +hash). `internal/auth` forwards the credential to an external validator, the +Traefik `forwardAuth` pattern: + +``` +ingest ──► $AUTH_VALIDATE_URL (Authorization header forwarded verbatim) + ◄── 200 → valid; tenant/org id captured if returned + ◄── 401/403 → reject + ◄── other/timeout → treat as unavailable (see fail-closed below) +``` + +Four requirements: + +- **Caching is mandatory, not an optimization.** 3 req/s uncached is ~10,800 + validator calls per hour for credentials that essentially never change. + Positive TTL ~5 min, negative TTL ~30 s to blunt credential-stuffing + amplification. +- **Cache keys are `HMAC(pepper, system_id + ":" + secret)`.** The raw secret is + never stored, never logged, never written to the database. +- **Fail closed.** If the validator is unreachable and there is no cache hit, + respond `503`. Failing open on an auth path would let anyone write into + another tenant's stream. The cost of failing closed is an ingestion gap the + edge retries — recoverable. The cost of failing open is not. +- **Bind the identity.** The `system_id` in the request body must equal the + authenticated `system_id`, or reject `403`. Otherwise a node holding valid + credentials can attribute bundles to another system. + +`internal/auth` therefore has no `store` dependency: an HTTP client plus a TTL +cache, stubbable in tests. + +## 5. Wire protocol + +`POST /v1/bundles` · `Authorization: Basic` · `Content-Encoding: gzip` · +body cap 1 MB · → `202 Accepted` + +```json +{ + "schema_version": 1, + "system_id": "abc123", + "collector_version": "2.0.0", + "masking_version": 1, + "window": { "start": 1754380800000, "end": 1754381700000 }, + "digest": [ + { "module_id": "traefik1", "priority": 3, + "observed": 42, "expected": 3.2, "ratio": 13.1 } + ], + "templates": [ + { "template": "<3> [n1:traefik1:traefik] connection refused to :", + "count": 37, "module_id": "traefik1", "priority": 3, + "category": "security", + "first_seen": 1754380811000, "last_seen": 1754381690000, + "samples": ["<3> [n1:traefik1:traefik] connection refused to 10.0.0.4:8080"] } + ], + "budget": { + "max_lines": 500, "lines_seen": 4210, "lines_kept": 500, + "truncated_modules": [ { "module_id": "traefik1", "dropped": 3200 } ] + } +} +``` + +### 5.1 Template identity is the template text + +There is no template hash. `system_templates` is keyed on +`(system_id, template)` and the novelty gate is an indexed lookup. Templates are +~150 bytes; 2700 systems × ~500 distinct templates ≈ 200 MB. + +This is a deliberate simplification over hashing. A hash column buys nothing at +this scale and costs readability: `SELECT template FROM system_templates WHERE +system_id = ?` tells an operator what a system looks like, which a hash never +does. + +`masking_version` is recorded metadata only — it computes nothing. Its purpose +is diagnostic: when a collector upgrade changes the masking rules, every +template's text changes, so every template looks novel and every fingerprint +changes, producing a one-time duplicate-insight burst across the fleet. The +recorded version makes that burst explainable rather than mysterious. See §9.3 +for why this is also a thundering-herd risk. + +### 5.2 Idempotency + +The key is `(system_id, window.start)`. Edge retries after a 5xx are certain; +without this a retry double-counts the digest and re-raises findings. A +duplicate window returns `200 {"duplicate": true}`. + +### 5.3 Drop statistics, not a boolean + +`budget.truncated_modules` reports which modules lost lines and how many: + +```json +{ "module_id": "traefik1", "dropped": 3200, "truncated": true } +``` + +The gate reads it: truncation alone is noise, but truncation *plus* deviation +means an incident is being under-sampled, which is a stronger signal than +either alone. + +`truncated` is carried separately from `dropped` because the two can disagree. +`dropped` is derived from the digest, and the digest is the query that fails on +a busy cluster. When it is unavailable the collector still knows it hit the +line cap but cannot say by how much, so it reports `dropped: 0` with +`truncated: true`. Without the flag, that module would be indistinguishable +from a healthy one — read as nominal precisely when the cluster is busiest. + +### 5.3.1 The host bucket + +`module_id` is a stream label present only on module streams. Every host-level +journal record — `sshd`, `systemd`, `runagent` — carries no `module_id` at all. +Verified on a live cluster: `label/module_id/values` returned only +`['ldapproxy1', 'loki1', 'metrics1', 'traefik1']`, while the SSH traffic that +dominates the security signal appeared under no module at all. + +Those records travel with `module_id: ""`, which the server must treat as an +ordinary module for baselines, allocation and findings. It is a real bucket, +not a missing value, and rejecting an empty `module_id` would discard the most +security-relevant stream on the node. + +### 5.4 Validation before produce + +Never poison the topic. Rejected with `400`, logged with `system_id`, not +retried: + +- unknown `schema_version` +- `window.end - window.start` outside tolerance +- window in the future, or `window.start` older than 6 hours +- `templates` longer than 1000 +- more than 2 `samples` per template +- compressed body over 1 MB, or decompressed body over 8 MB (decompression is + bounded by `io.LimitReader`, so a zip bomb is rejected rather than buffered) + +The 6-hour acceptance window is what gives the edge room to retry across +several failed cycles without the server rejecting recovered data. + +### 5.5 Topics + +| Topic | Key | Partitions | Retention | Purpose | +|---|---|---|---|---| +| `bundles` | `system_id` | 12 | 48 h, zstd | ingest → analyzer, per-system ordering | +| `bundles.dlq` | `system_id` | 1 | 7 d | permanently failed bundles | + +Consumer group `analyzer`, manual commit **after** the store write — +at-least-once delivery, made harmless by the `(system_id, window_start)` +uniqueness constraint. + +### 5.6 Read API + +`GET /v1/findings?since=&status=` — scoped to the +authenticated `system_id`. Returns findings sorted severity-descending, then +`last_seen` descending. + +## 6. Data model + +Six tables. + +| Table | Key | Contents | +|---|---|---| +| `systems` | `system_id` PK | tenant_id, collector_version, first_seen, last_seen | +| `system_templates` | UNIQUE `(system_id, template)` | first_seen, last_seen, total_count | +| `module_baselines` | UNIQUE `(system_id, module_id, priority)` | ewma_rate, updated_at | +| `findings` | UNIQUE `(system_id, fingerprint)` | severity, title, summary, suggested_action, modules (JSON TEXT), evidence (JSON TEXT), status, occurrence_count, first_seen, last_seen, reopened_at, llm_model, prompt_version | +| `analyses` | UNIQUE `(system_id, window_start)` | window_end, gated, gate_reasons (JSON TEXT), llm_called, input_tokens, output_tokens, cost_micros, model, duration_ms, error | +| `schema_migrations` | — | `golang-migrate` state | + +There is no `bundles` table. Per §10, bundles are not persisted: they live only +in the topic under 48-hour retention. `analyses` records that a window was +processed and what it cost; the digest is absorbed into `module_baselines`. + +### 6.1 Why a server-side baseline exists + +The edge sends `expected` from a Loki metric query that is known to fail on busy +clusters — the `max_query_series` limit, handled by graceful degradation in +`ns8-loki` commit `4b6971f`. When the edge degrades, `expected` is absent and a +deviation gate relying on it goes blind exactly when the cluster is busiest. + +`module_baselines` holds a server-computed EWMA over received `observed` counts. +Edge `expected` is preferred when present; server EWMA is the fallback. + +### 6.2 Finding identity + +One hash, in one place, `internal/fingerprint`: + +```go +// A finding's identity is the set of evidence templates it cites. +sha256("v1\x00" + system_id + "\x00" + module_id + "\x00" + category + "\x00" + + strings.Join(sortedEvidenceTemplates, "\x1f")) +``` + +A variable-length set of templates needs a fixed-width dedup key, so this hash +is earned where the template hash was not. + +The `v1` prefix is also earned: if the formula ever changes, every existing +finding's identity changes with it, and that must be a deliberate versioned +migration rather than a silent re-raise of every open insight on the fleet. + +Identity is computed **server-side from evidence**, never from model-authored +text. An inconsistently worded restatement of a known problem therefore +collapses onto the same fingerprint. + +### 6.3 Finding lifecycle + +``` +open ──(no recurrence for 24 h / 96 windows)──► stale +stale ──(recurrence)──► open, reopened_at stamped +``` + +Recurrence never inserts a row. It bumps `last_seen` and `occurrence_count`. +Consumers treat a reopen as alert-worthy and a bump as not. There is no +`acknowledged` state — the API is read-only — but the `status` column is a +string enum so one can be added without a schema change. + +## 7. Analyzer pipeline + +``` +consume bundle + 1. INSERT analyses(system_id, window_start) → conflict? commit offset, done + 2. read known templates + baselines → compute novel set, deviations + 3. gate(bundle, state) → Decision + 4. gated out? → record decision, commit offset, done ← zero LLM cost + 5. build prompt (deterministic assembly, §8.2) + 6. LLM call (strict json_schema, no temperature) + 7. parse, validate, sort severity-descending + 8. per finding: fingerprint → INSERT, or bump/reopen + 9. upsert system_templates + module_baselines ← only now +10. mark absent open findings stale +11. finalize analyses row (tokens, cost, model), commit offset +``` + +**Step ordering is a correctness requirement, not style.** Two constraints: + +- Novelty must be read (step 2) before templates are recorded, or every + template looks known and the gate never fires. +- Templates must be written only after a successful analysis (step 9). If the + LLM call fails at step 6 with templates already recorded, the retry sees them + as known, the gate declines, and the anomaly is lost permanently. Deferring + the write makes an LLM failure a clean nack-and-retry. + +## 8. Gating and inference + +### 8.1 The gate + +A pure function. It calls the LLM if **any** condition holds: + +- a template is new for this `system_id` +- a digest ratio exceeds tolerance (default 3.0), from edge `expected` or + server EWMA +- any template carries `category=security` (edge-assigned in the bundle; the + server does not classify) +- a module appears in `truncated_modules` **and** deviates + +Every decision records `gate_reasons` in `analyses`, so both "why did this cost +money" and "why was this not caught" are answerable from stored data. + +The gate is the primary cost control, not an optimization. At 15-minute +cadence, 2700 systems calling the LLM on every bundle is ~$16,000/month on +`gpt-4o-mini` (§11). Steady-state systems must cost approximately zero. + +### 8.2 Consistency of model output + +Five levers: + +1. `prompt_version` is a constant in code, stamped on every finding. +2. Strict `response_format: json_schema` with `strict: true`. **No + `temperature` field** — some models reject any non-default value outright, + as seen in `ns8-loki` commit `6ef8fd0`. +3. Deterministic prompt assembly: templates sorted by + `(module_id, priority, template)`, digest sorted likewise. Identical input + produces byte-identical prompts. +4. Currently-open findings are included in the prompt with an explicit + instruction to report only new or changed conditions. +5. Identity is server-computed (§6.2), so consistency of *wording* is not + relied upon for dedup — only consistency of *evidence*. + +## 9. Error handling + +### 9.1 Ingest + +| Condition | Response | Edge behaviour | +|---|---|---| +| validation failure | `400` | do not retry | +| `system_id` mismatch | `403` | do not retry | +| invalid credentials | `401` | do not retry | +| validator unreachable, no cache hit | `503` | retry with backoff | +| Redpanda produce failure | `503` | retry with backoff | +| duplicate window | `200 {"duplicate": true}` | treat as success | + +### 9.2 Analyzer + +| Condition | Action | +|---|---| +| LLM `4xx` (bad request, schema rejection) | DLQ immediately — deterministic, retry cannot help | +| LLM `429` / `5xx` / timeout | retry with exponential backoff, max 5, then DLQ | +| response parse or schema-validation failure | retry once, then DLQ | +| store write failure | nack, redeliver | +| SQLite busy | `busy_timeout` then bounded retry | + +Every DLQ message carries the failure reason and the original bundle. A +non-empty DLQ is an operational alert, not a normal state. + +### 9.3 Thundering herd + +Two events can make the whole fleet's templates novel at the same moment: a +fleet-wide collector upgrade that changes masking rules, and a `prompt_version` +or fingerprint-formula change. Either would put all 2700 bundles of a single +window (10,800/hour) through the gate with the novelty condition satisfied for +every one. + +Three defences, all required: + +- **Global LLM concurrency cap** (`LLM_MAX_CONCURRENCY`). The analyzer is one + consumer group; excess work waits in the topic, which is what the topic is + for. +- **Daily spend ceiling** (`LLM_DAILY_SPEND_CAP_USD`) computed from the + `analyses` cost ledger. On breach, the gate degrades to + security-category-only and logs loudly. Degraded is better than a surprise + invoice, and better than silence. +- **Per-system ingest rate limit** — roughly 10 bundles/hour burst, so one + misbehaving node cannot flood the topic. + +### 9.4 Degradation summary + +The system is designed so each dependency failure costs one capability, not the +run: + +| Failure | Consequence | +|---|---| +| edge metric query fails | server EWMA covers the deviation gate (§6.1) | +| validator down | ingestion pauses, edge retries within the 6 h window | +| LLM provider down | bundles accumulate in the topic, analysed on recovery | +| spend cap hit | gate narrows to security only | +| Redpanda restart | s6 restarts it; ingest returns 503 meanwhile | + +## 10. Data protection + +Log lines now leave customer premises. The persisted surface is deliberately +minimal: + +- The edge scrubs before shipping (existing `SCRUB_RULES` in + `imageroot/bin/anomaly-detector`) and masks lines to templates. +- The server persists **only** template text, counts, digests and findings. +- Representative raw `samples` exist only in the `bundles` topic under 48-hour + retention and are **never** copied into the database. +- Secrets (`LLM_API_KEY`, `AUTH_PEPPER`) come from the environment, are never + written to the database, and are never logged. +- The auth cache stores credential HMACs, never secrets (§4). + +## 11. Cost model + +Measured input is ~12.4k tokens per call at `max_lines=500`, ~300 output tokens +blended. At 15-minute cadence that is 96 calls/day per system, 2920/month. + +| Model | Per call | Per system/month | 2700 systems/month | +|---|---|---|---| +| `gpt-4o-mini` | ~$0.0020 | ~$5.96 | **~$16,100** | +| `gpt-4o` | ~$0.034 | ~$99 | ~$268,000 | + +Those are ungated upper bounds. The gate (§8.1) is what makes the number real: +steady-state systems should gate out almost every bundle, and template +deduplication at the edge shrinks per-call input for exactly the noisiest +windows. + +`gpt-4o-mini` is the recommended tier. The task is bounded, schema-constrained +log classification — the same complexity class already validated against free +models on OpenRouter. + +The OpenRouter Batch API was evaluated and rejected: its documentation +advertises no discount versus synchronous calls, and its 24-hour completion +window is incompatible with 15-minute detection latency. + +## 12. Configuration + +| Variable | Purpose | +|---|---| +| `LISTEN_ADDR` | ingest + read API bind address | +| `AUTH_VALIDATE_URL` | external forward-auth endpoint | +| `AUTH_CACHE_TTL`, `AUTH_NEG_CACHE_TTL` | positive/negative cache lifetimes | +| `AUTH_PEPPER` | HMAC pepper for cache keys — secret | +| `DB_DRIVER` | `sqlite` or `postgres` | +| `DB_DSN` | connection string | +| `REDPANDA_BROKERS`, `TOPIC_BUNDLES`, `TOPIC_DLQ` | queue wiring | +| `LLM_BASE_URL`, `LLM_MODEL`, `LLM_API_KEY` | provider — key is secret | +| `LLM_MAX_CONCURRENCY` | global inference cap | +| `LLM_DAILY_SPEND_CAP_USD` | spend ceiling, gate degrades on breach | +| `GATE_TOLERANCE` | deviation ratio threshold, default 3.0 | +| `STALE_AFTER` | finding staleness threshold, default 24 h | +| `LOG_LEVEL` | — | + +`PROMPT_VERSION` is a code constant, not configuration. It must change with the +prompt, and an environment variable would let the two drift apart. + +## 13. Testing + +**Unit** — no I/O, table-driven: + +- `gate`: every condition in isolation and in combination; absent `expected` + falling back to EWMA; truncation with and without deviation +- `fingerprint`: stability under evidence reordering; distinctness across + systems, modules and categories +- protocol validation: each `400` condition +- prompt assembly: golden files proving byte-identical output for identical + input +- `auth`: cache hit/miss, negative caching, fail-closed on validator error + +**Integration** — stub `llm` via `httptest`, temp-file SQLite, in-memory queue: + +- gated-out bundle writes an `analyses` row and never calls the LLM +- new finding inserts; recurrence bumps `occurrence_count` without inserting +- absence past `STALE_AFTER` marks stale; later recurrence reopens with + `reopened_at` +- LLM failure leaves templates unrecorded, so the retry still sees them as + novel — the §7 correctness constraint, asserted directly +- duplicate `(system_id, window_start)` is idempotent + +**Container** — real Redpanda via compose: post bundles with `curl`, assert +findings through the read API. + +**Migration** — run `golang-migrate` against both SQLite and Postgres in CI, so +the portability rules in §3.4 are enforced by the build rather than by memory. + +**Load smoke** — sustain 3 req/s for several minutes; assert no SQLite +`database is locked` errors and no consumer lag growth. + +## 14. Cutover + +| Phase | Edge | Server | +|---|---|---| +| 1 | unchanged, calls LLM directly | built and deployed, receiving nothing | +| 2 | `anomaly-detector` **replaced** by `insights-collector` | sole analysis path | + +There is no `ANOMALY_MODE` and no dual-running phase. The edge is replaced +outright: the LLM call, prompt rendering, findings parsing, webhook and +`recall_findings()` are deleted from the node, and with them the API key. + +Keeping both paths alive would mean every prompt or schema change landing in +two places — a Python edge implementation and a Go server one — which is how +they drift into two different definitions of a finding. It would also leave the +LLM API key on 2700 machines, the credential-sprawl problem in §1 that this +work exists to remove. + +The cost of replacement is a window where a node ships to a server that may not +yet be reachable. That is absorbed by the collector's spool (§5.2 idempotency +plus the 6-hour acceptance window), and by the collector doing nothing at all +until `INSIGHTS_SERVER_URL` is configured. + +Phase 2 depends on the edge collector spec, which is separate work. The server +is useful and testable before that spec exists, which is why it is built first. + +## 15. Items to verify during planning + +- **Exact field names in the `cluster/subscription` Redis hash.** The module + reads it today (`imageroot/actions/set-clm-forwarder/10set:16`) but only tests + truthiness, so the `system_id` / secret field names are unconfirmed. Must be + read from a live node. +- **The external validator's contract**: endpoint, method, response codes, and + whether it returns a tenant or organisation identifier the server can use for + scoping. diff --git a/imageroot/actions/get-configuration/10get b/imageroot/actions/get-configuration/10get index d0d99d0..543258c 100755 --- a/imageroot/actions/get-configuration/10get +++ b/imageroot/actions/get-configuration/10get @@ -65,13 +65,38 @@ try: except: syslog["last_timestamp"] = "" +# Insights + +insights = {} + +insights_status = subprocess.run(['systemctl', '--user', 'is-active', 'insights-collector.timer'], capture_output=True, text=True) +match insights_status.stdout.strip(): + case 'active': + insights["status"] = 'active' + case 'failed': + insights["status"] = 'failed' + case _: + insights["status"] = 'inactive' + +insights["base_url"] = os.getenv('INSIGHTS_SERVER_URL', '') +insights["verify_tls"] = os.getenv('INSIGHTS_VERIFY_TLS', '1') != '0' + +# This is what tells the UI why an enabled collector ships nothing: the +# collector reads identity from cluster/subscription at run time and has no +# module secret of its own. +insights["subscription_configured"] = bool(rdb.hgetall('cluster/subscription')) + +insights_last_run = subprocess.run(['systemctl', '--user', 'show', 'insights-collector.service', '-p', 'ExecMainExitTimestamp', '--value'], capture_output=True, text=True) +insights["last_run"] = insights_last_run.stdout.strip() + # General response = { "retention_days": int(os.getenv('LOKI_RETENTION_PERIOD')), "active_from": os.getenv('LOKI_ACTIVE_FROM'), "cloud_log_manager": cloud_log_manager, - "syslog": syslog + "syslog": syslog, + "insights": insights } if os.getenv('LOKI_ACTIVE_TO') is not None: diff --git a/imageroot/actions/get-configuration/validate-output.json b/imageroot/actions/get-configuration/validate-output.json index 18f178f..f50f215 100644 --- a/imageroot/actions/get-configuration/validate-output.json +++ b/imageroot/actions/get-configuration/validate-output.json @@ -8,7 +8,8 @@ "retention_days", "active_from", "cloud_log_manager", - "syslog" + "syslog", + "insights" ], "properties": { "retention_days": { @@ -58,7 +59,6 @@ "description": "Log filter type." } } - }, "syslog": { "type": "object", @@ -96,6 +96,42 @@ "description": "Log filter type." } } + }, + "insights": { + "type": "object", + "title": "Insights collector", + "description": "State of the collector that ships deduplicated log bundles to the nethesis-insights service.", + "required": [ + "status", + "subscription_configured" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "active", + "failed", + "inactive" + ], + "description": "State of insights-collector.timer." + }, + "base_url": { + "type": "string", + "description": "Base URL of the nethesis-insights server that receives the bundles." + }, + "verify_tls": { + "type": "boolean", + "description": "Whether the server TLS certificate is verified." + }, + "subscription_configured": { + "type": "boolean", + "description": "True when cluster/subscription holds identity data. An enabled collector with no subscription ships nothing." + }, + "last_run": { + "type": "string", + "description": "ExecMainExitTimestamp of insights-collector.service, empty if never run." + } + } } } } diff --git a/imageroot/actions/restore-module/06copyenv b/imageroot/actions/restore-module/06copyenv index c77cc6a..f78a82e 100755 --- a/imageroot/actions/restore-module/06copyenv +++ b/imageroot/actions/restore-module/06copyenv @@ -21,3 +21,14 @@ for evar in [ # NOTE: LOKI_ACTIVE_TO is restored by a later step ]: agent.set_env(evar, original_environment[evar]) + +# Timer enablement is not part of module state, so 95insights re-enables the +# insights-collector timer from these values after a restore. Without them a +# previously-configured collector would come back silently disabled. These +# are optional, because insights is disabled by default. +for evar in [ + "INSIGHTS_SERVER_URL", + "INSIGHTS_VERIFY_TLS", + ]: + if evar in original_environment: + agent.set_env(evar, original_environment[evar]) diff --git a/imageroot/actions/restore-module/95insights b/imageroot/actions/restore-module/95insights new file mode 100755 index 0000000..5f83215 --- /dev/null +++ b/imageroot/actions/restore-module/95insights @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# +# Resume the insights collector after a restore. +# +# Timer enablement is not module state, so without this step a restored node +# comes back with the collector configured but never firing. +# + +import os +import subprocess +import sys + +if not os.getenv('INSIGHTS_SERVER_URL'): + sys.exit(0) + +subprocess.run(["systemctl", "--user", "enable", "--now", "insights-collector.timer"], + stdout=sys.stderr, + stderr=sys.stderr, + text=True, + check=True) diff --git a/imageroot/actions/set-insights/10set b/imageroot/actions/set-insights/10set new file mode 100755 index 0000000..534640c --- /dev/null +++ b/imageroot/actions/set-insights/10set @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import json +import subprocess +import sys + +import agent + +request = json.load(sys.stdin) + +if request['active']: + # Unlike set-clm-forwarder, a missing subscription does not abort the + # action: the collector re-reads cluster/subscription from Redis at each + # fire, so a subscription registered later starts working with no + # reconfiguration here. + subscription = agent.redis_connect().hgetall('cluster/subscription') + if not subscription: + print("No subscription found: bundles will not ship until the node has a subscription", + file=sys.stderr) + + agent.set_env('INSIGHTS_SERVER_URL', request['base_url'].rstrip('/')) + agent.set_env('INSIGHTS_VERIFY_TLS', '1' if request.get('verify_tls', True) else '0') + action = 'enable' +else: + agent.munset_env(['INSIGHTS_SERVER_URL', 'INSIGHTS_VERIFY_TLS']) + action = 'disable' + +# The unit files may be new to this installation +subprocess.run(["systemctl", "--user", "daemon-reload"], + stdout=sys.stderr, + stderr=sys.stderr, + text=True, + check=True) + +# Enable or disable the timer, never the oneshot service. Re-running while +# active only rewrites the environment: the oneshot reads its environment +# at each fire, so the timer needs no restart. +subprocess.run(["systemctl", "--user", action, "--now", "insights-collector.timer"], + stdout=sys.stderr, + stderr=sys.stderr, + text=True, + check=True) diff --git a/imageroot/actions/set-insights/validate-input.json b/imageroot/actions/set-insights/validate-input.json new file mode 100644 index 0000000..f4d46df --- /dev/null +++ b/imageroot/actions/set-insights/validate-input.json @@ -0,0 +1,40 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "http://schema.nethserver.org/loki/set-insights.json", + "title": "Configure the insights collector", + "description": "Configure the collector that ships deduplicated log bundles to the nethesis-insights service.", + "type": "object", + "properties": { + "active": { + "type": "boolean", + "description": "Enable or disable the insights collector timer." + }, + "base_url": { + "type": "string", + "format": "uri", + "description": "Base URL of the nethesis-insights server that receives the bundles." + }, + "verify_tls": { + "type": "boolean", + "default": true, + "description": "Verify the server TLS certificate. Disable only when pointing at a self-signed test server." + } + }, + "oneOf": [ + { + "properties": { + "active": {"enum": [true]} + }, + "required": [ + "active", + "base_url" + ] + }, + { + "properties": { + "active": {"enum": [false]} + }, + "required": ["active"] + } + ] +} diff --git a/imageroot/bin/insights-collector b/imageroot/bin/insights-collector new file mode 100755 index 0000000..a145a8f --- /dev/null +++ b/imageroot/bin/insights-collector @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Collect one window of logs, deduplicate to templates, ship a bundle. + +This node does no analysis. It filters, masks, counts and sends; the server +gates, infers and stores. +""" + +import argparse +import base64 +import datetime +import json +import os +import ssl +import sys +import urllib.error +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "pypkg")) + +from insights import bundle as bundle_mod # noqa: E402 +from insights import select # noqa: E402 +from insights.loki import Client, HOST_BUCKET, LokiError # noqa: E402 + +COLLECTOR_VERSION = "2.0.0" +DEFAULT_MAX_LINES = 500 +WINDOW_MINUTES = 15 +BASELINE_HOURS = 168 +DEVIATION_TOLERANCE = 3.0 + +# Structurally noisy but benign patterns, filtered at the Loki query so they +# never travel. Ships empty: curating it needs real fleet data, not a guess. +DENYLIST = [] + +# Field names in the cluster/subscription Redis hash. Pending confirmation +# against a live node; the secret fallback list exists only until then. +SUBSCRIPTION_ID_FIELD = "system_id" +SUBSCRIPTION_SECRET_FIELDS = ("auth_token", "secret", "password") + + +def read_identity(): + """(system_id, secret) from the cluster subscription, or (None, None). + + Identity is not module configuration: it is the node's existing + NethServer subscription, so a configured collector cannot be pointed at + a different tenant by editing module state. + """ + import agent # deferred: unavailable in the unit-test container + + raw = agent.redis_connect().hgetall("cluster/subscription") + fields = {} + for key, value in raw.items(): + if isinstance(key, bytes): + key = key.decode() + if isinstance(value, bytes): + value = value.decode() + fields[key] = value + + system_id = fields.get(SUBSCRIPTION_ID_FIELD) or None + secret = None + for name in SUBSCRIPTION_SECRET_FIELDS: + value = fields.get(name) + if value: + secret = value + break + + if not system_id or not secret: + print("cluster/subscription is missing required fields; present: {0}".format( + sorted(fields.keys())), file=sys.stderr) + return None, None + + return system_id, secret + + +def compute_window(now, minutes=WINDOW_MINUTES): + end = now.replace(second=0, microsecond=0) + end -= datetime.timedelta(minutes=end.minute % minutes) + return end - datetime.timedelta(minutes=minutes), end + + +def ms(when): + return int(when.timestamp() * 1000) + + +def collect(client, window, max_lines, self_identifier, system_id): + start, end = window + modules = client.module_ids(start, end) + + window_seconds = (end - start).total_seconds() + observed = client.digest(end, window_seconds) + baseline = client.digest(end, BASELINE_HOURS * 3600) + digest_available = bool(observed) + + # Digest rows, and the set of modules that deviate from baseline. + digest_rows, deviating = [], set() + for (module_id, priority), count in sorted(observed.items()): + row = {"module_id": module_id, "priority": priority, + "observed": int(count)} + base = baseline.get((module_id, priority)) + if base: + expected = base / BASELINE_HOURS * (window_seconds / 3600.0) + if expected > 0: + row["expected"] = round(expected, 3) + if count / expected > DEVIATION_TOLERANCE: + deviating.add(module_id) + digest_rows.append(row) + + shares = select.allocate(modules, max_lines, prioritised=deviating) + + templates, truncated = [], [] + lines_seen = lines_kept = 0 + for module_id in modules: + share = shares.get(module_id, select.MIN_SHARE) + limit = select.fetch_limit(share) + try: + forward = client.lines(module_id, start, end, limit, + DENYLIST, self_identifier, "forward") + except LokiError as exc: + print("module {0}: {1}".format(module_id or "", exc), + file=sys.stderr) + continue + + hit_cap = len(forward) >= limit + if hit_cap: + # Only now pay for a second page, so a late-developing incident + # is not hidden behind an early burst. + try: + backward = client.lines(module_id, start, end, limit // 2, + DENYLIST, self_identifier, "backward") + except LokiError: + backward = [] + page = bundle_mod.sample_ends(forward, backward, limit) + else: + page = forward + + module_templates, kept, fetched = bundle_mod.group_templates( + page, module_id, share) + templates.extend(module_templates) + lines_seen += fetched + lines_kept += kept + + if hit_cap: + total = sum(int(c) for (m, _p), c in observed.items() if m == module_id) + truncated.append({ + "module_id": module_id, + "dropped": max(0, total - fetched) if digest_available else 0, + # Without this flag a module that hit the cap during a digest + # outage reports dropped=0 and is indistinguishable from a + # healthy one -- read by the server as nominal. + "truncated": True, + }) + + budget = {"max_lines": max_lines, "lines_seen": lines_seen, + "lines_kept": lines_kept} + if truncated: + budget["truncated_modules"] = truncated + + return bundle_mod.build( + system_id, + COLLECTOR_VERSION, (ms(start), ms(end)), digest_rows, templates, budget) + + +def ship(payload, url, system_id, secret, verify=True, timeout=60): + body = json.dumps(payload).encode() + request = urllib.request.Request(url.rstrip("/") + "/v1/bundles", data=body) + request.add_header("Content-Type", "application/json") + token = base64.b64encode("{0}:{1}".format(system_id, secret).encode()).decode() + request.add_header("Authorization", "Basic " + token) + if verify: + ctx = ssl.create_default_context() + else: + # Self-signed test servers only; verification is on by default. + ctx = ssl._create_unverified_context() + with urllib.request.urlopen(request, timeout=timeout, context=ctx) as response: + return response.status, response.read().decode() + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--max-lines", type=int, default=DEFAULT_MAX_LINES) + parser.add_argument("--minutes", type=int, default=WINDOW_MINUTES) + parser.add_argument("--print", dest="show", action="store_true", + help="write the bundle to stdout instead of shipping") + args = parser.parse_args(argv) + + try: + port = os.environ["LOKI_HTTP_PORT"] + client = Client("http://127.0.0.1:" + port, + os.environ["LOKI_API_AUTH_USERNAME"], + os.environ["LOKI_API_AUTH_PASSWORD"]) + except KeyError as exc: + print("missing Loki variable {0}; run under runagent".format(exc), + file=sys.stderr) + return 1 + + module_id = os.environ.get("MODULE_ID", "loki1") + window = compute_window(datetime.datetime.now(datetime.timezone.utc), + args.minutes) + + if args.show: + try: + payload = collect(client, window, args.max_lines, + "{0}/insights-collector".format(module_id), + "unknown") + except LokiError as exc: + print("collection failed: {0}".format(exc), file=sys.stderr) + return 1 + json.dump(payload, sys.stdout, indent=2) + print() + return 0 + + # Both cheap checks run before the Loki queries: an unconfigured node + # should cost nothing per fire, not a full collection it then discards. + url = os.environ.get("INSIGHTS_SERVER_URL") + if not url: + print("INSIGHTS_SERVER_URL is not set", file=sys.stderr) + return 1 + verify = os.environ.get("INSIGHTS_VERIFY_TLS", "").strip().lower() not in ( + "0", "false", "no", "off") + + system_id, secret = read_identity() + if not system_id: + print("no subscription found; cannot ship bundles", file=sys.stderr) + return 1 + + try: + payload = collect(client, window, args.max_lines, + "{0}/insights-collector".format(module_id), + system_id) + except LokiError as exc: + print("collection failed: {0}".format(exc), file=sys.stderr) + return 1 + + try: + status, text = ship(payload, url, system_id, secret, verify=verify) + except urllib.error.HTTPError as exc: + print("ship failed {0}: {1}".format(exc.code, + exc.read().decode()[:300]), file=sys.stderr) + return 1 + except urllib.error.URLError as exc: + print("ship failed: {0}".format(exc.reason), file=sys.stderr) + return 1 + except TimeoutError: + # A hang after the request is fully sent surfaces here, not as + # URLError: urllib only wraps send-phase failures, not response-read + # timeouts on an already-open connection. + print("ship failed: timed out waiting for a response", file=sys.stderr) + return 1 + + print("shipped {0} templates, {1} lines -> {2} {3}".format( + len(payload["templates"]), payload["budget"]["lines_kept"], status, text.strip())) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/imageroot/events/subscription-changed/10check_forwarder b/imageroot/events/subscription-changed/10check_forwarder index a379a21..65ddf12 100755 --- a/imageroot/events/subscription-changed/10check_forwarder +++ b/imageroot/events/subscription-changed/10check_forwarder @@ -17,7 +17,18 @@ if data['action'] == 'terminated': agent.unset_env('CLOUD_LOG_MANAGER_ADDRESS') agent.unset_env('CLOUD_LOG_MANAGER_TENANT') - subprocess.run(["systemctl", "--user", "disable", "--now", "cloud-log-manager-forwarder.service"], + subprocess.run(["systemctl", "--user", "disable", "--now", "cloud-log-manager-forwarder.service"], stdout=sys.stderr, stderr=sys.stderr, check=True) + + # A terminated subscription would otherwise leave a timer firing every 15 + # minutes against an identity that no longer validates. + agent.munset_env(['INSIGHTS_SERVER_URL', 'INSIGHTS_VERIFY_TLS']) + + # check=False: the timer may never have been enabled, and that must not + # fail the event handler. + subprocess.run(["systemctl", "--user", "disable", "--now", "insights-collector.timer"], + stdout=sys.stderr, + stderr=sys.stderr, + check=False) diff --git a/imageroot/pypkg/insights/__init__.py b/imageroot/pypkg/insights/__init__.py new file mode 100644 index 0000000..6d70fef --- /dev/null +++ b/imageroot/pypkg/insights/__init__.py @@ -0,0 +1,5 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Edge collection library for the Nethesis Insights bundle producer.""" diff --git a/imageroot/pypkg/insights/bundle.py b/imageroot/pypkg/insights/bundle.py new file mode 100644 index 0000000..e68cbbb --- /dev/null +++ b/imageroot/pypkg/insights/bundle.py @@ -0,0 +1,93 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Turn collected lines into the wire bundle.""" + +import collections +import re + +from . import select +from .masking import MASKING_VERSION, mask +from .scrub import sanitize_line + +SCHEMA_VERSION = 1 + +# The identifier is carried in the rendered line; the priority prefix is +# parsed back out so templates can be grouped by it. +PRIORITY_RE = re.compile(r'^<(\d+)>') + + +def group_templates(lines, module_id, share): + """Deduplicate raw lines into counted templates. + + `lines` is [(ts_ms, text, category)]. Returns (templates, kept, fetched). + Templates are emitted most-frequent-first and truncated to `share`, so a + module's budget buys distinct events rather than repetitions of one. + """ + groups = collections.OrderedDict() + for ts, raw, category in lines: + clean = sanitize_line(raw) + template = mask(clean) + match = PRIORITY_RE.match(clean) + priority = int(match.group(1)) if match else 6 + key = (template, priority, category) + entry = groups.get(key) + if entry is None: + groups[key] = { + "template": template, "count": 1, + "module_id": module_id, "priority": priority, + "category": category, + "first_seen": ts, "last_seen": ts, + "samples": [clean], + } + else: + entry["count"] += 1 + entry["first_seen"] = min(entry["first_seen"], ts) + entry["last_seen"] = max(entry["last_seen"], ts) + # Keep the first and last raw line only; the server caps at 2. + if len(entry["samples"]) == 1: + entry["samples"].append(clean) + else: + entry["samples"][1] = clean + + ordered = sorted(groups.values(), key=lambda t: (-t["count"], t["template"])) + kept = ordered[:share] + for entry in kept: + if not entry.get("category"): + entry.pop("category", None) + return kept, sum(t["count"] for t in kept), len(lines) + + +def sample_ends(forward, backward, limit): + """Merge a forward and a backward page, keeping both ends of the window. + + A pure first-N page lets an early burst hide a late-developing incident. + """ + seen = set() + merged = [] + for row in list(forward) + list(backward): + key = (row[0], row[1]) + if key in seen: + continue + seen.add(key) + merged.append(row) + merged.sort() + if len(merged) <= limit: + return merged + half = limit // 2 + return merged[:half] + merged[-(limit - half):] + + +def build(system_id, collector_version, window, digest_rows, templates, budget): + """Assemble the wire payload.""" + return { + "schema_version": SCHEMA_VERSION, + "system_id": system_id, + "collector_version": collector_version, + "masking_version": MASKING_VERSION, + "window": {"start": window[0], "end": window[1]}, + "digest": digest_rows, + "templates": templates, + "budget": budget, + } diff --git a/imageroot/pypkg/insights/loki.py b/imageroot/pypkg/insights/loki.py new file mode 100644 index 0000000..d1a08ec --- /dev/null +++ b/imageroot/pypkg/insights/loki.py @@ -0,0 +1,122 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Loki HTTP client. + +Everything goes over the HTTP API. logcli is deliberately not used: the +two-pass design issues one query per module per run, and a subprocess spawn +per module is both slower and harder to diagnose than a status code. +""" + +import base64 +import datetime +import json +import urllib.error +import urllib.parse +import urllib.request + +TIMEOUT = 60 + +# Host-level journal records (sshd, systemd, runagent) carry no module_id +# label at all, so the label-values endpoint never lists them. They are +# collected under this synthetic bucket instead. Measured on a live cluster, +# these lines are the majority of security-relevant traffic, so dropping them +# would defeat the point of the collector. +HOST_BUCKET = "" + + +class LokiError(RuntimeError): + pass + + +class Client: + def __init__(self, base_url, username, password, timeout=TIMEOUT): + self._base = base_url.rstrip("/") + self._auth = base64.b64encode( + "{0}:{1}".format(username, password).encode()).decode() + self._timeout = timeout + + def _get(self, path, params): + url = "{0}{1}?{2}".format(self._base, path, urllib.parse.urlencode(params)) + request = urllib.request.Request(url) + request.add_header("Authorization", "Basic " + self._auth) + try: + with urllib.request.urlopen(request, timeout=self._timeout) as response: + return json.load(response) + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", "replace")[:500] + raise LokiError("{0} {1}: {2}".format(path, exc.code, body)) from exc + except urllib.error.URLError as exc: + raise LokiError("{0}: {1}".format(path, exc.reason)) from exc + + @staticmethod + def _ns(when): + return str(int(when.timestamp() * 1e9)) + + def module_ids(self, start, end): + """Module IDs present in the window, plus the host bucket. + + The label endpoint reads the index rather than series data, so unlike + a metric query it is not subject to max_query_series. + """ + payload = self._get("/loki/api/v1/label/module_id/values", + {"start": self._ns(start), "end": self._ns(end)}) + return sorted(payload.get("data") or []) + [HOST_BUCKET] + + def digest(self, at, range_seconds): + """{(module_id, priority): count} over the range ending at `at`. + + Returns {} on failure. A busy cluster can exceed max_query_series on + this aggregation, and losing the digest costs prioritisation and the + `expected` field, not the run. + """ + query = ('sum by (module_id, priority) ' + '(count_over_time({{node_id=~".+"}} | json priority="PRIORITY" [{0}s]))' + ).format(int(range_seconds)) + try: + payload = self._get("/loki/api/v1/query", + {"query": query, "time": at.isoformat()}) + except LokiError: + return {} + out = {} + for series in (payload.get("data") or {}).get("result") or []: + labels = series.get("metric") or {} + try: + priority = int(labels.get("priority", -1)) + value = float(series["value"][1]) + except (KeyError, IndexError, TypeError, ValueError): + continue + out[(labels.get("module_id", HOST_BUCKET), priority)] = value + return out + + def lines(self, module_id, start, end, limit, denylist, self_identifier, + direction="forward"): + """Prefiltered lines for one module. Returns a list of (ts_ms, text).""" + selector = ('{{module_id="{0}", node_id=~".+"}}'.format(module_id) + if module_id else '{node_id=~".+", module_id=""}') + stages = [ + selector, + '| json priority="PRIORITY", identifier="SYSLOG_IDENTIFIER", message="MESSAGE"', + '| identifier != "{0}"'.format(self_identifier), + '| priority < 5 or category="security"', + ] + if denylist: + stages.append('!~ "{0}"'.format("|".join(denylist))) + stages.append( + '| line_format "<{{.priority}}> [{{.identifier}}] {{.message}}"') + payload = self._get("/loki/api/v1/query_range", { + "query": " ".join(stages), + "start": self._ns(start), "end": self._ns(end), + "limit": str(int(limit)), "direction": direction, + }) + # category is a stream label, not a line field, so it is read per + # series rather than parsed back out of the rendered line. Losing it + # would make the server's security gate condition unreachable. + out = [] + for series in (payload.get("data") or {}).get("result") or []: + category = (series.get("stream") or {}).get("category", "") + for ns, text in series.get("values") or []: + out.append((int(ns) // 1000000, text, category)) + out.sort() + return out diff --git a/imageroot/pypkg/insights/masking.py b/imageroot/pypkg/insights/masking.py new file mode 100644 index 0000000..99706af --- /dev/null +++ b/imageroot/pypkg/insights/masking.py @@ -0,0 +1,153 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Normalise volatile tokens in a log line so repeats collapse to one template. + +This runs *after* `scrub()` and does a different job. `scrub()` removes +secrets and deliberately preserves IP addresses, hostnames and module IDs +because they carry signal. Masking replaces exactly those volatile parts, +because two lines that differ only by a PID or an address are the same event +and must group together. + +Both outputs are kept downstream: the template (masked) is what gets counted +and hashed into a finding's identity, while the retained sample line is the +scrubbed-but-unmasked original, so an operator can still see which address +was actually refused. + +Rule order is load-bearing and is documented per rule below. +""" + +import re + +# Bumped by hand whenever MASKING_RULES changes. A change alters every +# template's text, so it invalidates every server-side template and +# fingerprint at once. Travelling in the bundle makes the resulting one-time +# duplicate burst explainable instead of mysterious. +MASKING_VERSION = 1 + +# Placeholders already present in the input are left alone, which is what +# makes mask() idempotent. Listed here so one rule can protect all of them. +_PLACEHOLDER = r'<(?:TS|PID|UUID|IP|PORT|HEX|NUM|PATH|USER|redacted[a-z-]*)>' + +MASKING_RULES = [ + # 1. Volatile paths. Before any number rule, or /proc/12345 loses its + # shape and stops matching as a path at all. + ( + re.compile(r'(?:/tmp|/run|/var/tmp)/\S+'), + '', + ), + ( + re.compile(r'/proc/\d+(?:/\S*)?'), + '', + ), + # 2. ISO-8601 timestamps. Before clock times, which are a prefix of them. + ( + re.compile( + r'\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}' + r'(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?' + ), + '', + ), + # 3. Bare clock times. Before the port rule, which would otherwise claim + # the ":32" tail, and before IPv6, which also uses colons. + ( + re.compile(r'\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b'), + '', + ), + # 4. UUIDs. Must precede the hex rule, which would otherwise shred a UUID + # into --... and produce a template that never groups. + ( + re.compile(r'\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}' + r'-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b'), + '', + ), + # 5. IPv6. Must precede the hex rule for the same reason as UUIDs, and + # precede IPv4 so an IPv4-mapped form is taken whole. The compressed + # "::" form may be followed by several more groups, so the tail is a + # repeating group rather than a single optional one. + ( + re.compile(r'(?', + ), + # 6. IPv4, with an optional port taken in the same match. + # + # The port cannot be a separate rule with a (?<=) lookbehind: by + # the time that rule ran, would be a protected placeholder and + # excluded from the text the rule is applied to, so the lookbehind + # could never match. Consuming the port here also keeps the port from + # reaching the bare-number rule. + ( + re.compile(r'(?:' if m.group(1) else '', + ), + # 7. PIDs in their three common shapes. Before the bare-number rule. + ( + re.compile(r'\[\d+\]'), + '[]', + ), + ( + re.compile(r'(?i)\b(pid)\b([=: ]+)\d+'), + r'\1\2', + ), + ( + re.compile(r'\((\d{2,})\)'), + '()', + ), + # 9. Hex runs. After UUID and IPv6. Eight is the shortest run that is + # more often a checksum than a word: "cafe" and "deadbeef" both exist + # in prose, but eight-plus hex characters rarely do. + ( + re.compile(r'\b(?=[0-9a-fA-F]*\d)[0-9a-fA-F]{8,}\b'), + '', + ), + # 10. Account names in authentication messages. + # + # Measured against six hours of real cluster logs: an SSH dictionary + # attack produced 319 distinct templates that differed only by the + # account being tried. Left unmasked, every window of an ongoing + # attack looks like hundreds of brand-new templates, which opens the + # server's novelty gate forever and defeats the cost control it + # exists to provide. The retained sample line still carries the + # actual account name, and the count carries the volume. + ( + re.compile(r'(?i)\b(user)\s+(?!<)[^\s]+'), + r'\1 ', + ), + # 11. Bare integers, last of all: every rule above embeds digits, and + # running this earlier would dismantle them. Two digits minimum, so + # priority markers like <3> and names like traefik1 survive. + ( + re.compile(r'(?])'), + '', + ), +] + +# One pass that alternates "an existing placeholder" against "the next rule". +# Matching placeholders first and re-emitting them unchanged is what stops a +# second mask() call from rewriting into <[]> or into <>. +_PROTECTED = re.compile(_PLACEHOLDER) + + +def mask(line): + """Return the template form of one already-scrubbed log line.""" + for pattern, replacement in MASKING_RULES: + line = _apply(pattern, replacement, line) + return line + + +def _apply(pattern, replacement, line): + """Apply one rule to the parts of `line` that are not already placeholders.""" + out = [] + position = 0 + for protected in _PROTECTED.finditer(line): + out.append(pattern.sub(replacement, line[position:protected.start()])) + out.append(protected.group(0)) + position = protected.end() + out.append(pattern.sub(replacement, line[position:])) + return "".join(out) diff --git a/imageroot/pypkg/insights/scrub.py b/imageroot/pypkg/insights/scrub.py new file mode 100644 index 0000000..9c23ed7 --- /dev/null +++ b/imageroot/pypkg/insights/scrub.py @@ -0,0 +1,78 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Secret removal for collected log lines. + +Distinct from masking: this removes secrets and deliberately PRESERVES IP +addresses, hostnames and module IDs because they carry signal. mask() then +replaces those for grouping. Both outputs are kept downstream. +""" + +import re + + + +# Ordered: keyword assignments (Bearer's space-separated value first, then +# other secret keywords requiring an explicit assignment character), then +# Authorization headers, then email addresses, then long opaque blobs. Email +# runs before the blob rule so a long local part is still reported as an +# email. Both keyword rules allow optional whitespace around the separator +# (aligned/structured logs commonly pad it, e.g. "key : value") without +# reopening the bare-whitespace false positive: the character class still +# requires an actual "=", ":" or quote, which plain prose never supplies. +SCRUB_RULES = [ + ( + re.compile(r'(?i)\b(bearer)\b\s*[=:\s"\']+\s*\S+'), + r'\1=', + ), + ( + re.compile(r'(?i)\b(tokens?|api[-_]?keys?|secrets?|passwords?|passwd|pwd)\b\s*[=:"\']+\s*\S+'), + r'\1=', + ), + ( + re.compile(r'(?i)\bauthorization:\s*\S+(?:\s+\S+)?'), + 'authorization: ', + ), + # The negative lookahead keeps systemd templated unit names intact. + # `agent@nethvoice2.service` is not an address, and those PRIORITY=3 lines + # are how a crash loop names the module that is failing. + ( + re.compile( + r'\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9\-]+(?:\.[A-Za-z0-9\-]+)*' + r'\.(?!service\b|timer\b|socket\b|target\b|slice\b|scope\b|mount\b' + r'|path\b|device\b|swap\b|automount\b)[A-Za-z]{2,}\b' + ), + '', + ), + # `/` is deliberately excluded: a long filesystem path or URL is signal, + # not an opaque blob, and the base64 alphabet would otherwise swallow it. + ( + re.compile(r'\b[A-Za-z0-9+]{32,}={0,2}\b'), + '', + ), +] + + +WHITESPACE_RUN = re.compile(r'\s+') + + +def scrub(line): + """Remove likely secrets from a log line. + + Defence in depth, not a guarantee. IP addresses, hostnames, module IDs + and usernames are deliberately preserved: they carry the signal. + """ + for pattern, replacement in SCRUB_RULES: + line = pattern.sub(replacement, line) + return line + + +def sanitize_line(raw): + """Flatten a collected log line to exactly one prompt line, then scrub. + + Journal messages can contain newlines. Left alone they would break the + one-record-per-line structure of the prompt's LINES block, letting a log + message forge additional lines or close the fence. + """ + return scrub(WHITESPACE_RUN.sub(' ', raw).strip()) diff --git a/imageroot/pypkg/insights/select.py b/imageroot/pypkg/insights/select.py new file mode 100644 index 0000000..d402ef8 --- /dev/null +++ b/imageroot/pypkg/insights/select.py @@ -0,0 +1,49 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# +"""Divide the line budget between modules, fairly.""" + +# Every module is guaranteed this many lines before any weighting applies. +# This is what actually fixes starvation: a crash-looping module can no +# longer consume the whole budget, because every other module's floor is +# reserved first. +MIN_SHARE = 20 + +# Deviating or security-carrying modules get this much of the remainder +# relative to a quiet one. +PRIORITY_WEIGHT = 3.0 + +# Lines are fetched at OVERFETCH times a module's share, then deduplicated +# down to that share's worth of distinct templates. Without this, "dedup +# before capping" is impossible: Loki applies the cap at query time, before +# the collector sees a line, so one repeated line would still consume the +# whole allocation. +OVERFETCH = 4 +OVERFETCH_CEILING = 2000 + + +def allocate(module_ids, max_lines, prioritised=()): + """Return {module_id: share}. The total never exceeds max_lines.""" + modules = list(module_ids) + if not modules: + return {} + + count = len(modules) + if count * MIN_SHARE >= max_lines: + # More modules than floor space. Equal shares, floor unreachable. + share = max(1, max_lines // count) + return {m: share for m in modules} + + prioritised = set(prioritised) + weights = {m: (PRIORITY_WEIGHT if m in prioritised else 1.0) for m in modules} + total_weight = sum(weights.values()) + remainder = max_lines - count * MIN_SHARE + + return {m: MIN_SHARE + int(remainder * weights[m] / total_weight) + for m in modules} + + +def fetch_limit(share): + """How many raw lines to pull so dedup has something to work with.""" + return min(OVERFETCH_CEILING, max(share, share * OVERFETCH)) diff --git a/imageroot/systemd/user/insights-collector.service b/imageroot/systemd/user/insights-collector.service new file mode 100644 index 0000000..6c460bf --- /dev/null +++ b/imageroot/systemd/user/insights-collector.service @@ -0,0 +1,10 @@ +[Unit] +Description=Nethesis insights collector +Requires=loki-server.service +After=loki-server.service + +[Service] +Type=oneshot +EnvironmentFile=%E/state/environment +ExecStart=runagent %E/bin/insights-collector +SyslogIdentifier=%u/%N diff --git a/imageroot/systemd/user/insights-collector.timer b/imageroot/systemd/user/insights-collector.timer new file mode 100644 index 0000000..5e564b9 --- /dev/null +++ b/imageroot/systemd/user/insights-collector.timer @@ -0,0 +1,15 @@ +[Unit] +Description=Nethesis insights collector timer + +# compute_window() in the collector floors to the previous 15-minute +# boundary, so a randomised delay of up to 2 minutes still resolves to the +# just-closed window: no gap, no overlap. FixedRandomDelay spreads the fleet +# deterministically per node instead of re-rolling every fire. +[Timer] +OnCalendar=*:0/15 +Persistent=true +RandomizedDelaySec=2m +FixedRandomDelay=true + +[Install] +WantedBy=timers.target diff --git a/imageroot/update-module.d/15systemd b/imageroot/update-module.d/15systemd new file mode 100755 index 0000000..ed15306 --- /dev/null +++ b/imageroot/update-module.d/15systemd @@ -0,0 +1,19 @@ +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +set -e + +# Redirect any output to the journal (stderr) +exec 1>&2 + +# Best-effort teardown of the previous generation for dev nodes that +# installed the branch build. That generation was never released, so no +# ANOMALY_* environment migration is warranted. +systemctl --user disable --now anomaly-detector.timer 2>/dev/null || : + +# Pick up unit files added by this update (insights-collector.service/.timer) +systemctl --user daemon-reload diff --git a/test-unit.sh b/test-unit.sh new file mode 100755 index 0000000..2f586fa --- /dev/null +++ b/test-unit.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# Run the unit tests in a container. +# +# ./test-unit.sh [PYTEST ARG]... + +set -e -a + +venvroot=/usr/local/venv + +exec podman run -i --rm \ + --volume=.:/srv/source:z \ + --volume=pytest-cache:${venvroot}:z \ + --replace --name=pytest-unit \ + --env=venvroot \ + docker.io/python:3.11-alpine \ + ash -l -s -- "${@}" <<'EOF' +set -e +if [ ! -x ${venvroot}/bin/pytest ] ; then + python3 -mvenv ${venvroot} --upgrade + ${venvroot}/bin/pip3 install -q -r /srv/source/tests/unit/requirements.txt +fi +cd /srv/source +exec ${venvroot}/bin/pytest -q "${@}" tests/unit/ +EOF diff --git a/tests/20__insights.robot b/tests/20__insights.robot new file mode 100644 index 0000000..c278b90 --- /dev/null +++ b/tests/20__insights.robot @@ -0,0 +1,119 @@ +*** Settings *** +Library SSHLibrary +Suite Setup Start the stub insights server +Suite Teardown Tear down insights + +*** Variables *** +${MID} loki1 +# 9100 is node_exporter's well-known default port and it already runs on +# every NS8 test node, so the stub silently loses the bind and this suite's +# health check ends up talking to node_exporter instead. Stay clear of the +# whole Prometheus-ecosystem port neighborhood (9090-9100 and friends). +${STUB_PORT} 19100 +${STUB_URL} http://127.0.0.1:${STUB_PORT} +${RECORD_FILE} /tmp/insights-stub.jsonl + +# verify_tls=true vs. false is proven end-to-end against a self-signed +# server manually (verification step 6 of the plan), not here: standing up +# a self-signed HTTPS stub is more machinery than the assertion is worth in +# this suite, which only needs to prove the collector reaches the server +# and authenticates with its own subscription identity. + +*** Keywords *** +Start the stub insights server + Put File ${CURDIR}/insights-stub.py /tmp/insights-stub.py + Execute Command + ... setsid nohup python3 /tmp/insights-stub.py ${STUB_PORT} ${RECORD_FILE} /tmp/insights-stub.log 2>&1 & + Wait Until Keyword Succeeds 30s 2s The stub insights server answers + +The stub insights server answers + ${output} ${rc} = Execute Command + ... curl -sf http://127.0.0.1:${STUB_PORT}/ return_rc=${True} + Should Be Equal As Integers ${rc} 0 + Should Be Equal As Strings ${output} ok + +Tear down insights + Execute Command api-cli run module/${MID}/set-insights --data '{"active":false}' + Execute Command pkill -f insights-stub.py + +Run module action + [Arguments] ${action} ${data}=${EMPTY} + IF '${data}' == '${EMPTY}' + ${output} ${rc} = Execute Command + ... api-cli run module/${MID}/${action} return_rc=${True} + ELSE + ${output} ${rc} = Execute Command + ... api-cli run module/${MID}/${action} --data '${data}' return_rc=${True} + END + Should Be Equal As Integers ${rc} 0 action ${action} failed: ${output} + RETURN ${output} + +The injected noise reached Loki + ${query} = Set Variable {node_id=~\\".+\\"} |= \\"robot synthetic error\\" + ${cmd} = Catenate SEPARATOR=${SPACE} + ... runagent -m ${MID} bash -c "LOKI_ADDR=http://127.0.0.1:\\$LOKI_HTTP_PORT + ... LOKI_USERNAME=\\$LOKI_API_AUTH_USERNAME LOKI_PASSWORD=\\$LOKI_API_AUTH_PASSWORD + ... logcli query --since 15m --limit 10 --forward --no-labels -q -o raw '${query}'" + ${output} ${rc} = Execute Command ${cmd} return_rc=${True} + Should Be Equal As Integers ${rc} 0 logcli failed: ${output} + Should Contain ${output} robot synthetic error + +The stub recorded a bundle from the collector + ${output} ${rc} = Execute Command cat ${RECORD_FILE} return_rc=${True} + Should Be Equal As Integers ${rc} 0 ${RECORD_FILE} was not created: ${output} + Should Not Be Empty ${output} + # A non-empty, non-"unknown" system_id proves identity came from the + # node's own subscription, not a placeholder. + Should Match Regexp ${output} "system_id":\\s*"(?!unknown")[^"]+" + Should Match Regexp ${output} "auth":\\s*"Basic [^"]+" + # The digit in "error ${i}" masks to , but the fixed prefix survives + # in the template, so the injected noise is still recognisable here. + Should Contain ${output} robot synthetic error + +*** Test Cases *** +Inject synthetic noise for the window + FOR ${i} IN RANGE 5 + Execute Command logger -p daemon.err -t robot-noise robot synthetic error ${i} + END + Wait Until Keyword Succeeds 90s 10s The injected noise reached Loki + +Configure insights against the stub + Run module action set-insights + ... {"active":true,"base_url":"${STUB_URL}","verify_tls":false} + ${output} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user is-active insights-collector.timer + ... return_rc=${True} + Should Be Equal As Strings ${output} active + +get-configuration reports the active collector and its stub target + ${output} = Run module action get-configuration + Should Contain ${output} + ... "status": "active", "base_url": "${STUB_URL}", "verify_tls": false + Should Contain ${output} "subscription_configured" + +The oneshot service ships a bundle authenticated with the subscription + ${output} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user start insights-collector.service + ... return_rc=${True} + Should Be Equal As Integers ${rc} 0 service failed to run: ${output} + Wait Until Keyword Succeeds 60s 5s The stub recorded a bundle from the collector + +--print emits a bundle without shipping or authenticating + ${cmd} = Catenate SEPARATOR=${SPACE} + ... runagent -m ${MID} ../bin/insights-collector --print | python3 -m json.tool + ${output} ${rc} = Execute Command ${cmd} return_rc=${True} + Should Be Equal As Integers ${rc} 0 --print did not emit parseable JSON: ${output} + Should Contain ${output} schema_version + Should Contain ${output} templates + Should Contain ${output} budget + +Disabling insights stops the timer and clears the module environment + Run module action set-insights {"active":false} + ${timer} ${rc} = Execute Command + ... runagent -m ${MID} systemctl --user is-active insights-collector.timer + ... return_rc=${True} + Should Not Be Equal As Strings ${timer} active + # Read the environment back through the module's own API rather than + # redis-cli, which needs credentials this suite does not carry. + ${output} = Run module action get-configuration + Should Contain ${output} "base_url": "" diff --git a/tests/insights-stub.py b/tests/insights-stub.py new file mode 100644 index 0000000..6d407e3 --- /dev/null +++ b/tests/insights-stub.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +# +# Canned insights server for the Robot suite. +# No real server in CI: no network egress, no cost, deterministic. +# +# python3 insights-stub.py [PORT] [RECORD_FILE] +# + +import json +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +RECORD_FILE = sys.argv[2] if len(sys.argv) > 2 else "/tmp/insights-stub.jsonl" + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + if self.path != '/v1/bundles': + self._send(404, b'not found') + return + length = int(self.headers.get('Content-Length') or 0) + body = self.rfile.read(length) + try: + bundle = json.loads(body.decode()) + except ValueError: + bundle = {} + record = dict(bundle) if isinstance(bundle, dict) else {"bundle": bundle} + record["auth"] = self.headers.get('Authorization', '') + with open(RECORD_FILE, 'a') as handle: + handle.write(json.dumps(record) + "\n") + payload = json.dumps({"status": "accepted"}).encode() + self._send(202, payload) + + def do_GET(self): + if self.path != '/': + self._send(404, b'not found') + return + self._send(200, b'ok') + + def _send(self, status, body): + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, fmt, *args): + sys.stderr.write("insights-stub: " + (fmt % args) + "\n") + + +if __name__ == "__main__": + port = int(sys.argv[1]) if len(sys.argv) > 1 else 9099 + HTTPServer(('127.0.0.1', port), Handler).serve_forever() diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000..793d5d1 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,37 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import importlib.machinery +import importlib.util +import pathlib +import sys + +import pytest + +ROOT = pathlib.Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "imageroot" / "bin" / "insights-collector" + +# The collector library ships at imageroot/pypkg, which maps to %E/pypkg in +# the installed module. The entrypoint puts it on sys.path relative to its own +# location; tests do the same so the package imports identically in both. +sys.path.insert(0, str(ROOT / "imageroot" / "pypkg")) + + +def _load(): + loader = importlib.machinery.SourceFileLoader("insights_collector", str(SCRIPT)) + spec = importlib.util.spec_from_loader("insights_collector", loader) + module = importlib.util.module_from_spec(spec) + loader.exec_module(module) + return module + + +@pytest.fixture(scope="session") +def collector(): + """The insights-collector script loaded as a module. + + The file has no .py extension, so it cannot be imported normally. + Loading it must not perform I/O nor import the `agent` SDK. + """ + return _load() diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt new file mode 100644 index 0000000..547de5c --- /dev/null +++ b/tests/unit/requirements.txt @@ -0,0 +1,2 @@ +pytest +requests diff --git a/tests/unit/test_bundle.py b/tests/unit/test_bundle.py new file mode 100644 index 0000000..f210261 --- /dev/null +++ b/tests/unit/test_bundle.py @@ -0,0 +1,116 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +from insights import bundle + + +def line(ts, text, category=""): + return (ts, text, category) + + +class TestGroupTemplates: + def test_repeats_collapse_to_one_counted_template(self): + lines = [line(i, "smbd[%d] connection refused to 10.0.0.%d:8080" % (i, i)) + for i in range(1, 38)] + templates, kept, fetched = bundle.group_templates(lines, "samba2", 20) + assert len(templates) == 1 + assert templates[0]["count"] == 37 + assert kept == 37 + assert fetched == 37 + + def test_the_share_caps_distinct_templates_not_lines(self): + # 100 distinct events, budget of 5: five templates survive, and they + # are the five most frequent rather than the five earliest. + # + # Names are letter pairs on purpose. Digits would mask to and + # merge, and chr() walks into Unicode whitespace such as NEL and + # NBSP, which sanitize_line correctly collapses. + alphabet = "abcdefghij" + lines = [] + for i in range(100): + name = alphabet[i // 10] + alphabet[i % 10] + lines.extend([line(i, "event kind %s happened" % name)] * (i + 1)) + templates, _kept, _fetched = bundle.group_templates(lines, "m", 5) + assert len(templates) == 5 + counts = [t["count"] for t in templates] + assert counts == sorted(counts, reverse=True) + assert counts[0] == 100 + + def test_samples_keep_first_and_last_only(self): + lines = [line(i, "smbd[%d] refused 10.0.0.4:80" % i) for i in range(10)] + templates, _k, _f = bundle.group_templates(lines, "m", 20) + samples = templates[0]["samples"] + assert len(samples) == 2 + assert "smbd[0]" in samples[0] + assert "smbd[9]" in samples[1] + + def test_samples_are_scrubbed_but_not_masked(self): + lines = [line(1, "refused 10.0.0.4:8080 password=hunter2")] + templates, _k, _f = bundle.group_templates(lines, "m", 20) + sample = templates[0]["samples"][0] + assert "password=" in sample, "secrets must be removed" + assert "10.0.0.4" in sample, "detail must survive in the sample" + assert "" in templates[0]["template"], "the template must be masked" + + def test_priority_is_parsed_from_the_rendered_line(self): + templates, _k, _f = bundle.group_templates( + [line(1, "<3> [traefik] backend down")], "traefik1", 20) + assert templates[0]["priority"] == 3 + + def test_category_is_carried_through(self): + templates, _k, _f = bundle.group_templates( + [line(1, "<6> [sshd] Invalid user bob", "security")], "", 20) + assert templates[0]["category"] == "security" + + def test_absent_category_is_omitted_entirely(self): + templates, _k, _f = bundle.group_templates( + [line(1, "<6> [systemd] Started thing")], "m", 20) + assert "category" not in templates[0] + + def test_same_text_in_different_categories_stays_separate(self): + templates, _k, _f = bundle.group_templates( + [line(1, "<6> [x] same text"), line(2, "<6> [x] same text", "security")], + "m", 20) + assert len(templates) == 2 + + def test_first_and_last_seen_span_the_group(self): + lines = [line(500, "<6> [x] a"), line(100, "<6> [x] a"), line(900, "<6> [x] a")] + templates, _k, _f = bundle.group_templates(lines, "m", 20) + assert templates[0]["first_seen"] == 100 + assert templates[0]["last_seen"] == 900 + + def test_empty_input(self): + templates, kept, fetched = bundle.group_templates([], "m", 20) + assert templates == [] and kept == 0 and fetched == 0 + + +class TestSampleEnds: + def test_keeps_both_ends_of_the_window(self): + # An early burst must not hide a late-developing incident. + forward = [line(i, "early %d" % i) for i in range(10)] + backward = [line(100 + i, "late %d" % i) for i in range(10)] + merged = bundle.sample_ends(forward, backward, 6) + texts = [row[1] for row in merged] + assert any(t.startswith("early") for t in texts) + assert any(t.startswith("late") for t in texts) + assert len(merged) == 6 + + def test_deduplicates_the_overlap(self): + shared = [line(5, "same")] + merged = bundle.sample_ends(shared, shared, 10) + assert len(merged) == 1 + + def test_returns_everything_when_under_the_limit(self): + forward = [line(1, "a"), line(2, "b")] + assert len(bundle.sample_ends(forward, [], 10)) == 2 + + +class TestBuild: + def test_payload_shape(self): + payload = bundle.build("sys1", "2.0.0", (1000, 1900), [], [], {"max_lines": 500}) + assert payload["schema_version"] == bundle.SCHEMA_VERSION + assert payload["system_id"] == "sys1" + assert payload["window"] == {"start": 1000, "end": 1900} + assert isinstance(payload["masking_version"], int) diff --git a/tests/unit/test_collector.py b/tests/unit/test_collector.py new file mode 100644 index 0000000..b553a3c --- /dev/null +++ b/tests/unit/test_collector.py @@ -0,0 +1,343 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import base64 +import datetime +import json +import ssl +import sys +import types + +import pytest + + +UTC = datetime.timezone.utc + + +class TestComputeWindow: + """compute_window floors `now` to the previous window boundary. + + The systemd timer fires with a 2-minute randomised delay, so the window + it asks for must be derived purely from wall-clock time, not from "the + last N minutes" relative to whenever the timer happened to wake up. + Otherwise two consecutive runs could double-count or skip a slice. + """ + + @pytest.mark.parametrize("now,expected_start,expected_end", [ + (datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 9, 45, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC)), + (datetime.datetime(2026, 1, 1, 10, 7, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 9, 45, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC)), + (datetime.datetime(2026, 1, 1, 10, 14, 59, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 9, 45, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC)), + (datetime.datetime(2026, 1, 1, 10, 15, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 15, 0, tzinfo=UTC)), + (datetime.datetime(2026, 1, 1, 10, 44, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 15, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 30, 0, tzinfo=UTC)), + ]) + def test_floors_to_the_previous_15_minute_boundary( + self, collector, now, expected_start, expected_end): + start, end = collector.compute_window(now) + assert (start, end) == (expected_start, expected_end) + + def test_window_length_always_matches_the_requested_minutes(self, collector): + for now in ( + datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 7, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 14, 59, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 15, 0, tzinfo=UTC), + datetime.datetime(2026, 1, 1, 10, 44, 0, tzinfo=UTC), + ): + for minutes in (5, 15, 60): + start, end = collector.compute_window(now, minutes) + assert end - start == datetime.timedelta(minutes=minutes) + + def test_non_default_window_size(self, collector): + # A 5-minute window floors to a 5-minute boundary, not 15. + # 10:07 -> minute 7 % 5 == 2, so end steps back to 10:05. + now = datetime.datetime(2026, 1, 1, 10, 7, 0, tzinfo=UTC) + start, end = collector.compute_window(now, 5) + assert start == datetime.datetime(2026, 1, 1, 10, 0, 0, tzinfo=UTC) + assert end == datetime.datetime(2026, 1, 1, 10, 5, 0, tzinfo=UTC) + + +class TestMs: + def test_converts_to_integer_unix_millis(self, collector): + assert collector.ms(datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=UTC)) == 1000 + + def test_truncates_sub_millisecond_precision(self, collector): + when = datetime.datetime(1970, 1, 1, 0, 0, 0, 500000, tzinfo=UTC) + assert collector.ms(when) == 500 + + def test_result_is_an_int(self, collector): + when = datetime.datetime(2026, 1, 1, 0, 0, 0, tzinfo=UTC) + assert isinstance(collector.ms(when), int) + + +class FakeResponse: + def __init__(self, status=202, body=b'{"ok":true}'): + self.status = status + self._body = body + + def read(self): + return self._body + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + +class TestShip: + """ship() must produce a request an insights server can authenticate.""" + + def _capture(self, monkeypatch): + captured = {} + + def fake_urlopen(request, timeout=None, context=None): + captured["request"] = request + captured["timeout"] = timeout + captured["context"] = context + return FakeResponse() + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + return captured + + @pytest.mark.parametrize("base_url", ["https://x", "https://x/"]) + def test_url_has_exactly_one_bundles_path(self, collector, monkeypatch, base_url): + captured = self._capture(monkeypatch) + collector.ship({"a": 1}, base_url, "sys1", "secret1") + assert captured["request"].full_url == "https://x/v1/bundles" + + def test_content_type_is_json(self, collector, monkeypatch): + captured = self._capture(monkeypatch) + collector.ship({"a": 1}, "https://x", "sys1", "secret1") + assert captured["request"].get_header("Content-type") == "application/json" + + def test_authorization_is_basic_system_id_and_secret(self, collector, monkeypatch): + captured = self._capture(monkeypatch) + collector.ship({"a": 1}, "https://x", "sys1", "s3cr3t") + header = captured["request"].get_header("Authorization") + assert header.startswith("Basic ") + decoded = base64.b64decode(header[len("Basic "):]).decode() + assert decoded == "sys1:s3cr3t" + + def test_body_round_trips_to_the_payload(self, collector, monkeypatch): + captured = self._capture(monkeypatch) + payload = {"schema_version": 1, "templates": [{"template": "x"}]} + collector.ship(payload, "https://x", "sys1", "secret1") + assert json.loads(captured["request"].data.decode()) == payload + + def test_verify_true_builds_a_verifying_context(self, collector, monkeypatch): + captured = self._capture(monkeypatch) + collector.ship({"a": 1}, "https://x", "sys1", "secret1", verify=True) + ctx = captured["context"] + assert ctx.verify_mode == ssl.CERT_REQUIRED + assert ctx.check_hostname is True + + def test_verify_false_builds_a_non_verifying_context(self, collector, monkeypatch): + captured = self._capture(monkeypatch) + collector.ship({"a": 1}, "https://x", "sys1", "secret1", verify=False) + ctx = captured["context"] + assert ctx.verify_mode == ssl.CERT_NONE + + def test_returns_status_and_decoded_body(self, collector, monkeypatch): + def fake_urlopen(request, timeout=None, context=None): + return FakeResponse(status=202, body=b'{"queued":true}') + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + status, text = collector.ship({"a": 1}, "https://x", "sys1", "secret1") + assert status == 202 + assert text == '{"queued":true}' + + +class TestReadIdentity: + """read_identity() reads the node's own subscription, never module state. + + The `agent` SDK is only ever imported inside the function, so it can be + replaced with a fake module for these tests without the real SDK being + installed in the unit-test container. + """ + + @pytest.fixture + def fake_agent(self): + previous = sys.modules.get("agent") + + def install(hash_data): + stub = types.SimpleNamespace(hgetall=lambda key: dict(hash_data)) + module = types.ModuleType("agent") + module.redis_connect = lambda: stub + sys.modules["agent"] = module + return module + + yield install + + if previous is not None: + sys.modules["agent"] = previous + else: + sys.modules.pop("agent", None) + + def test_full_hash_returns_system_id_and_first_secret_field(self, collector, fake_agent): + fake_agent({"system_id": "sys1", "auth_token": "tok1", + "secret": "sec1", "password": "pw1"}) + assert collector.read_identity() == ("sys1", "tok1") + + @pytest.mark.parametrize("field", ["auth_token", "secret", "password"]) + def test_each_secret_field_is_used_when_it_is_the_only_one_present( + self, collector, fake_agent, field): + fake_agent({"system_id": "sys1", field: "value-for-" + field}) + assert collector.read_identity() == ("sys1", "value-for-" + field) + + def test_secret_fields_are_tried_in_declared_order(self, collector, fake_agent): + assert list(collector.SUBSCRIPTION_SECRET_FIELDS) == \ + ["auth_token", "secret", "password"] + fake_agent({"system_id": "sys1", "secret": "sec1", "password": "pw1"}) + assert collector.read_identity() == ("sys1", "sec1") + + def test_empty_hash_yields_none_none(self, collector, fake_agent): + fake_agent({}) + assert collector.read_identity() == (None, None) + + def test_missing_id_field_yields_none_none(self, collector, fake_agent): + fake_agent({"auth_token": "tok1"}) + assert collector.read_identity() == (None, None) + + def test_missing_secret_yields_none_none(self, collector, fake_agent): + fake_agent({"system_id": "sys1"}) + assert collector.read_identity() == (None, None) + + def test_bytes_values_are_decoded_to_str(self, collector, fake_agent): + fake_agent({b"system_id": b"sys1", b"auth_token": b"tok1"}) + result = collector.read_identity() + assert result == ("sys1", "tok1") + assert all(isinstance(part, str) for part in result) + + def test_failure_message_lists_field_names_but_never_values( + self, collector, fake_agent, capsys): + fake_agent({"password": "top-secret-value"}) + result = collector.read_identity() + assert result == (None, None) + err = capsys.readouterr().err + assert "password" in err + assert "top-secret-value" not in err + + +class TestVerifyTlsParsing: + """INSIGHTS_VERIFY_TLS fails safe: anything unrecognised means verify. + + There is no dedicated helper for this, so it is exercised through + main() with ship() and collect() monkeypatched to avoid any I/O. + """ + + @pytest.fixture + def run_main(self, collector, monkeypatch): + monkeypatch.setenv("LOKI_HTTP_PORT", "3100") + monkeypatch.setenv("LOKI_API_AUTH_USERNAME", "u") + monkeypatch.setenv("LOKI_API_AUTH_PASSWORD", "p") + monkeypatch.setenv("INSIGHTS_SERVER_URL", "https://insights.example/") + monkeypatch.setattr(collector, "read_identity", lambda: ("sys1", "secret1")) + monkeypatch.setattr(collector, "collect", + lambda *a, **k: {"schema_version": 1, "templates": [], + "budget": {"lines_kept": 0}}) + captured = {} + + def fake_ship(payload, url, system_id, secret, verify=True, timeout=60): + captured["verify"] = verify + return 202, "ok" + + monkeypatch.setattr(collector, "ship", fake_ship) + + def call(): + rc = collector.main([]) + return rc, captured.get("verify") + + return call + + @pytest.mark.parametrize("value", ["0", "false", "False", " false ", + "NO", "no", "off", "OFF"]) + def test_false_spellings_disable_verification(self, monkeypatch, run_main, value): + monkeypatch.setenv("INSIGHTS_VERIFY_TLS", value) + rc, verify = run_main() + assert rc == 0 + assert verify is False + + @pytest.mark.parametrize("value", ["", "1", "true", "True", "yes", "maybe", "garbage"]) + def test_everything_else_verifies(self, monkeypatch, run_main, value): + monkeypatch.setenv("INSIGHTS_VERIFY_TLS", value) + rc, verify = run_main() + assert rc == 0 + assert verify is True + + def test_unset_verifies(self, monkeypatch, run_main): + # Fail-safe direction: an operator who forgets to set this variable + # gets certificate verification, not a silently-open channel. + monkeypatch.delenv("INSIGHTS_VERIFY_TLS", raising=False) + rc, verify = run_main() + assert rc == 0 + assert verify is True + + +class TestMainFailurePaths: + def _set_loki_env(self, monkeypatch): + monkeypatch.setenv("LOKI_HTTP_PORT", "3100") + monkeypatch.setenv("LOKI_API_AUTH_USERNAME", "u") + monkeypatch.setenv("LOKI_API_AUTH_PASSWORD", "p") + + def test_fails_when_server_url_is_unset(self, collector, monkeypatch): + self._set_loki_env(monkeypatch) + monkeypatch.delenv("INSIGHTS_SERVER_URL", raising=False) + monkeypatch.setattr(collector, "read_identity", lambda: ("sys1", "secret1")) + monkeypatch.setattr(collector, "collect", + lambda *a, **k: {"templates": [], "budget": {"lines_kept": 0}}) + assert collector.main([]) == 1 + + def test_fails_when_identity_is_missing(self, collector, monkeypatch): + self._set_loki_env(monkeypatch) + monkeypatch.setenv("INSIGHTS_SERVER_URL", "https://insights.example/") + monkeypatch.setattr(collector, "read_identity", lambda: (None, None)) + assert collector.main([]) == 1 + + def test_ship_timeout_fails_cleanly(self, collector, monkeypatch, capsys): + # urllib only wraps send-phase failures as URLError; a timeout while + # waiting for the response on an already-open connection surfaces as + # a bare TimeoutError, which main() must not let escape as a + # traceback. + self._set_loki_env(monkeypatch) + monkeypatch.setenv("INSIGHTS_SERVER_URL", "https://insights.example/") + monkeypatch.setattr(collector, "read_identity", lambda: ("sys1", "secret1")) + monkeypatch.setattr(collector, "collect", + lambda *a, **k: {"templates": [], "budget": {"lines_kept": 0}}) + + def fake_ship(*a, **k): + raise TimeoutError("timed out") + + monkeypatch.setattr(collector, "ship", fake_ship) + + assert collector.main([]) == 1 + assert "timed out" in capsys.readouterr().err + + def test_print_succeeds_with_neither_subscription_nor_server_url( + self, collector, monkeypatch, capsys): + # --print is the documented zero-cost inspection path: it must work + # on a node that has no insights server configured at all yet. + self._set_loki_env(monkeypatch) + monkeypatch.delenv("INSIGHTS_SERVER_URL", raising=False) + payload = {"schema_version": 1, "templates": [], + "budget": {"max_lines": 500, "lines_seen": 0, "lines_kept": 0}} + monkeypatch.setattr(collector, "collect", lambda *a, **k: payload) + + def fail_if_called(): + raise AssertionError("read_identity should not be needed for --print") + + monkeypatch.setattr(collector, "read_identity", fail_if_called) + + assert collector.main(["--print"]) == 0 + assert json.loads(capsys.readouterr().out) == payload diff --git a/tests/unit/test_masking.py b/tests/unit/test_masking.py new file mode 100644 index 0000000..837afb9 --- /dev/null +++ b/tests/unit/test_masking.py @@ -0,0 +1,142 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import pytest + +from insights.masking import MASKING_VERSION, mask + + +class TestIndividualRules: + @pytest.mark.parametrize("raw,expected", [ + ("started at 2026-08-05T14:42:32Z", "started at "), + ("started at 2026-08-05 14:42:32.123456+00:00", "started at "), + ("elapsed 14:42:32", "elapsed "), + ("elapsed 14:42:32.918", "elapsed "), + ]) + def test_timestamps(self, raw, expected): + assert mask(raw) == expected + + @pytest.mark.parametrize("raw,expected", [ + ("smbd[12345]: oplock", "smbd[]: oplock"), + ("killed pid=987", "killed pid="), + ("killed pid 987", "killed pid "), + ("main process (4211) exited", "main process () exited"), + ]) + def test_pids(self, raw, expected): + assert mask(raw) == expected + + def test_uuid(self): + assert mask("job 3f2504e0-4f89-11d3-9a0c-0305e82c3301 done") == "job done" + + def test_uuid_is_not_shredded_into_hex(self): + # The hex rule would otherwise eat a UUID piecewise, producing + # --... and a template that never groups with itself. + assert "" not in mask("id=3f2504e0-4f89-11d3-9a0c-0305e82c3301") + + @pytest.mark.parametrize("raw,expected", [ + ("from 10.0.0.4", "from "), + ("from 192.168.100.200", "from "), + ("peer fe80::1", "peer "), + ("peer 2001:db8::8a2e:370:7334", "peer "), + ]) + def test_ip_addresses(self, raw, expected): + assert mask(raw) == expected + + def test_ipv6_is_not_shredded_into_hex(self): + assert "" not in mask("peer 2001:db8::8a2e:370:7334") + + def test_ip_with_port(self): + assert mask("connection refused to 10.0.0.4:8080") == \ + "connection refused to :" + + def test_hex_blob(self): + assert mask("sha 9f86d081884c7d65") == "sha " + + def test_short_hex_is_left_alone(self): + # "cafe" is a word far more often than it is a checksum. + assert mask("the cafe is open") == "the cafe is open" + + @pytest.mark.parametrize("raw,expected", [ + ("wrote /tmp/abc123/file", "wrote "), + ("read /proc/12345/status", "read "), + ("socket /run/user/1000/bus", "socket "), + ]) + def test_volatile_paths(self, raw, expected): + assert mask(raw) == expected + + def test_stable_paths_are_preserved(self): + # A real config path is signal and must survive. + assert mask("/etc/loki/loki-config.yaml") == "/etc/loki/loki-config.yaml" + + def test_bare_numbers(self): + assert mask("retried 37 times") == "retried times" + + def test_single_digits_are_preserved(self): + # Priority markers like <3> and version suffixes carry meaning. + assert mask("<3> module traefik1 failed") == "<3> module traefik1 failed" + + +class TestGrouping: + def test_same_event_different_volatiles_yields_one_template(self): + a = mask("smbd[1234]: connection refused to 10.0.0.4:8080 after 37 tries") + b = mask("smbd[9876]: connection refused to 10.9.9.9:9090 after 4210 tries") + assert a == b + + def test_genuinely_different_events_stay_distinct(self): + a = mask("smbd[1234]: connection refused to 10.0.0.4:8080") + b = mask("smbd[1234]: permission denied for 10.0.0.4:8080") + assert a != b + + +class TestSafety: + def test_is_idempotent(self): + # Templates get compared and hashed; masking twice must not drift. + raw = "smbd[1234] 2026-08-05T14:42:32Z 10.0.0.4:8080 id=3f2504e0-4f89-11d3-9a0c-0305e82c3301 n=37" + once = mask(raw) + assert mask(once) == once + + def test_placeholders_are_never_re_masked(self): + assert mask(" ") == \ + " " + + def test_scrub_markers_survive(self): + # masking runs after scrub(); its redactions must not be mangled. + assert mask("token= blob=") == \ + "token= blob=" + + def test_empty_and_whitespace(self): + assert mask("") == "" + assert mask(" ") == " " + + def test_version_is_an_int(self): + assert isinstance(MASKING_VERSION, int) + assert MASKING_VERSION >= 1 + + +class TestAccountNames: + """An SSH dictionary attack must collapse to one template, not hundreds. + + Measured on six hours of real cluster logs: 319 of 561 templates were + single-occurrence brute-force attempts differing only by account name. + """ + + def test_invalid_user_collapses(self): + a = mask("Connection closed by invalid user admin 10.0.0.4 port 5000") + b = mask("Connection closed by invalid user alice 10.9.9.9 port 6000") + assert a == b + assert "" in a + + def test_authenticating_user_collapses(self): + a = mask("Connection closed by authenticating user root 10.0.0.4 port 1") + b = mask("Connection closed by authenticating user nobody 10.0.0.4 port 1") + assert a == b + + def test_distinct_auth_events_stay_distinct(self): + assert mask("Invalid user admin from 10.0.0.4") != \ + mask("Accepted password for admin from 10.0.0.4") + + def test_is_still_idempotent_with_user_rule(self): + once = mask("Invalid user admin from 10.0.0.4 port 5000") + assert mask(once) == once diff --git a/tests/unit/test_select.py b/tests/unit/test_select.py new file mode 100644 index 0000000..4eeaaa6 --- /dev/null +++ b/tests/unit/test_select.py @@ -0,0 +1,51 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +from insights import select + + +class TestAllocate: + def test_equal_shares_when_nothing_deviates(self): + shares = select.allocate(["a", "b", "c", "d"], 500) + assert len(set(shares.values())) == 1 + + def test_total_never_exceeds_the_budget(self): + for count in (1, 3, 7, 12, 24): + modules = ["m%d" % i for i in range(count)] + assert sum(select.allocate(modules, 500).values()) <= 500 + + def test_every_module_gets_the_floor(self): + # The whole point: a crash-looping module cannot starve the others. + shares = select.allocate(["quiet", "noisy"], 500, prioritised=["noisy"]) + assert shares["quiet"] >= select.MIN_SHARE + + def test_deviating_modules_get_more(self): + shares = select.allocate(["quiet", "noisy"], 500, prioritised=["noisy"]) + assert shares["noisy"] > shares["quiet"] + + def test_floor_is_abandoned_when_there_is_no_room(self): + # 40 modules x 20 floor = 800 > 500. Equal shares instead, and the + # budget still holds. + modules = ["m%d" % i for i in range(40)] + shares = select.allocate(modules, 500) + assert sum(shares.values()) <= 500 + assert min(shares.values()) >= 1 + + def test_no_modules(self): + assert select.allocate([], 500) == {} + + def test_host_bucket_is_allocated_like_any_module(self): + # Host-level logs carry no module_id label; the empty-string bucket + # must not be silently skipped. + shares = select.allocate(["", "traefik1"], 500) + assert shares[""] >= select.MIN_SHARE + + +class TestFetchLimit: + def test_overfetches_so_dedup_has_room(self): + assert select.fetch_limit(100) == 100 * select.OVERFETCH + + def test_is_capped(self): + assert select.fetch_limit(10000) == select.OVERFETCH_CEILING