diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index d779808a1..086b74c21 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -821,6 +821,12 @@ jobs: chmod +x scripts/validate-context-map-drift.sh bash scripts/validate-context-map-drift.sh + - name: Validate skill-flow connectivity and closed consumes vocabulary + if: needs.changes.outputs.skills == 'true' || needs.changes.outputs.contracts == 'true' || needs.changes.outputs.shell == 'true' || needs.changes.outputs.ci == 'true' + run: | + chmod +x scripts/validate-skill-flow.sh + bash scripts/validate-skill-flow.sh + - name: Verify embedded lib/skills are in sync if: needs.changes.outputs.hooks == 'true' || needs.changes.outputs.go == 'true' || needs.changes.outputs.ci == 'true' run: | diff --git a/docs/contracts/context-map.md b/docs/contracts/context-map.md index c987a6543..c36490da3 100644 --- a/docs/contracts/context-map.md +++ b/docs/contracts/context-map.md @@ -117,6 +117,7 @@ graph LR complexity -- "shared-kernel" --> standards council -- "shared-kernel" --> standards crank -- "shared-kernel" --> standards + deps -- "supplier-to" --> vibe design -- "shared-kernel" --> standards discovery -- "shared-kernel" --> standards evolve -- "customer-of" --> rpi @@ -134,13 +135,17 @@ graph LR pr-validate -- "customer-of" --> validation pre-mortem -- "shared-kernel" --> standards product -- "shared-kernel" --> standards + provenance -- "supplier-to" --> trace quickstart -- "customer-of" --> rpi ratchet -- "shared-kernel" --> standards + red-team -- "supplier-to" --> vibe + release -- "supplier-to" --> ship-loop retro -- "shared-kernel" --> standards review -- "customer-of" --> validation rpi -- "customer-of" --> crank rpi -- "customer-of" --> discovery rpi -- "customer-of" --> validation + scenario -- "supplier-to" --> validation scope -- "supplier-to" --> domain security -- "supplier-to" --> vibe security-suite -- "supplier-to" --> vibe @@ -156,6 +161,7 @@ graph LR skill-builder -- "customer-of" --> automation-shape-routing skill-builder -- "supplier-to" --> skill-auditor swarm -- "customer-of" --> crank + trace -- "customer-of" --> provenance validate -- "customer-of" --> validation validation -- "shared-kernel" --> standards vibe -- "shared-kernel" --> standards diff --git a/docs/contracts/skill-flow.md b/docs/contracts/skill-flow.md new file mode 100644 index 000000000..ecc1809c5 --- /dev/null +++ b/docs/contracts/skill-flow.md @@ -0,0 +1,90 @@ +# Skill-Flow Connectivity Contract + +> Gate: `scripts/validate-skill-flow.sh` (CI job `validate-skill-flow`). +> Allowlist: `scripts/skill-flow-standalone.txt`. +> Source of truth: `skills/*/SKILL.md` frontmatter. +> Sibling: `scripts/audit-skill-metadata.sh` owns `context_rel.with` resolution; +> this contract owns the `consumes` vocabulary and connectivity. + +## Why this exists + +"Do all skills flow together?" was unanswerable until this gate. Skill +dependencies are declared in **three overlapping frontmatter fields** that +historically drifted apart: + +| Field | Meaning | Read by context-map? | +|-------|---------|----------------------| +| `consumes` | upstream inputs (skill slug, external input, or produced artifact) | yes (data-flow table) | +| `context_rel` | DDD bounded-context relationship (`kind` + `with`) | yes (mermaid graph) | +| `metadata.dependencies` | upstream skill slugs | **no** | + +Because the three are not reconciled, a skill can look "orphaned" in one field +while being well-connected in another (e.g. `trace` declares no `consumes` but +depends on `provenance` via `metadata.dependencies`). This contract defines a +single connectivity model that spans all three and a closed vocabulary for +`consumes`. + +## Rules (enforced — gate fails on violation) + +### 1. Closed `consumes` vocabulary + +Every `consumes` token MUST resolve to exactly one of: + +1. a **peer skill slug** (a directory under `skills/`), or +2. a **whitelisted external input**, or +3. an **artifact produced by some skill** (appears in another skill's `produces`). + +Anything else is a typo or an undeclared dependency and fails the gate. + +**External inputs** (the closed whitelist — extend deliberately, in both +`scripts/validate-skill-flow.sh` and this doc): + +| Token | What it is | +|-------|-----------| +| `repo-context` | the repository working tree / source under analysis | +| `external-api` | an upstream API or doc site outside the corpus | +| `bd` | the beads issue store | +| `github-pr` | a GitHub pull request under review | +| `onboard` | the session onboarding handshake | + +### 2. `metadata.dependencies` resolution + +Every `metadata.dependencies` entry MUST name an existing skill slug. + +### 3. Connectivity (no silent orphans) + +A skill is **connected** if it shares at least one skill-to-skill edge with a +peer, counting all three layers (`consumes` skill-slugs, `context_rel.with` +skill-slugs, `metadata.dependencies`). A skill with **zero** skill-to-skill +edges is an **orphan** and fails the gate **unless** it is listed in +`scripts/skill-flow-standalone.txt` with a rationale. + +Standalone skills are intentional leaves: boundary adapters (`push`, +`openai-docs`), orchestration/install adapters (`codex-team`, +`session-bootstrap`), and human-facing explainers (`using-agentops`). Listing a +skill there asserts "this is a leaf by design." If it later gains an edge, the +gate reports a **stale allowlist entry** — remove it. + +## Reported, not enforced (informational) + +The gate prints (without failing) two reconciliation signals: + +- **`consumes` vs `metadata.dependencies` disagreement** — the two skill-slug + fields that should agree but historically drifted. Reconciling them (picking + one canonical field) is tracked work, not a blocker. +- **Dead-end produced artifacts** — artifacts in some skill's `produces` that no + skill `consumes`. Most are output-type annotations (`result.json`, + `verdict.json`, `stdout`), not edges, so this is informational. + +## How to fix a failure + +```bash +bash scripts/validate-skill-flow.sh # human-readable findings +bash scripts/validate-skill-flow.sh --json # machine-readable verdict +``` + +- **consumes-vocabulary**: fix the typo, or declare the producer, or (if it is a + genuinely new external input) add it to the whitelist above and in the script. +- **metadata-dependencies**: point at a real skill slug or drop the entry. +- **orphan**: wire a real `context_rel`/`consumes`/`metadata.dependencies` edge, + or add the slug to `scripts/skill-flow-standalone.txt` with a one-line reason. diff --git a/docs/documentation-index.md b/docs/documentation-index.md index a303fdf60..0c728166f 100644 --- a/docs/documentation-index.md +++ b/docs/documentation-index.md @@ -276,6 +276,7 @@ Bridge / framing docs: - [Local Pre-Push Gate Retirement](contracts/local-pre-push-gate-retirement.md) — ADR (soc-g2r9): CI is the sole authoritative push gate; `scripts/pre-push-gate.sh` and its helpers are retired in follow-up waves; AP#7 mechanical enforcement migrates from pre-push to a validate.yml job - [Skill Dispositions (yaml)](contracts/skill-dispositions.yaml) — Canonical per-skill domain/disposition/rationale data; source-of-truth for `agentops-skill-domain-map.md`. Hand-edits to the .md forbidden — edit yaml and run `scripts/generate-skill-domain-map.sh` (golden-file gate, soc-zxia.3) - [Context Map](contracts/context-map.md) — Auto-generated bounded-context map of skills by hexagonal role with relationship and data-flow views (see ADR-0001) +- [Skill-Flow Connectivity](contracts/skill-flow.md) — Closed `consumes` vocabulary + cross-layer connectivity model (`consumes`/`context_rel`/`metadata.dependencies`); gate `scripts/validate-skill-flow.sh` (`validate-skill-flow`) fails on unresolved tokens or un-allowlisted orphans; standalone leaves in `scripts/skill-flow-standalone.txt` - [PMF Evidence Gate](contracts/pmf-evidence.md) — Public docs (PRODUCT.md, README, launch artifacts) must promote `.agents/` evidence to `docs/evidence//` via `scripts/export-evidence.sh`; `scripts/check-pmf-evidence.sh` is the gate (soc-m6v5.8) - [Skill Domain Map](contracts/skill-domain-map.md) — V0 DDD map assigning every shared skill to one explicit skill domain with ports, artifacts, and adapters - [Registry as derived artifact](contracts/registry-as-derived.md) — Design contract (soc-jbea, status:design): move `registry.json` out of version control to eliminate sibling-PR conflict cascade (40-50% of waste in the 2026-05-20 PR-cleanup session per Council 220-240). Same pattern for `skills-codex/.agentops-manifest.json` and `skills-codex/*/.agentops-generated.json`. Implementation deferred to soc-jbea.1 through soc-jbea.7. diff --git a/scripts/skill-flow-standalone.txt b/scripts/skill-flow-standalone.txt new file mode 100644 index 000000000..bd2aa4a85 --- /dev/null +++ b/scripts/skill-flow-standalone.txt @@ -0,0 +1,35 @@ +# skill-flow-standalone.txt — intentionally-standalone skills. +# +# Skills listed here are permitted to have ZERO skill-to-skill edges (no peer +# referenced in consumes/context_rel/metadata.dependencies, and not referenced +# by any peer). They are boundary adapters, meta-tooling, or human-facing +# explainers that legitimately do not participate in the inter-skill flow. +# +# Enforced by scripts/validate-skill-flow.sh. Adding a skill here is a +# deliberate act: it asserts "this skill is a leaf by design." If a skill here +# later gains an edge, the gate flags it as a stale entry — remove it. +# +# Contract: docs/contracts/skill-flow.md +# Format: one skill slug per line; everything after '#' is a comment. + +# --- knowledge / .agents boundary writers (emit .agents/research/*.md; the +# flywheel reads them out-of-band, not via a declared skill edge) --- +curate # mine transcripts/.agents/bd/git into knowledge diffs +dream # retired pointer; out-of-session compounding via Gas City +handoff # write compact session handoff notes +reverse-engineer-rpi # reverse-engineer product specs into research notes + +# --- external-boundary adapters (driven adapters to systems outside the corpus) --- +openai-docs # reads upstream OpenAI docs (external-api) +pr-research # researches an upstream OSS repo (external-api) +status # reports bd work status (external bd store) +push # commits/pushes git-changes (terminal VCS sink) + +# --- orchestration / install adapters (drive other agents or install artifacts) --- +codex-team # coordinate multiple Codex agents (orchestration) +session-bootstrap # universal init entry point (customer-of AGENTS*.md docs) +system-tuning # restore system responsiveness (process/host hygiene) + +# --- human-facing explainers (documentation skills, no pipeline role) --- +using-agentops # explain AgentOps workflows +using-gc # explain running AgentOps on the Gas City substrate diff --git a/scripts/validate-skill-flow.sh b/scripts/validate-skill-flow.sh new file mode 100755 index 000000000..cfcc05efe --- /dev/null +++ b/scripts/validate-skill-flow.sh @@ -0,0 +1,329 @@ +#!/usr/bin/env bash +# validate-skill-flow.sh — Enforce skill-flow connectivity across every +# skills//SKILL.md. +# +# Motivation (follow-up to scripts/audit-skill-metadata.sh, ag-f0i): +# audit-skill-metadata.sh owns `context_rel.with` resolution. It explicitly +# deferred two checks as "discovered follow-ups, not enforced": +# 1. `consumes` vocabulary canonicality (open vocabulary, no registry). +# 2. skill-to-skill connectivity ("do all skills flow together?"). +# This gate closes both, plus checks `metadata.dependencies` resolution. +# +# What it checks (FAIL = exit 1): +# 1. CLOSED CONSUMES VOCABULARY. Every `consumes` token must be either a real +# skill slug or one of the whitelisted EXTERNAL_INPUTS (see below). This +# turns `consumes` from an open free-text field into a closed contract so +# the producer->consumer graph can be reasoned about. +# 2. metadata.dependencies RESOLUTION. Every `metadata.dependencies` entry +# must name an existing skill slug. +# 3. ORPHAN DETECTION. A skill is "connected" if it shares at least one +# skill-to-skill edge with another skill, counting ALL THREE edge layers +# (consumes skill-slugs, context_rel.with skill-slugs, metadata.dependencies). +# Orphans must be listed in the standalone allowlist +# (scripts/skill-flow-standalone.txt) — intentionally-standalone meta / +# utility / boundary skills. An un-allowlisted orphan FAILS. +# +# What it REPORTS (informational, never fails): +# - Cross-layer disagreement: `consumes` skill-slugs vs `metadata.dependencies` +# (the two fields drifted historically; surfaced so they can be reconciled). +# - Dead-end produced artifacts (produced but consumed by no skill). +# +# context_rel.with resolution stays owned by audit-skill-metadata.sh — this gate +# does not re-litigate it (and tolerates `*.md` doc targets used by entry-point +# skills such as session-bootstrap). +# +# Usage: +# bash scripts/validate-skill-flow.sh [--json] [--skills-root DIR] [--allowlist FILE] +# +# --json emit a machine-readable verdict on stdout (stdout = data only) +# --skills-root directory holding skills//SKILL.md +# (default: /skills, or $SKILL_FLOW_SKILLS_ROOT) +# --allowlist standalone-skill allowlist file +# (default: /scripts/skill-flow-standalone.txt) +# -h, --help show this help +# +# Exit codes: 0 = clean, 1 = findings, 2 = usage / environment error. +# +# Contract reference: docs/contracts/skill-flow.md +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +JSON=0 +SKILLS_ROOT="${SKILL_FLOW_SKILLS_ROOT:-${REPO_ROOT}/skills}" +ALLOWLIST="${SKILL_FLOW_ALLOWLIST:-${REPO_ROOT}/scripts/skill-flow-standalone.txt}" + +usage() { + cat <<'USAGE' +validate-skill-flow.sh — enforce skill-flow connectivity across all SKILL.md. + +Checks (fail): closed consumes vocabulary, metadata.dependencies resolution, +and orphan detection (un-allowlisted skill with zero skill-to-skill edges). +Reports (advisory): consumes vs metadata.dependencies disagreement, dead-end +produced artifacts. + +Usage: + bash scripts/validate-skill-flow.sh [--json] [--skills-root DIR] [--allowlist FILE] + + --json emit a machine-readable verdict on stdout (stdout = data only) + --skills-root directory holding skills//SKILL.md + (default: /skills, or $SKILL_FLOW_SKILLS_ROOT) + --allowlist standalone-skill allowlist file + (default: /scripts/skill-flow-standalone.txt) + -h, --help show this help + +Exit codes: 0 = clean, 1 = findings, 2 = usage / environment error. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --json) JSON=1; shift ;; + --skills-root) SKILLS_ROOT="${2:?--skills-root needs a value}"; shift 2 ;; + --allowlist) ALLOWLIST="${2:?--allowlist needs a value}"; shift 2 ;; + -h|--help) usage; exit 0 ;; + --*) echo "ERROR: unknown flag: $1 (try: bash scripts/validate-skill-flow.sh --help)" >&2; exit 2 ;; + *) echo "ERROR: unexpected argument: $1 (try: bash scripts/validate-skill-flow.sh --help)" >&2; exit 2 ;; + esac +done + +if [[ ! -d "${SKILLS_ROOT}" ]]; then + echo "ERROR: skills directory not found at ${SKILLS_ROOT}" >&2 + exit 2 +fi + +SKILLS_ROOT="${SKILLS_ROOT}" ALLOWLIST="${ALLOWLIST}" JSON="${JSON}" python3 - <<'PYEOF' +import json +import os +import re +import sys +from pathlib import Path + +try: + import yaml # type: ignore +except Exception as e: # pragma: no cover - environment guard + sys.stderr.write("ERROR: PyYAML is required (pip install pyyaml). underlying: %s\n" % e) + sys.exit(2) + +SKILLS_ROOT = Path(os.environ["SKILLS_ROOT"]) +ALLOWLIST = Path(os.environ["ALLOWLIST"]) +JSON = os.environ.get("JSON") == "1" + +# Closed vocabulary for non-skill `consumes` tokens. These are the *external +# inputs* a skill may read that are not themselves produced by a peer skill +# (VCS state, the bd issue store, an upstream API, the repo working tree, the +# onboarding handshake). Adding a new external input is a deliberate act: extend +# this list AND document it in docs/contracts/skill-flow.md. +EXTERNAL_INPUTS = { + "repo-context", + "external-api", + "bd", + "github-pr", + "onboard", +} + +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n", re.DOTALL) + + +def parse_frontmatter(skill_md): + try: + text = skill_md.read_text(encoding="utf-8") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + try: + data = yaml.safe_load(m.group(1)) + except Exception: + return {} + return data if isinstance(data, dict) else {} + + +def str_list(value): + if not isinstance(value, list): + return [] + return [str(x) for x in value if x is not None] + + +def load_allowlist(path): + """Return set of allowlisted standalone skill slugs (# comments allowed).""" + if not path.is_file(): + return set() + out = set() + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.split("#", 1)[0].strip() + if line: + out.add(line) + return out + + +# 1. Load every skill's frontmatter. +skill_dirs = sorted( + p for p in SKILLS_ROOT.iterdir() if p.is_dir() and (p / "SKILL.md").is_file() +) +names = {p.name for p in skill_dirs} + +skills = {} +for sd in skill_dirs: + fm = parse_frontmatter(sd / "SKILL.md") + consumes = str_list(fm.get("consumes")) + produces = str_list(fm.get("produces")) + ctx = [] + for e in (fm.get("context_rel") or []): + if isinstance(e, dict) and isinstance(e.get("with"), str): + ctx.append(e["with"].strip()) + md = fm.get("metadata") or {} + mdeps = str_list(md.get("dependencies")) if isinstance(md, dict) else [] + skills[sd.name] = { + "consumes": consumes, + "produces": produces, + "ctx": ctx, + "mdeps": mdeps, + } + +allowlist = load_allowlist(ALLOWLIST) + +# Every artifact produced by some skill — a `consumes` token may legitimately +# name one (e.g. push consumes git-changes; beads consumes bd-issue). +produced_artifacts = set() +for d in skills.values(): + produced_artifacts.update(d["produces"]) + +failures = [] # list of (kind, slug, detail) + +# CHECK 1: closed consumes vocabulary. A consumes token must resolve to one of: +# a peer skill slug, a whitelisted external input, or an artifact produced by +# some skill. Anything else is a typo or an undeclared dependency. +for slug in sorted(skills): + for tok in skills[slug]["consumes"]: + if tok in names or tok in EXTERNAL_INPUTS or tok in produced_artifacts: + continue + failures.append(( + "consumes-vocabulary", + slug, + "consumes '%s' resolves to nothing: not a skill slug, not a " + "whitelisted external input (%s), and not an artifact any skill " + "produces" % (tok, ", ".join(sorted(EXTERNAL_INPUTS))), + )) + +# CHECK 2: metadata.dependencies resolution. +for slug in sorted(skills): + for tok in skills[slug]["mdeps"]: + if tok not in names: + failures.append(( + "metadata-dependencies", + slug, + "metadata.dependencies '%s' does not resolve to a skill slug" % tok, + )) + +# Build undirected skill-to-skill edge set across all three layers. +edges = set() +for slug, d in skills.items(): + for tok in d["consumes"]: + if tok in names and tok != slug: + edges.add(frozenset((slug, tok))) + for tok in d["ctx"]: + if tok in names and tok != slug: + edges.add(frozenset((slug, tok))) + for tok in d["mdeps"]: + if tok in names and tok != slug: + edges.add(frozenset((slug, tok))) + +degree = {slug: 0 for slug in skills} +for e in edges: + for slug in e: + degree[slug] += 1 + +orphans = sorted(s for s in skills if degree[s] == 0) + +# CHECK 3: orphans must be allowlisted. +unallowed_orphans = [s for s in orphans if s not in allowlist] +for slug in unallowed_orphans: + failures.append(( + "orphan", + slug, + "no skill-to-skill edge in consumes/context_rel/metadata.dependencies; " + "wire an edge or add to scripts/skill-flow-standalone.txt with a rationale", + )) + +# Stale allowlist entries (allowlisted but actually connected, or not a skill). +stale_allowlist = sorted( + s for s in allowlist if s not in skills or (s in skills and degree[s] > 0) +) + +# Informational: consumes-skill vs metadata.dependencies disagreement. +disagreements = [] +for slug in sorted(skills): + cs = {t for t in skills[slug]["consumes"] if t in names} + md = set(skills[slug]["mdeps"]) + if (cs or md) and cs != md: + disagreements.append((slug, sorted(cs), sorted(md))) + +# Informational: dead-end produced artifacts (produced, consumed by no skill). +consumed_tokens = set() +for d in skills.values(): + consumed_tokens.update(d["consumes"]) +produced = {} +for slug, d in skills.items(): + for art in d["produces"]: + produced.setdefault(art, []).append(slug) +dead_end = sorted(a for a in produced if a not in consumed_tokens) + +verdict = "PASS" if not failures else "FAIL" + +if JSON: + print(json.dumps({ + "verdict": verdict, + "skills_checked": len(skills), + "edges": len(edges), + "orphans": orphans, + "failures": [ + {"kind": k, "skill": s, "detail": d} for (k, s, d) in failures + ], + "stale_allowlist": stale_allowlist, + "disagreements": [ + {"skill": s, "consumes": cs, "metadata_dependencies": md} + for (s, cs, md) in disagreements + ], + "dead_end_artifacts": dead_end, + }, indent=2, sort_keys=True)) +else: + print("validate-skill-flow: %d skill(s), %d skill-to-skill edge(s)" % ( + len(skills), len(edges))) + print(" orphans: %d (allowlisted standalone: %d)" % ( + len(orphans), len([o for o in orphans if o in allowlist]))) + if disagreements: + print("") + print("INFO: %d skill(s) where consumes(skills) != metadata.dependencies " + "(reconcile, not fatal):" % len(disagreements)) + for slug, cs, md in disagreements: + print(" - %-26s consumes=%s metadata.deps=%s" % ( + slug, cs or "-", md or "-")) + if dead_end: + print("") + print("INFO: %d produced artifact(s) consumed by no skill " + "(output-type annotation, not fatal):" % len(dead_end)) + for art in dead_end: + print(" - %s (from: %s)" % (art, ", ".join(produced[art]))) + if stale_allowlist: + print("") + print("WARN: %d stale allowlist entry/entries (now connected or not a " + "skill — remove from scripts/skill-flow-standalone.txt):" + % len(stale_allowlist)) + for slug in stale_allowlist: + print(" - %s" % slug) + if failures: + print("") + print("FAIL: %d finding(s):" % len(failures)) + for kind, slug, detail in failures: + print(" [%s] %s/SKILL.md: %s" % (kind, slug, detail)) + print("") + print("fix: see docs/contracts/skill-flow.md") + else: + print("") + print("OK: skill flow is connected and the consumes vocabulary is closed.") + +sys.exit(0 if verdict == "PASS" else 1) +PYEOF diff --git a/skills/deps/SKILL.md b/skills/deps/SKILL.md index d17a7da67..eb68f406f 100644 --- a/skills/deps/SKILL.md +++ b/skills/deps/SKILL.md @@ -10,7 +10,9 @@ consumes: - repo-context produces: - result.json -context_rel: [] +context_rel: +- kind: supplier-to + with: vibe skill_api_version: 1 context: window: fork diff --git a/skills/provenance/SKILL.md b/skills/provenance/SKILL.md index 216d6680e..cd7b444c7 100644 --- a/skills/provenance/SKILL.md +++ b/skills/provenance/SKILL.md @@ -9,7 +9,9 @@ hexagonal_role: driven-adapter consumes: [] produces: - result.json -context_rel: [] +context_rel: +- kind: supplier-to + with: trace skill_api_version: 1 allowed-tools: Read, Grep, Glob, Bash context: diff --git a/skills/red-team/SKILL.md b/skills/red-team/SKILL.md index 7ada23d42..35703bc6b 100644 --- a/skills/red-team/SKILL.md +++ b/skills/red-team/SKILL.md @@ -10,7 +10,9 @@ consumes: - repo-context produces: - result.json -context_rel: [] +context_rel: +- kind: supplier-to + with: vibe skill_api_version: 1 metadata: tier: judgment diff --git a/skills/release/SKILL.md b/skills/release/SKILL.md index 16fbaf5ce..2e12686db 100644 --- a/skills/release/SKILL.md +++ b/skills/release/SKILL.md @@ -9,7 +9,9 @@ hexagonal_role: supporting consumes: [] produces: - result.json -context_rel: [] +context_rel: +- kind: supplier-to + with: ship-loop skill_api_version: 1 context: window: fork diff --git a/skills/scenario/SKILL.md b/skills/scenario/SKILL.md index 429292592..3e38d447f 100644 --- a/skills/scenario/SKILL.md +++ b/skills/scenario/SKILL.md @@ -9,7 +9,9 @@ hexagonal_role: supporting consumes: [] produces: - result.json -context_rel: [] +context_rel: +- kind: supplier-to + with: validation skill_api_version: 1 metadata: tier: execution diff --git a/skills/trace/SKILL.md b/skills/trace/SKILL.md index b51468e55..eadaad1c3 100644 --- a/skills/trace/SKILL.md +++ b/skills/trace/SKILL.md @@ -9,7 +9,9 @@ hexagonal_role: supporting consumes: [] produces: - result.json -context_rel: [] +context_rel: +- kind: customer-of + with: provenance skill_api_version: 1 allowed-tools: Read, Grep, Glob, Bash context: diff --git a/tests/scripts/validate-skill-flow.bats b/tests/scripts/validate-skill-flow.bats new file mode 100644 index 000000000..decf9868d --- /dev/null +++ b/tests/scripts/validate-skill-flow.bats @@ -0,0 +1,153 @@ +#!/usr/bin/env bats +# Acceptance surface for scripts/validate-skill-flow.sh — the skill-flow +# connectivity gate (follow-up to audit-skill-metadata.sh, which deferred the +# `consumes` vocabulary and connectivity checks). +# +# Contract: docs/contracts/skill-flow.md +# Fixtures live in a tmp tree so the repo-wide SKILL.md scanners never see them. + +setup() { + REPO_ROOT="$(cd "$BATS_TEST_DIRNAME/../.." && pwd)" + SCRIPT="$REPO_ROOT/scripts/validate-skill-flow.sh" + ROOT="$(mktemp -d)" + ALLOW="$(mktemp)" + : > "$ALLOW" # empty allowlist by default +} + +teardown() { + [ -n "${ROOT:-}" ] && rm -rf "$ROOT" + [ -n "${ALLOW:-}" ] && rm -f "$ALLOW" +} + +# mkskill [consumes_csv] [produces_csv] [ctx_with] [mdeps_csv] +# Writes a minimal valid SKILL.md frontmatter. CSV args are comma-separated. +mkskill() { + local root="$1" name="$2" consumes="${3:-}" produces="${4:-}" ctx="${5:-}" mdeps="${6:-}" + mkdir -p "$root/$name" + { + echo "---" + echo "name: $name" + echo "description: fixture skill $name" + echo "hexagonal_role: supporting" + echo "practices:" + echo "- tdd" + if [ -n "$consumes" ]; then + echo "consumes:" + IFS=',' read -ra items <<< "$consumes" + for i in "${items[@]}"; do echo "- $i"; done + fi + if [ -n "$produces" ]; then + echo "produces:" + IFS=',' read -ra items <<< "$produces" + for i in "${items[@]}"; do echo "- $i"; done + fi + if [ -n "$ctx" ]; then + echo "context_rel:" + echo "- kind: customer-of" + echo " with: $ctx" + fi + if [ -n "$mdeps" ]; then + echo "metadata:" + echo " dependencies:" + IFS=',' read -ra items <<< "$mdeps" + for i in "${items[@]}"; do echo " - $i"; done + fi + echo "---" + echo "# $name" + } > "$root/$name/SKILL.md" +} + +run_gate() { run bash "$SCRIPT" --skills-root "$ROOT" --allowlist "$ALLOW" "$@"; } + +@test "gate exists and is executable" { + [ -f "$SCRIPT" ] + [ -x "$SCRIPT" ] +} + +@test "two skills connected via context_rel -> PASS" { + mkskill "$ROOT" alpha "" "" beta "" + mkskill "$ROOT" beta + # beta is referenced by alpha so it is connected; alpha references beta. + run_gate + [ "$status" -eq 0 ] + [[ "$output" == *"skill flow is connected"* ]] +} + +@test "consumes a whitelisted external input -> PASS (not an orphan target failure)" { + mkskill "$ROOT" alpha "repo-context" "" beta "" + mkskill "$ROOT" beta + run_gate + [ "$status" -eq 0 ] +} + +@test "consumes an artifact produced by another skill -> vocabulary PASS" { + mkskill "$ROOT" producer "" "git-changes" alpha "" + mkskill "$ROOT" alpha "git-changes" "" producer "" + run_gate + [ "$status" -eq 0 ] +} + +@test "consumes a dangling token -> FAIL with consumes-vocabulary finding" { + mkskill "$ROOT" alpha "totally-bogus" "" beta "" + mkskill "$ROOT" beta + run_gate + [ "$status" -eq 1 ] + [[ "$output" == *"consumes-vocabulary"* ]] + [[ "$output" == *"totally-bogus"* ]] + [[ "$output" == *"alpha/SKILL.md"* ]] +} + +@test "metadata.dependencies pointing at non-skill -> FAIL" { + mkskill "$ROOT" alpha "" "" "" "ghost" + mkskill "$ROOT" beta "" "" alpha "" + run_gate + [ "$status" -eq 1 ] + [[ "$output" == *"metadata-dependencies"* ]] + [[ "$output" == *"ghost"* ]] +} + +@test "un-allowlisted orphan -> FAIL" { + mkskill "$ROOT" alpha "" "" beta "" + mkskill "$ROOT" beta + mkskill "$ROOT" lonely "repo-context" "result.json" # zero skill edges + run_gate + [ "$status" -eq 1 ] + [[ "$output" == *"[orphan] lonely/SKILL.md"* ]] +} + +@test "allowlisted orphan -> PASS" { + mkskill "$ROOT" alpha "" "" beta "" + mkskill "$ROOT" beta + mkskill "$ROOT" lonely "repo-context" "result.json" + echo "lonely # boundary leaf" > "$ALLOW" + run_gate + [ "$status" -eq 0 ] +} + +@test "metadata.dependencies edge alone counts as connectivity" { + # alpha has no consumes/context_rel; only metadata.dependencies -> beta. + mkskill "$ROOT" alpha "" "" "" "beta" + mkskill "$ROOT" beta + run_gate + [ "$status" -eq 0 ] +} + +@test "--json emits machine-readable verdict with failures and orphans" { + mkskill "$ROOT" alpha "totally-bogus" "" beta "" + mkskill "$ROOT" beta + run bash "$SCRIPT" --skills-root "$ROOT" --allowlist "$ALLOW" --json + [ "$status" -eq 1 ] + echo "$output" | jq empty + [ "$(echo "$output" | jq -r '.verdict')" = "FAIL" ] + [ "$(echo "$output" | jq -r '.failures[0].kind')" = "consumes-vocabulary" ] + [ "$(echo "$output" | jq -r '.failures[0].skill')" = "alpha" ] +} + +@test "consumes vs metadata.dependencies disagreement is reported, not fatal" { + mkskill "$ROOT" alpha "beta" "" "" "gamma" + mkskill "$ROOT" beta + mkskill "$ROOT" gamma + run_gate + [ "$status" -eq 0 ] + [[ "$output" == *"consumes(skills) != metadata.dependencies"* ]] +}