[AIAA] Add deterministic data-collection scripts

.github/workflows/assessment-test.yml

@@ -0,0 +1,19 @@
+name: Assessment tests
+
+on:
+  pull_request:
+    paths:
+      - scripts/assessment/**
+      - .github/workflows/assessment-test.yml
+
+permissions:
+  contents: read
+
+jobs:
+  assessment-test:
+    name: ASSESSMENT tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - uses: ./.github/actions/npm-ci-via-cached-nvmrc
+      - run: npm run test:assessment

.prettierignore

@@ -1 +1,5 @@
 /docs/localization/ja
+
+# Test fixtures are frozen inputs; formatting would change the bytes
+# the assessment collection tests hash.
+/scripts/assessment/test/fixtures/

package.json

@@ -37,6 +37,7 @@
     "precheck:links": "npm run build",
     "seq": "bash -c 'for cmd in \"$@\"; do npm run $cmd || exit 1; done' - ",
     "serve": "npm run docus:serve",
+    "test:assessment": "node --test scripts/assessment/test/*.test.mjs",
     "test:unit": "node --experimental-strip-types --test 'lib/**/*.test.mts'",
     "test": "npm run check && npm run test:unit",
     "typecheck": "tsc",

scripts/assessment/collect.mjs

@@ -0,0 +1,142 @@
+#!/usr/bin/env node
+// Deterministic data collection for documentation assessments.
+// Inventories one or more repository checkouts, extracts markdown
+// links, optionally fetches sites and checks external links, and
+// writes a content-hash manifest so every quantitative claim traces
+// to a committed, re-runnable step and post-collection edits are
+// detectable.
+//
+//   node scripts/assessment/collect.mjs --repo <path> [--repo <path>]
+//     [--site <url>] [--check-links] --out <dir>
+//   node scripts/assessment/collect.mjs verify --out <dir>
+
+import path from 'node:path';
+import process from 'node:process';
+import { buildInventory, markdownPaths } from './lib/inventory.mjs';
+import { buildLinkReport } from './lib/links.mjs';
+import { resolveHeadSha } from './lib/git.mjs';
+import { checkLinks, fetchSites } from './lib/sitefetch.mjs';
+import { verifyManifest, writeCollection } from './lib/manifest.mjs';
+import { stableStringify } from './lib/util.mjs';
+
+const USAGE =
+  'usage: collect.mjs [verify] --repo <path> [--site <url>] [--check-links] --out <dir>';
+
+function parseArgs(argv) {
+  const args = {
+    verify: false,
+    repos: [],
+    sites: [],
+    checkLinks: false,
+    out: null,
+  };
+  let i = 0;
+  if (argv[0] === 'verify') {
+    args.verify = true;
+    i = 1;
+  }
+  for (; i < argv.length; i += 1) {
+    const flag = argv[i];
+    if (flag === '--repo') args.repos.push(argv[(i += 1)]);
+    else if (flag === '--site') args.sites.push(argv[(i += 1)]);
+    else if (flag === '--out') args.out = argv[(i += 1)];
+    else if (flag === '--check-links') args.checkLinks = true;
+    else throw new Error(`unknown flag: ${flag}`);
+  }
+  if (!args.out) throw new Error('missing --out');
+  if (!args.verify && args.repos.length === 0 && args.sites.length === 0) {
+    throw new Error('nothing to collect: pass --repo or --site');
+  }
+  if (args.repos.some((v) => v == null) || args.sites.some((v) => v == null)) {
+    throw new Error('flag missing its value');
+  }
+  return args;
+}
+
+// The manifest records the command without --out: the output location
+// is wherever the manifest lives, and omitting it keeps two runs into
+// different directories byte-identical.
+function commandLine(args) {
+  const parts = ['collect.mjs'];
+  for (const repo of args.repos) parts.push('--repo', repo);
+  for (const site of args.sites) parts.push('--site', site);
+  if (args.checkLinks) parts.push('--check-links');
+  return parts.join(' ');
+}
+
+async function collect(args) {
+  const outputs = {};
+  const repos = [];
+  const repoReports = [];
+  const linkReports = [];
+  for (const repoPath of args.repos) {
+    const inventory = buildInventory(repoPath);
+    const report = buildLinkReport(repoPath, markdownPaths(inventory));
+    repos.push({ path: repoPath, sha: resolveHeadSha(repoPath) });
+    repoReports.push({ path: repoPath, ...inventory });
+    linkReports.push({ path: repoPath, ...report });
+  }
+  outputs['inventory.json'] = stableStringify({ repos: repoReports }) + '\n';
+  outputs['links.json'] = stableStringify({ repos: linkReports }) + '\n';
+
+  const sites = [];
+  if (args.sites.length > 0) {
+    const fetched = await fetchSites(args.sites);
+    fetched.forEach((entry, index) => {
+      const { body, ...meta } = entry;
+      const host = new URL(entry.url).host;
+      if (body) {
+        const name = `sites/${String(index + 1).padStart(3, '0')}-${host}.body`;
+        outputs[name] = body;
+        meta.bodyFile = name;
+      }
+      sites.push(meta);
+    });
+    outputs['site-fetches.json'] = stableStringify({ sites }) + '\n';
+  }
+
+  if (args.checkLinks) {
+    const external = [
+      ...new Set(linkReports.flatMap((report) => report.external)),
+    ].sort();
+    outputs['link-status.json'] =
+      stableStringify({ checked: await checkLinks(external) }) + '\n';
+  }
+
+  writeCollection(args.out, {
+    command: commandLine(args),
+    repos,
+    sites,
+    outputs,
+  });
+  process.stdout.write(`collected into ${args.out}\n`);
+}
+
+function verify(args) {
+  const result = verifyManifest(args.out);
+  if (result.ok) {
+    process.stdout.write('manifest ok\n');
+    return;
+  }
+  for (const name of result.mismatched) {
+    process.stderr.write(`mismatched: ${name}\n`);
+  }
+  for (const name of result.missing) {
+    process.stderr.write(`missing: ${name}\n`);
+  }
+  process.exit(1);
+}
+
+async function main() {
+  let args;
+  try {
+    args = parseArgs(process.argv.slice(2));
+  } catch (err) {
+    process.stderr.write(`${err.message}\n${USAGE}\n`);
+    process.exit(2);
+  }
+  if (args.verify) verify(args);
+  else await collect(args);
+}
+
+await main();

scripts/assessment/lib/git.mjs

@@ -0,0 +1,17 @@
+import { execFileSync } from 'node:child_process';
+
+function defaultExec(cmd, args) {
+  return execFileSync(cmd, args, {
+    encoding: 'utf8',
+    stdio: ['ignore', 'pipe', 'pipe'],
+  });
+}
+
+export function resolveHeadSha(dir, execImpl = defaultExec) {
+  try {
+    const out = execImpl('git', ['-C', dir, 'rev-parse', 'HEAD']).trim();
+    return /^[0-9a-f]{40}$/i.test(out) ? out : null;
+  } catch {
+    return null;
+  }
+}

scripts/assessment/lib/inventory.mjs

@@ -0,0 +1,55 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { sha256Hex } from './util.mjs';
+
+const MARKDOWN_EXTENSIONS = new Set(['.md', '.mdx', '.markdown']);
+const SKIPPED_DIRECTORIES = new Set(['.git', 'node_modules']);
+
+function walk(root, rel, paths) {
+  const entries = fs.readdirSync(path.join(root, rel), {
+    withFileTypes: true,
+  });
+  for (const entry of entries) {
+    if (entry.isSymbolicLink()) continue;
+    const relPath = rel ? `${rel}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) {
+      if (!SKIPPED_DIRECTORIES.has(entry.name)) walk(root, relPath, paths);
+    } else if (entry.isFile()) {
+      paths.push(relPath);
+    }
+  }
+}
+
+export function buildInventory(root) {
+  const paths = [];
+  walk(root, '', paths);
+  paths.sort();
+  const files = paths.map((relPath) => {
+    const content = fs.readFileSync(path.join(root, relPath));
+    return { path: relPath, bytes: content.length, sha256: sha256Hex(content) };
+  });
+  const byExtension = {};
+  let markdownCount = 0;
+  for (const file of files) {
+    const ext = path.posix.extname(file.path).toLowerCase();
+    if (ext) byExtension[ext] = (byExtension[ext] ?? 0) + 1;
+    if (MARKDOWN_EXTENSIONS.has(ext)) markdownCount += 1;
+  }
+  return {
+    files,
+    totals: {
+      fileCount: files.length,
+      byteCount: files.reduce((n, f) => n + f.bytes, 0),
+      markdownCount,
+      byExtension,
+    },
+  };
+}
+
+export function markdownPaths(inventory) {
+  return inventory.files
+    .map((f) => f.path)
+    .filter((p) =>
+      MARKDOWN_EXTENSIONS.has(path.posix.extname(p).toLowerCase()),
+    );
+}

scripts/assessment/lib/links.mjs

@@ -0,0 +1,44 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const INLINE_LINK = /!?\[[^\]]*\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+const REFERENCE_DEFINITION = /^\s*\[[^\]]+\]:\s*(\S+)/gm;
+const AUTOLINK = /<(https?:\/\/[^>\s]+)>/g;
+
+function classify(url) {
+  if (url.startsWith('#')) return 'anchor';
+  if (/^https?:\/\//i.test(url)) return 'external';
+  return 'internal';
+}
+
+export function extractLinks(markdown) {
+  const urls = new Set();
+  for (const pattern of [INLINE_LINK, REFERENCE_DEFINITION, AUTOLINK]) {
+    for (const match of markdown.matchAll(pattern)) {
+      urls.add(match[1]);
+    }
+  }
+  return [...urls].sort().map((url) => ({ url, kind: classify(url) }));
+}
+
+export function buildLinkReport(root, markdownRelPaths) {
+  const perFile = {};
+  const byKind = {
+    external: new Set(),
+    internal: new Set(),
+    anchor: new Set(),
+  };
+  for (const relPath of markdownRelPaths) {
+    const links = extractLinks(
+      fs.readFileSync(path.join(root, relPath), 'utf8'),
+    );
+    perFile[relPath] = links;
+    for (const link of links) byKind[link.kind].add(link.url);
+  }
+  return {
+    perFile,
+    external: [...byKind.external].sort(),
+    internal: [...byKind.internal].sort(),
+    anchors: [...byKind.anchor].sort(),
+  };
+}

scripts/assessment/lib/manifest.mjs

@@ -0,0 +1,51 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { sha256Hex, stableStringify } from './util.mjs';
+
+export const MANIFEST_NAME = 'manifest.json';
+
+export function buildManifest({ command, repos, sites, outputs }) {
+  const hashed = {};
+  for (const name of Object.keys(outputs).sort()) {
+    const content = Buffer.from(outputs[name]);
+    hashed[name] = { bytes: content.length, sha256: sha256Hex(content) };
+  }
+  return { command, sources: { repos, sites }, outputs: hashed };
+}
+
+export function writeCollection(dir, { command, repos, sites, outputs }) {
+  const manifest = buildManifest({ command, repos, sites, outputs });
+  for (const [name, content] of Object.entries(outputs)) {
+    const target = path.join(dir, name);
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.writeFileSync(target, content);
+  }
+  fs.writeFileSync(
+    path.join(dir, MANIFEST_NAME),
+    stableStringify(manifest) + '\n',
+  );
+  return manifest;
+}
+
+export function verifyManifest(dir) {
+  const manifest = JSON.parse(
+    fs.readFileSync(path.join(dir, MANIFEST_NAME), 'utf8'),
+  );
+  const mismatched = [];
+  const missing = [];
+  for (const name of Object.keys(manifest.outputs).sort()) {
+    const target = path.join(dir, name);
+    if (!fs.existsSync(target)) {
+      missing.push(name);
+      continue;
+    }
+    if (sha256Hex(fs.readFileSync(target)) !== manifest.outputs[name].sha256) {
+      mismatched.push(name);
+    }
+  }
+  return {
+    ok: mismatched.length === 0 && missing.length === 0,
+    mismatched,
+    missing,
+  };
+}

scripts/assessment/lib/sitefetch.mjs

@@ -0,0 +1,46 @@
+import { sha256Hex } from './util.mjs';
+
+function isoDate(now) {
+  return now().toISOString().slice(0, 10);
+}
+
+export async function fetchSites(
+  urls,
+  { fetchImpl = fetch, now = () => new Date() } = {},
+) {
+  const results = [];
+  for (const url of [...new Set(urls)].sort()) {
+    try {
+      const res = await fetchImpl(url);
+      const body = Buffer.from(await res.arrayBuffer());
+      results.push({
+        url,
+        status: res.status,
+        retrievedDate: isoDate(now),
+        sha256: sha256Hex(body),
+        body,
+      });
+    } catch (err) {
+      results.push({
+        url,
+        status: null,
+        retrievedDate: isoDate(now),
+        error: err.message,
+      });
+    }
+  }
+  return results;
+}
+
+export async function checkLinks(urls, { fetchImpl = fetch } = {}) {
+  const results = [];
+  for (const url of [...new Set(urls)].sort()) {
+    try {
+      const res = await fetchImpl(url, { method: 'HEAD', redirect: 'follow' });
+      results.push({ url, status: res.status });
+    } catch (err) {
+      results.push({ url, status: null, error: err.message });
+    }
+  }
+  return results;
+}