diff --git a/.github/workflows/smoke.yaml b/.github/workflows/smoke.yaml index 543c75809..1e0ddac0a 100644 --- a/.github/workflows/smoke.yaml +++ b/.github/workflows/smoke.yaml @@ -41,7 +41,7 @@ jobs: - name: Setup workspace run: | - npm install + npm run install:all -- --omit=optional mkdir -p tmp && cd tmp && npm init -y shell: bash diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a0353797e..4d6bf9f0c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -12,7 +12,7 @@ on: workflow_dispatch: permissions: contents: read - + jobs: build: runs-on: ${{ matrix.os }} @@ -29,7 +29,7 @@ jobs: cache: npm cache-dependency-path: package.json - - run: npm install --omit=optional + - run: npm run install:all -- --omit=optional - if: runner.os != 'Windows' run: npm run ci:test diff --git a/package.json b/package.json index ef4c236e7..e53db95c9 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "scripts": { "__spv": "node scripts/set-package-version/index.mjs", "_check:afdocs": "npm run -s -C docsy.dev _check:afdocs", - "_check:format": "npx prettier --check theme/assets *.md theme/i18n scripts tasks", + "_check:format": "npx prettier --check theme/assets *.md theme/i18n scripts tasks tests", "_check:links": "npm run -C docsy.dev _check:links", "_check:markdown": "npx markdownlint-cli2", "_commit:public": "npm run -C docsy.dev _commit:public", @@ -23,9 +23,9 @@ "_prepare": "npm run _sync:theme-deps && npm run _cp:bs-rfs && npm run _prepare:scrollspy-patch && npm run _refresh-forward-sass-var && npm run _gen-chroma-styles && npm run get:hugo-modules", "_refresh-forward-sass-var": "bash -c scripts/refresh-sass-variables.pl", "_serve": "npm run -C docsy.dev _serve --", - "_sync:theme-deps": "node scripts/sync-theme-deps.mjs", "_spv:example": "echo TBC - npm run -s __spv docsy.dev/config/example/params.yaml --", "_spv": "npm run -s __spv docsy.dev/config/*/params.yaml --", + "_sync:theme-deps": "node scripts/sync-theme-deps.mjs", "build": "npm run -C docsy.dev build --", "check:afdocs:dev": "npm run -s _check:afdocs -- http://localhost:1313 | tee docsy.dev/content/en/docs/content/agent-support/afdocs-scorecard.txt", "check:format": "npm list prettier && npm run _check:format || (echo '[help] Run: npm run fix:format'; exit 1)", @@ -44,6 +44,7 @@ "fix:markdown": "npm run check:markdown -- --fix", "fix": "npm run fix:format && npm run fix:markdown && npm run -C docsy.dev fix", "get:hugo-modules": "node scripts/getHugoModules/index.mjs", + "install:all": "npm run docsy.dev-install && npm install", "post-update": "echo; echo 'IMPORTANT! Run the following in case the ScrollSpy patch needs to be updated:\n npm run _prepare'; echo", "postinstall": "npm run _mkdir:hugo-mod", "postupdate:dep": "npm run _sync:theme-deps && npm run -s post-update", @@ -56,6 +57,8 @@ "set:version:example": "cd ../docsy-example && node ../docsy/scripts/set-package-version/index.mjs", "set:version:git-info": "npm run -s _spv -- --version \"$(scripts/get-build-id.sh)\"", "set:version": "npm run -s _spv --", + "test:smoke-note": "echo 'FIXME: test:smoke pins --branch task/repo-reorg-2026-05 during the TOF rollout; remove that flag once the move merges to main (issue #2617).'", + "test:smoke": "node tests/smoke.test.mjs --branch task/repo-reorg-2026-05", "test:tooling": "node --test 'scripts/**/*.test.mjs'", "test:website": "npm run -C docsy.dev test", "test": "npm run fix-and-test", diff --git a/scripts/_gen-chroma-style.sh b/scripts/_gen-chroma-style.sh index 384f05915..fb9446b6a 100755 --- a/scripts/_gen-chroma-style.sh +++ b/scripts/_gen-chroma-style.sh @@ -4,7 +4,8 @@ set -eo pipefail -HUGO="npx hugo" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HUGO="node $SCRIPT_DIR/run-hugo.mjs" CHROMA_STYLE= DEST_DIR=theme/assets/scss/td/chroma DEST_FILE= diff --git a/scripts/getHugoModules/index.mjs b/scripts/getHugoModules/index.mjs index 6d6a83309..d72b51693 100644 --- a/scripts/getHugoModules/index.mjs +++ b/scripts/getHugoModules/index.mjs @@ -2,12 +2,15 @@ // It gets dependency versions from `package.json`. import fs from 'fs'; +import path from 'path'; import { execSync } from 'child_process'; +const SCRIPT_DIR = path.join(process.cwd(), 'scripts'); const packageJson = readPackageJson(); let exitStatus = 0; const exit = () => process.exit(exitStatus); +const hugoCmd = () => `node ${SCRIPT_DIR}/run-hugo.mjs`; function getHugoModule(npmPkgNm, hugoModuleRefAtV) { try { @@ -21,7 +24,7 @@ function getHugoModule(npmPkgNm, hugoModuleRefAtV) { throw new Error(msg); } - const command = `npx hugo mod get ${hugoModuleRefAtV}${pkgVers}`; + const command = `${hugoCmd()} mod get ${hugoModuleRefAtV}${pkgVers}`; console.log(`> (cd theme && ${command})`); const output = execSync(command, { cwd: 'theme' }); console.log(output.toString()); diff --git a/scripts/make-site.sh b/scripts/make-site.sh index 35720c90a..2cfc480ef 100755 --- a/scripts/make-site.sh +++ b/scripts/make-site.sh @@ -2,13 +2,15 @@ # cSpell:ignore autoprefixer docsy postcss themesdir github oneline set -eo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + DEPS="autoprefixer postcss-cli" DOCSY_REPO_DEFAULT="google/docsy" DOCSY_REPO=$DOCSY_REPO_DEFAULT DOCSY_VERS="" DOCSY_SRC="NPM" FORCE_DELETE=false -: ${HUGO:=npx hugo} +: "${HUGO:=node $SCRIPT_DIR/run-hugo.mjs}" SITE_NAME="test-site" THEMESDIR="node_modules" VERBOSE=1 diff --git a/scripts/run-hugo.mjs b/scripts/run-hugo.mjs new file mode 100644 index 000000000..1148943dd --- /dev/null +++ b/scripts/run-hugo.mjs @@ -0,0 +1,49 @@ +#!/usr/bin/env node +// Resolve the installed `hugo-extended` and exec it with the forwarded args. +// Reuses docsy.dev's install (the repo's single hugo-extended declaration, +// version-pinned via package.json `config.hugo_version`) rather than `npx hugo`. +// Cross-OS: runs the package's Node bin wrapper, not a POSIX symlink / Win shim. + +import { spawnSync } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..', +); +const searchBases = [path.join(repoRoot, 'docsy.dev'), repoRoot]; + +function resolveHugoBin() { + for (const base of searchBases) { + // By node_modules path, not require.resolve('hugo-extended/package.json'): + // the package's `exports` blocks that (ERR_PACKAGE_PATH_NOT_EXPORTED). + const pkgDir = path.join(base, 'node_modules', 'hugo-extended'); + const pkgJson = path.join(pkgDir, 'package.json'); + if (!existsSync(pkgJson)) continue; + const { bin } = JSON.parse(readFileSync(pkgJson, 'utf8')); + const rel = typeof bin === 'string' ? bin : bin?.hugo; + if (rel) return path.join(pkgDir, rel); + } + return null; +} + +const hugoBin = resolveHugoBin(); +if (!hugoBin) { + console.error( + '[run-hugo] hugo-extended not found — run `npm run install:all` first.', + ); + process.exit(127); +} + +const { status, error } = spawnSync( + process.execPath, + [hugoBin, ...process.argv.slice(2)], + { stdio: 'inherit' }, +); +if (error) { + console.error(`[run-hugo] ${error.message}`); + process.exit(1); +} +process.exit(status ?? 1); diff --git a/tasks/0.16/repo-reorg/README.md b/tasks/0.16/repo-reorg/README.md index 90ea0c4f1..c1a350919 100644 --- a/tasks/0.16/repo-reorg/README.md +++ b/tasks/0.16/repo-reorg/README.md @@ -22,25 +22,32 @@ Upstream tracking: issue [#2617][]. [spike]: ./spike-notes.md [analysis]: ./monorepo-extra-analysis.md [#2617]: https://github.com/google/docsy/issues/2617 +[#2640]: https://github.com/google/docsy/pull/2640 ## Status at a glance -Updated 2026-05-24. Per-phase detail (with exit criteria) lives in the -[execution plan][exec]. +Updated 2026-05-25. Per-phase detail (with exit criteria) lives in the +[execution plan][exec]. **Phases 2 and 3 are now iterated as a pair** (like 0 +and 1): a local-only smoke pass does not close Phase 2 until the same matrix is +green in CI. -| Phase | Status | -| ------------------------------------ | ----------------------------------------------------- | -| 0 — structural move | Landed; `_prepare` + `_diff:check` regression pending | -| 1 — `docsy.dev` consumes TOF | Local build green; Netlify preview pending | -| 2 — local smoke tests (CI emulation) | Pending; `make-site.sh` paths updated | -| 3 — GitHub CI | Pending | -| 4 — `docsy-example` | Pending (post-pre-release) | -| 5 — docs and release notes | Pending | +| Phase | Status | +| ------------------------------------ | --------------------------------------------------------------- | +| 0 — structural move | Merged; `_prepare` + `_diff:check` regression pending | +| 1 — `docsy.dev` consumes TOF | Local build green; Netlify deploy preview green ([#2640][]) | +| 2 — local smoke tests (CI emulation) | Done — CI smoke matrix green ([#2640][]) | +| 3 — GitHub CI | Done — run-hugo + install:all; full CI matrix green ([#2640][]) | +| 4 — `docsy-example` | Pending (post-pre-release) | +| 5 — docs and release notes | Pending | Next concrete steps, in order: -1. Re-run `npm run _prepare` and `npm run _diff:check` end-to-end against the - new layout to close out Phase 0. -2. Push the branch and confirm a Netlify deploy preview of `docsy.dev` builds. -3. Run `make-site.sh -s NPM`, then `-s HUGO_MODULE`, then a manual non-module - `themes/docsy/` clone-and-build. Record each in [spike-notes][spike]. +1. **Merge PR [#2640][]** — the full CI matrix is green (the Phase 3 decision + gate). The canonical move to `main` follows per the plan. +2. Phase 0 carry-over: re-run `npm run _prepare` and `npm run _diff:check` + end-to-end against the new layout. +3. Phase 4: land the `docsy-example` import-path bump (post pre-release). +4. Phase 5: update the get-started "clone" docs for the new non-module setup + procedure recorded in [spike-notes][spike] Phase 2, plus changelog/blog. +5. Later: formalize the smoke matrix as a local Vitest/TS harness — see the + execution plan's "Local test harness" section. diff --git a/tasks/0.16/repo-reorg/spike-notes.md b/tasks/0.16/repo-reorg/spike-notes.md index c0c76e36a..bf4982143 100644 --- a/tasks/0.16/repo-reorg/spike-notes.md +++ b/tasks/0.16/repo-reorg/spike-notes.md @@ -149,23 +149,216 @@ npm run docsy.dev-install # docsy.dev deps (site build + hugo-extended); npm run build # green ``` -Netlify deploy preview verification (the second half of the Phase 1 exit -criterion) is still pending. +The second half of the Phase 1 exit criterion — a green Netlify deploy preview — +is confirmed on [#2640](https://github.com/google/docsy/pull/2640). -## Phase 2: NPM (GitHub) — pending under current architecture +### Note: `docsy.dev` is a distinct (sibling-folder) install shape -The Phase 2 work (`make-site.sh -s NPM`, `-s HUGO_MODULE`, and the non-module -`themes/docsy/` install) is still to be run against the current architecture. -`make-site.sh` itself has been updated for the `/theme` suffix in both NPM and -HUGO_MODULE paths, but the runs themselves have not yet been executed. +`docsy.dev` consumes Docsy as an **in-repo sibling folder**, which is a fourth +shape beyond the three smoke modes (NPM, Hugo module, non-module clone): -Note: an earlier iteration in this spike tested an `npm pack` + consumer install -flow under a different architecture (root as a thin shim with a -`files: [theme, …]` whitelist, no devDeps, and a `prepare` script that synced -theme deps to root). That architecture was deferred to a follow-on plan; the -consumer-install footprint under the current shipped architecture needs to be -re-measured as part of Phase 2. +- **Theme access:** Hugo runs from `docsy.dev/` with `--themesDir ../..` (the + parent of the repo) and `theme: [docsy/theme]`, so `docsy/theme` re-descends + into the sibling `theme/` of the same repo. (Relies on the repo dir being + named `docsy`.) +- **Theme deps:** `docsy.dev`'s `postinstall` → `_install-theme-deps` + (`npm install ../theme --install-links --no-save && rm -rf node_modules/theme`) + installs the theme's runtime deps into `docsy.dev/node_modules/`, where Hugo's + `node_modules/bootstrap` mount resolves via the consumer-cwd lookup. +- **Module placeholders:** the theme's `imports:` resolve to the empty + `github.com/...` dirs the repo-root `npm install` postinstall creates at `..` + (the parent of the repo), which is exactly where `--themesDir ../..` looks. -## Phase 3: GitHub CI — pending +This shape is **already exercised** every time the `docsy.dev` site builds (it +is the Phase 1 exit criterion), so it is covered — just not in the `node:test` +smoke driver. For now it is treated as a maintainer/in-repo shape. It could +later be surfaced and documented as a supported **power-user** consumer setup +(and, if so, get its own smoke case); deferred until we decide to document it as +such. -To be recorded during Phase 3 work. +## Phase 2: local smoke tests (CI emulation) + +Ran all three install modes locally against the integration branch +(`google/docsy@task/repo-reorg-2026-05`). Each mode built a runnable site with +full styling (the compiled `main.min.css` is ~370 KB and the Font Awesome +`webfonts/` are emitted, confirming the Bootstrap + Font Awesome SCSS resolved — +not just an empty CSS shell). + +Local-run detail: `make-site.sh` defaults `HUGO` to `node scripts/run-hugo.mjs` +(see Phase 3), which resolves `docsy.dev`'s extended Hugo — so a local run needs +no manual Hugo setup beyond `npm run install:all` (an explicit `HUGO=` export +still overrides it). + +### Result matrix + +| # | Install mode | Consumer config edit (the one line) | Builds? | Result | +| --- | ---------------- | ----------------------------------------------------------------- | ------- | ---------------------------------------------------------- | +| 1 | Hugo module | import path `github.com/google/docsy` → `…/docsy/theme` | ✅ | one-line change | +| 2 | NPM (GitHub) | `theme: docsy` → `theme: docsy/theme` (`themesDir: node_modules`) | ✅ | one-line change | +| 3 | Non-module clone | `theme: docsy` → `theme: docsy/theme` | ✅ | one-line config; setup procedure gained a step (see below) | + +### Mode 2 — NPM (GitHub) + +```sh +mkdir -p tmp && cd tmp +# CI does: ../scripts/make-site.sh -s NPM -r $PR_REPO -v $BRANCH +../scripts/make-site.sh -s NPM -r google/docsy -v task/repo-reorg-2026-05 +``` + +The consumer-facing edit is the single `theme:` line. Under the hood: +`npm install` of the GitHub package lands the whole repo at +`node_modules/docsy/`, so `theme: docsy/theme` + `themesDir: node_modules` +resolves the theme at `node_modules/docsy/theme/`. The root `postinstall` +(`_mkdir:hugo-mod`) ran in the installed package and created the empty +Hugo-module placeholder dirs `node_modules/github.com/twbs/bootstrap` and +`node_modules/github.com/FortAwesome/Font-Awesome` (the documented "NPM install +side-effect"), so the theme's `imports:` resolve; the real SCSS comes from the +`node_modules/bootstrap` / `node_modules/@fortawesome/fontawesome-free` mounts +(brought in as deps of the `docsy` package). `mkdirp-hugo-mod.js` already reads +`theme/go.mod`, so no change was needed there. **Result: one-line change.** + +### Mode 1 — Hugo module + +```sh +cd tmp +# CI does: ../scripts/make-site.sh -s HUGO_MODULE -r $PR_REPO -v $BRANCH +../scripts/make-site.sh -s HUGO_MODULE -r google/docsy -v task/repo-reorg-2026-05 +``` + +The consumer-facing edit is the module import path gaining the `/theme` suffix: + +```diff + module: + imports: +- - path: github.com/google/docsy ++ - path: github.com/google/docsy/theme +``` + +Bootstrap and Font Awesome are supplied by the theme's own Hugo-module +`imports:` (`github.com/twbs/bootstrap`, `github.com/FortAwesome/Font-Awesome`), +so no `node_modules` are needed in this mode. With `-r google/docsy` (the +canonical module path) `make-site.sh` runs +`hugo mod get github.com/google/docsy/theme@` directly. (For a fork whose +branch lives off the canonical path, it instead clones the fork and writes a +`replace … => ./tmp/docsy/theme` directive — a harness mechanism, not part of +the user-facing migration.) **Result: one-line change.** + +### Mode 3 — non-module clone into `themes/docsy/` + +The Hugo **config** edit is one line (`theme: docsy` → `theme: docsy/theme`), +and the clone still lands at `themes/docsy/`. But the **setup procedure** +changed versus pre-TOF and needs a doc update in Phase 5. Exact working flow +(run from the site root; site created with `hugo new site --format yaml`): + +```sh +# 1. Clone the theme (whole repo) into themes/docsy +cd themes +git clone -b https://github.com/google/docsy +# 2. Install the theme's runtime deps INTO the theme/ subfolder +(cd docsy/theme && npm install) +# 3. Create the empty Hugo-module placeholder dirs under themesDir (themes/) +(cd docsy && node scripts/mkdirp-hugo-mod.js ..) +cd .. +# 4. PostCSS at the site root (unchanged prerequisite) +npm install --save-dev autoprefixer postcss-cli +# 5. config: theme: docsy/theme +``` + +Why steps 2 and 3 are now separate (pre-TOF, a single +`cd themes/docsy && npm install` did both): + +- The theme root Hugo sees is now `themes/docsy/theme/`, one level deeper. + Hugo's `node_modules/bootstrap` mount is resolved theme-dir-relative first, so + the deps must be installed at `themes/docsy/theme/node_modules/`. Verified: + with the deps one level up at `themes/docsy/node_modules/` the build fails + with + `File to import not found or unreadable: ../../vendor/bootstrap/scss/functions`; + moving them into `themes/docsy/theme/node_modules/` fixes it. +- `theme/package.json` is a script-less private mirror (no `postinstall`), by + design (acceptance criteria: "No other lifecycle scripts"). So the empty + `github.com/...` placeholder dirs that the root `postinstall` used to create + must now be generated explicitly with `mkdirp-hugo-mod.js` (step 3), which + writes them under `themesDir` (`themes/github.com/...`) where Hugo looks up + the theme's `imports:` for a non-module site. + +**Result: one-line config change ✅, builds ✅** (8 pages, ~370 KB CSS, webfonts +emitted). The config promise holds. The get-started "clone" docs (Option 2 in +`other-options.md`) need the updated multi-step install above — tracked for +Phase 5. No design change is required for the move to land; the clone path works +as documented here. + +### Phase 2 exit criterion — met (with Phase 3) + +All three install modes build with the same Hugo-resolution mechanism CI uses +(`run-hugo.mjs`, no `HUGO` override), and the CI smoke matrix is green (see +Phase 3). The earlier local-only passes had relied on an ad-hoc `HUGO` override, +which masked the CI gap that `run-hugo.mjs` + `install:all` then closed — so +Phases 2 and 3 closed together, as the same install matrix on two surfaces. +Standing follow-up for Phase 5: get-started clone docs. + +## Phase 3: GitHub CI — resolved (decision E); CI matrix green + +**Symptom:** the smoke matrix (`smoke.yaml`) fails on both OSes because the +throwaway site cannot find a Hugo executable. + +**Root cause:** `docsy.dev` used to be an npm **workspace**, so root +`npm install` hoisted `hugo-extended` (declared in `docsy.dev/devDependencies`) +up into the repo-root `node_modules/.bin/`. The smoke test runs `make-site.sh` +from `tmp/`, and its default `npx hugo` walks _up_ the tree, resolving +`repo-root/node_modules/.bin/hugo`. Under TOF, `workspaces: []`, so root +`npm install` no longer pulls `docsy.dev`'s deps — `hugo-extended` is gone from +root `node_modules`, and `npx hugo` resolves nothing. + +**Solution options under discussion** (constraints: single source of truth for +the Hugo version — `package.json` `config.hugo_version`; and ideally a mechanism +reusable by the planned local test harness across the dev's own OS): + +- **A. Restore workspace hoisting** (`workspaces: ["docsy.dev"]`). Smallest diff + but reopens the deferred `workspaces-todo` trade-off. +- **B. Declare `hugo-extended` at the repo root** (pinned to + `config.hugo_version`). Duplicates the declaration the plan keeps only in + `docsy.dev`. +- **C. Smoke site installs its own Hugo** + (`npm i hugo-extended@` into the throwaway site). Most + faithful to a real consumer; identical in CI and the local harness on any OS. + Adds an install per run; `make-site.sh` must read the version. +- **D. Job-level setup action** (`peaceiris/actions-hugo`, extended). Fast and + standard but CI-only — does not help the local harness. +- **E. Pass `HUGO` from an installed binary** (the local workaround). Reuses + `docsy.dev`'s single declaration but makes the consumer smoke test reach into + `docsy.dev`. + +**Decision: E, made cross-OS.** Added `scripts/run-hugo.mjs` — a Node helper +that locates an installed `hugo-extended` (searching `docsy.dev/node_modules` +first, then the repo root) and execs its `dist/cli.mjs` bin wrapper via the +current `node`. It is portable by construction: it relies on Node plus +`hugo-extended`'s own JS wrapper to pick the platform binary, so there is no +dependency on POSIX symlinks or Windows `.cmd`/`.ps1` shims. The Hugo _version_ +stays single-sourced — the helper runs whatever `docsy.dev` installed, which is +pinned via `package.json` `config.hugo_version` (currently 0.157.0). + +`make-site.sh` now defaults to it: `: "${HUGO:=node $SCRIPT_DIR/run-hugo.mjs}"` +(an explicit `HUGO=` export still wins, e.g. `HUGO='npx hugo'`). + +Implementation notes: + +- Resolve the package by its `node_modules/hugo-extended` path, **not** + `require.resolve('hugo-extended/package.json')` — the latter is blocked by the + package's `exports` map (`ERR_PACKAGE_PATH_NOT_EXPORTED`). +- Verified locally (macOS) with **no `HUGO` override**: `make-site.sh -s NPM` + and `-s HUGO_MODULE` against `google/docsy@task/repo-reorg-2026-05` both build + green (369 KB `main.min.css` each); `node scripts/run-hugo.mjs version` + reports extended 0.157.0 from any cwd. + +**CI step — done; matrix green.** The helper reuses `docsy.dev`'s +`hugo-extended`, but `smoke.yaml`'s setup previously ran only root +`npm install`. Added an `install:all` root script +(`npm run docsy.dev-install && npm install` — `docsy.dev-install` first so +`test.yaml`'s `install:all -- --omit=optional` applies the flag to the final +root `npm install`) and switched the smoke job's "Setup workspace" step to +`npm run install:all`, so `docsy.dev` (and its `hugo-extended`) is installed +before `make-site.sh`. Chose `install:all` over a lighter targeted install — +consistency across jobs and a useful onboarding command outweigh the few seconds +saved. **Confirmed on CI** ([#2640](https://github.com/google/docsy/pull/2640)): +`test` (build) and `smoke` (new-site NPM + HUGO_MODULE) pass on ubuntu-latest +and windows-latest, plus a green Netlify deploy preview. diff --git a/tasks/0.16/repo-reorg/theme-only-folder.execution.md b/tasks/0.16/repo-reorg/theme-only-folder.execution.md index 5a715ddf2..81b551451 100644 --- a/tasks/0.16/repo-reorg/theme-only-folder.execution.md +++ b/tasks/0.16/repo-reorg/theme-only-folder.execution.md @@ -29,6 +29,11 @@ a bird's-eye view of where we are and what's next, see the folder - **Phases 0 and 1 are intertwined in practice.** Expect to cycle between moving files (Phase 0) and trying the `docsy.dev` build (Phase 1) until both exit criteria are met. Treat them as a tight loop, not two sequential blocks. +- **Phases 2 and 3 are a pair, too.** Local smoke (Phase 2) and CI smoke + (Phase 3) validate the _same_ install matrix on two surfaces. A local-only + green does **not** close Phase 2 if it relies on environment fixups (e.g. an + ad-hoc `HUGO` override) that CI does not have. Iterate them as one loop until + the same matrix passes both locally and in CI. - **Single feature branch through Phases 0–3.** All consumer-surface validation runs against the same tree. Merge to `main` only after Phase 3 passes. - **Running spike notes** live at `tasks/0.16/repo-reorg/spike-notes.md`. They @@ -91,12 +96,12 @@ This is the most informative single check. Exit criterion: `docsy.dev` builds locally and on Netlify against the spike branch. -Status (2026-05-24): **local build exit criterion met.** `docsy.dev` builds from -`theme/` with the one-line `theme: [docsy/theme]` consumer config change and no -symlinks anywhere (223 EN + 218 FR pages). `docsy.dev/scripts/_install.sh` now -installs both the root and `docsy.dev` packages (workspaces are empty), so the -Netlify command sequence works in principle. Netlify deploy preview verification -is still pending. +Status (2026-05-25): **exit criterion met.** `docsy.dev` builds from `theme/` +with the one-line `theme: [docsy/theme]` consumer config change and no symlinks +anywhere (223 EN + 218 FR pages), and the Netlify deploy preview is green on +[#2640][]. `docsy.dev/scripts/_install.sh` installs both the root and +`docsy.dev` packages (workspaces are empty), which is what the Netlify build +sequence needs. ### Phase 2: local smoke tests (CI emulation) @@ -110,24 +115,55 @@ time. - Validate non-module theme install: `hugo new site` + `git clone` Docsy into `themes/docsy/`. Confirm; record. -Exit criterion: all three install modes build locally; the spike-notes matrix is -complete with exact one-line edits. +Exit criterion (paired with Phase 3): all three install modes build with the +_same_ Hugo-resolution mechanism CI uses, and the spike-notes matrix is complete +with exact one-line edits. A local-only pass that relies on an ad-hoc `HUGO` +override does not count. -Status (2026-05-24): pending. `make-site.sh` has been updated for the new -`/theme` suffix in both the NPM and HUGO_MODULE paths but the runs themselves -have not yet been executed against the current architecture. +Status (2026-05-25): **exit criterion met.** All three install modes build with +the same Hugo-resolution mechanism CI uses (`run-hugo.mjs`, no `HUGO` override), +and the CI smoke matrix is green ([#2640][]). Earlier local-only runs had masked +a gap — they set `HUGO` explicitly, while CI's `npx hugo` found nothing once +`docsy.dev` stopped being a workspace (so `hugo-extended` was no longer hoisted +to the repo root). Closing that gap is exactly what `run-hugo.mjs` + +`install:all` do (Phase 3). The non-module-clone setup-step follow-up for Phase +5 still stands. -### Phase 3: GitHub CI +[spike-notes]: ./spike-notes.md -Lift Phase 2 into Actions on the same branch. +### Phase 3: GitHub CI +Lift Phase 2 into Actions on the same branch. Paired with Phase 2 (see Working +principles). + +- **Resolve Hugo availability on the runner.** Root cause: `docsy.dev` is no + longer an npm workspace, so root `npm install` no longer hoists + `hugo-extended` into the repo-root `node_modules/.bin/` where the smoke test's + `npx hugo` resolved it. **Decided: option E, made cross-OS** — + `scripts/run-hugo.mjs` reuses `docsy.dev`'s installed `hugo-extended` (version + single-sourced via `config.hugo_version`) and is now the `make-site.sh` + default; verified locally for NPM + HUGO_MODULE. See [spike-notes][] Phase 3. +- **Make `hugo-extended` available to the smoke job** (done): added an + `install:all` root script (`npm run docsy.dev-install && npm install`) and + switched `smoke.yaml`'s "Setup workspace" step to `npm run install:all`, so + `docsy.dev`'s `hugo-extended` is installed where `run-hugo.mjs` looks. + (`docsy.dev-install` runs first so `test.yaml`'s + `install:all -- --omit=optional` lands the flag on the final root + `npm install`.) Chose `install:all` over a lean targeted install for + consistency + onboarding value. - Update `.github/workflows/smoke.yaml` and `.github/workflows/test.yaml` for - the new paths. -- Push the spike branch and watch the Windows + Ubuntu matrix go green. -- Fix any platform-specific issues (Windows in particular). + the new paths (done). +- Push the branch and watch the Windows + Ubuntu matrix go green (done). + +Exit criterion: full CI matrix green on the spike branch (which also closes +Phase 2). **Decision gate to merge to `main`.** If everything above held, the +canonical move lands. -Exit criterion: full CI matrix green on the spike branch. **Decision gate to -merge to `main`.** If everything above held, the canonical move lands. +Status (2026-05-25): **exit criterion met — CI matrix green ([#2640][]).** Both +`test` (build; ubuntu + windows) and `smoke` (new-site NPM + HUGO_MODULE; ubuntu + +- windows) pass, and the Netlify deploy preview is green. The merge gate is + satisfied. ### Phase 4: `docsy-example` @@ -151,6 +187,33 @@ The user-facing payload, derived from the spike notes. Exit criterion: a reviewer who has not seen the design conversation can upgrade to 0.16 by following only the release notes. +## Local test harness + +**Seed (done):** `tests/smoke.test.mjs`, run via `npm run test:smoke`. It uses +Node's built-in test runner (node:test) — **no new deps** — to drive the three +install modes on the developer's own OS and assert each produces a real, +fully-styled site — a rendered home page that links the exact compiled +stylesheet (> 100 KB), plus a generated sitemap — not just a zero exit. It +covers what the CI smoke matrix runs (NPM, HUGO_MODULE) **plus** the non-module +clone, which CI does not exercise. Target repo/branch are set with `--repo` / +`--branch` flags (defaults: repo `google/docsy`, branch `main`); the +`test:smoke` script pins `--branch task/repo-reorg-2026-05` during the TOF +rollout. Prereq: `npm run install:all` (for `hugo-extended`). Deliberately kept +out of `test:tooling` / CI `ci:post` (slow, network-bound). + +**Later (not started):** + +- Upgrade the seed to a **Vitest** harness with all code in **TypeScript + executed directly under Node** (no separate compile step). The `node:test` + `describe`/`test` shape maps closely onto Vitest, so the seed is the migration + starting point, not throwaway. +- Possibly add a 4th smoke case for the `docsy.dev` **sibling-folder** shape + (`--themesDir ../..`) if/when it is surfaced as a documented power-user setup + — see [spike-notes][] Phase 1. It is already exercised by the `docsy.dev` + build, so this is optional coverage, not a gap. + +Both are follow-ons to TOF, not part of the move. + ## Out of scope (this plan) See the plan's [What this plan does not change][plan-defer] section for the @@ -163,7 +226,9 @@ canonical list of deferred work. - Parent issue: [#2617][]. Keep a phase checklist there (or sub-issues, if the team prefers) so progress is visible. - Spike notes: `tasks/0.16/repo-reorg/spike-notes.md`, grown through Phases 1–3. -- One feature branch (currently `chalin-m24-monorepo-2026-0520`) carries Phases - 0–3. Phases 4–5 land as separate PRs against `main` after the spike merges. +- The first spike branch (the Phase 0 structural move, + `chalin-m24-monorepo-2026-0520`) has merged. Phases 2–3 continue on a + follow-up branch atop it. Phases 4–5 land as separate PRs against `main`. [#2617]: https://github.com/google/docsy/issues/2617 +[#2640]: https://github.com/google/docsy/pull/2640 diff --git a/tests/smoke.test.mjs b/tests/smoke.test.mjs new file mode 100644 index 000000000..abd9ba9bc --- /dev/null +++ b/tests/smoke.test.mjs @@ -0,0 +1,213 @@ +// Smoke tests: builds a Docsy-based site three ways and asserts each produces a +// real, fully-styled site (not merely a zero exit code). +// +// Uses Node's built-in test runner (node:test) — no extra test deps. +// +// Run: npm run test:smoke +// Prereqs: npm run install:all (provides hugo-extended for run-hugo.mjs) +// Target: defaults to repo "google/docsy", branch "main"; override with +// flags, e.g. npm run test:smoke -- --repo myfork/docsy --branch wip +// +// NOTE: slow and network-bound (npm + Hugo fetch from GitHub). Deliberately +// kept OUT of `test:tooling` / CI `ci:post`, which must stay fast and offline. + +import { test, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { + appendFileSync, + existsSync, + readFileSync, + readdirSync, + rmSync, + statSync, + writeSync, +} from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import path from 'node:path'; + +const repoRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..', +); +const TMP = path.join(repoRoot, 'tmp'); +const MAKE_SITE = path.join(repoRoot, 'scripts', 'make-site.sh'); +const RUN_HUGO = path.join(repoRoot, 'scripts', 'run-hugo.mjs'); + +// Read a `--name value` or `--name=value` CLI flag (after the `--` that npm +// forwards), falling back to a default. Last occurrence wins, so a flag passed +// on the command line overrides one baked into the `test:smoke` npm script. +function arg(name, fallback) { + let value = fallback; + for (let i = 2; i < process.argv.length; i++) { + const a = process.argv[i]; + if (a === `--${name}`) value = process.argv[i + 1] ?? value; + else if (a.startsWith(`--${name}=`)) value = a.slice(name.length + 3); + } + return value; +} + +const REPO = arg('repo', 'google/docsy'); +const BRANCH = arg('branch', 'main'); +const TARGET = `repo "${REPO}", branch "${BRANCH}"`; + +// Run a command; surface its output only on failure. +function run(cmd, args, opts = {}) { + const r = spawnSync(cmd, args, { encoding: 'utf8', ...opts }); + if (r.status !== 0) { + process.stderr.write( + `\n$ ${cmd} ${args.join(' ')}\n${r.stdout ?? ''}${r.stderr ?? ''}\n`, + ); + } + return r; +} + +function hugo(args, opts = {}) { + return run('node', [RUN_HUGO, ...args], opts); +} + +// Stream a progress line via fd 2 directly: node:test buffers a test's console +// output until the test finishes, which would otherwise hide progress mid-build. +function progress(msg) { + writeSync(2, `[smoke] ${msg}\n`); +} + +// Assert a real, fully-styled build (not just a zero exit): a non-trivial +// compiled stylesheet that the home page actually links, plus a sitemap. +function assertBuilt(name) { + const pub = path.join(TMP, name, 'public'); + + const scssDir = path.join(pub, 'scss'); + const mainCss = existsSync(scssDir) + ? readdirSync(scssDir).find((f) => /^main\.min.*\.css$/.test(f)) + : undefined; + assert.ok(mainCss, 'compiled main.min.*.css exists'); + assert.ok( + statSync(path.join(scssDir, mainCss)).size > 100_000, + 'compiled CSS is non-trivial (> 100 KB)', + ); + + const indexHtml = readFileSync(path.join(pub, 'index.html'), 'utf8'); + assert.match( + indexHtml, + /[^<]+<\/title>/, + 'index.html has a rendered <title>', + ); + assert.ok( + indexHtml.includes(mainCss), + 'index.html links the compiled stylesheet', + ); + + const sitemap = readFileSync(path.join(pub, 'sitemap.xml'), 'utf8'); + assert.match(sitemap, /<urlset/, 'sitemap.xml is a <urlset>'); + assert.match( + sitemap, + /<loc>https?:\/\/[^<]+<\/loc>/, + 'sitemap.xml lists a page URL', + ); +} + +before(() => { + progress(`Target — ${TARGET} (override with --repo / --branch)`); + const v = hugo(['version']); + assert.match( + v.stdout ?? '', + /extended/, + 'extended Hugo not found — run `npm run install:all` first', + ); +}); + +// --- make-site.sh modes (mirror the CI smoke matrix) ----------------------- +for (const src of ['NPM', 'HUGO_MODULE']) { + test(`make-site.sh -s ${src}`, () => { + const name = `smoke-${src.toLowerCase()}`; + progress(`${src}: make-site (npm/Hugo fetch + build) — ${TARGET}…`); + const r = run( + 'bash', + [MAKE_SITE, '-s', src, '-r', REPO, '-v', BRANCH, '-f', '-n', name], + { cwd: TMP }, + ); + assert.equal(r.status, 0, `${src} site build exited 0`); + assertBuilt(name); + progress(`${src}: ok`); + }); +} + +// --- non-module clone into themes/docsy/ (no CI smoke coverage otherwise) --- +test('non-module clone into themes/docsy', () => { + const name = 'smoke-clone'; + const site = path.join(TMP, name); + rmSync(site, { recursive: true, force: true }); + + progress('clone: hugo new site…'); + assert.equal( + hugo(['new', 'site', '--format', 'yaml', '--quiet', site], { cwd: TMP }) + .status, + 0, + 'hugo new site', + ); + const themesDocsy = path.join(site, 'themes', 'docsy'); + progress(`clone: git clone ${TARGET} into themes/docsy…`); + assert.equal( + run('git', [ + 'clone', + '-b', + BRANCH, + '--depth', + '1', + `https://github.com/${REPO}`, + themesDocsy, + ]).status, + 0, + 'git clone theme into themes/docsy', + ); + + // Theme deps must sit at themes/docsy/theme/ — the theme-dir-relative + // node_modules the `node_modules/bootstrap` mount resolves against. + progress('clone: npm install theme deps (themes/docsy/theme)…'); + assert.equal( + run('npm', ['install', '--no-audit', '--no-fund'], { + cwd: path.join(themesDocsy, 'theme'), + }).status, + 0, + 'install theme deps in themes/docsy/theme', + ); + + // theme/package.json has no postinstall, so create the empty Hugo-module + // placeholder dirs explicitly, under themesDir (themes/). + progress('clone: generate empty Hugo-module placeholder dirs…'); + assert.equal( + run('node', [path.join('scripts', 'mkdirp-hugo-mod.js'), '..'], { + cwd: themesDocsy, + }).status, + 0, + 'create empty Hugo-module dirs under themes/', + ); + + // PostCSS at the site root: a non-module install prerequisite. + progress('clone: npm install postcss at site root…'); + assert.equal(run('npm', ['init', '-y'], { cwd: site }).status, 0, 'npm init'); + assert.equal( + run( + 'npm', + [ + 'install', + '--save-dev', + '--no-audit', + '--no-fund', + 'autoprefixer', + 'postcss-cli', + ], + { cwd: site }, + ).status, + 0, + 'install postcss at site root', + ); + + // The one-line consumer config change. + appendFileSync(path.join(site, 'hugo.yaml'), '\ntheme: docsy/theme\n'); + progress('clone: hugo build…'); + assert.equal(hugo([], { cwd: site }).status, 0, 'hugo build'); + assertBuilt(name); + progress('clone: ok'); +});