diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 815c8dd23..d9f480129 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -138,10 +138,12 @@ jobs: path: dist - run: mkdir ./bundle - name: "Create Bundle" + id: bundle env: GITHUB_TOKEN: ${{ github.token }} run: | version=$(npx semantic-release --dry-run | grep -oP 'The next release version is \K[0-9]+\.[0-9]+\.[0-9]+') || true + echo "version=$version" >> "$GITHUB_OUTPUT" if [ $version ] then npm run bundle:webpack -- --name=v$version @@ -156,3 +158,43 @@ jobs: # See: https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow GH_TOKEN: ${{ secrets.GH_TOKEN }} run: npx semantic-release + - name: Prune superseded patch bundles from pages + # Keeps per-minor storage on the pages branch bounded: after x.y.z is + # published, remove the other per-patch dirs of minor x.y (their + # `latest` copies stay). See https://github.com/eyereasoner/eye-js/issues/1845 + if: steps.bundle.outputs.version != '' + env: + # Same token as Release: pushes made with github.token do not + # trigger the GitHub Pages build, so the pruned tree would never be + # deployed. + GH_TOKEN: ${{ secrets.GH_TOKEN }} + run: | + set -euo pipefail + version="${{ steps.bundle.outputs.version }}" + major="${version%%.*}" + rest="${version#*.}" + minor="${rest%%.*}" + pages="$RUNNER_TEMP/pages-prune" + # Blobless + sparse clone: only the released minor's directory is + # materialised, so the (large) pages branch never has to fit on the + # runner in full. dev/bench and everything else stay untouched. + git clone --no-checkout --depth 1 --filter=blob:none --branch pages \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$pages" + git -C "$pages" sparse-checkout set --cone "$major/$minor" + git -C "$pages" checkout pages + npm run pages:prune -- --name="v$version" --root="$pages" + if [ -n "$(git -C "$pages" status --porcelain)" ] + then + git -C "$pages" config user.name "github-actions[bot]" + git -C "$pages" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$pages" add -A -- "$major/$minor" + git -C "$pages" commit -m "prune patch bundles superseded by v$version" + # The benchmark workflow also pushes to this branch (dev/bench); + # rebase and retry on a lost race. + for attempt in 1 2 3 + do + if git -C "$pages" push origin HEAD:pages; then break; fi + git -C "$pages" fetch --depth=50 origin pages + git -C "$pages" rebase origin/pages + done + fi diff --git a/README.md b/README.md index bb0893ca6..4a5c3c230 100644 --- a/README.md +++ b/README.md @@ -173,11 +173,11 @@ eyereasoner --nope --quiet ./socrates.n3 --query ./socrates-query.n3 For convenience we provide deploy bundled versions of the eyereasoner on github pages which can be directly used in an HTML document as shown in [this example](https://github.com/eyereasoner/eye-js/tree/main/examples/prebuilt/index.html) which is also [deployed on github pages](https://eyereasoner.github.io/eye-js/example/index.html). -There is a bundled version for each release - which can be found at the url: +There is a bundled version for the most recent patch release of each minor version - which can be found at the url:

https://eyereasoner.github.io/eye-js/vMajor/vMinor/vPatch/index.js -for instance v2.3.14 has the url https://eyereasoner.github.io/eye-js/2/3/14/index.js. We also have shortcuts for: +for instance v2.3.14 has the url https://eyereasoner.github.io/eye-js/2/3/14/index.js (superseded patch releases of the same minor version are pruned to keep the deployed site within GitHub Pages size limits - use the [npm package](https://www.npmjs.com/package/eyereasoner) if you need an exact older patch). We also have shortcuts for: - the latest version https://eyereasoner.github.io/eye-js/latest/index.js, - the latest of each major version https://eyereasoner.github.io/eye-js/vMajor/latest/index.js, and - the latest of each minor version https://eyereasoner.github.io/eye-js/vMajor/vMinor/latest/index.js diff --git a/__tests__/prune-pages-test.ts b/__tests__/prune-pages-test.ts new file mode 100644 index 000000000..44a88b802 --- /dev/null +++ b/__tests__/prune-pages-test.ts @@ -0,0 +1,110 @@ +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import { prunePagesTree } from '../scripts/prune-pages'; + +// Mirrors the layout of the `pages` branch: per-patch bundle dirs, `latest` +// copies at every level, the deployed example and the benchmark data +// (dev/bench) written by a separate workflow. +const fixtureFiles = [ + '.nojekyll', + 'latest/index.js', + 'latest/dynamic-import.js', + '4/latest/index.js', + '4/10/latest/index.js', + '4/10/1/index.js', + '4/10/1/dynamic-import.js', + '4/10/2/index.js', + '4/10/3/index.js', + '4/10/3/dynamic-import.js', + '4/9/9/index.js', + '4/9/latest/index.js', + '3/22/0/index.js', + '3/22/latest/index.js', + '3/latest/index.js', + 'example/index.html', + 'dev/bench/data.js', + 'dev/bench/index.html', +]; + +function makeFixture(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'eye-js-pages-')); + for (const file of fixtureFiles) { + fs.mkdirSync(path.join(root, path.dirname(file)), { recursive: true }); + fs.writeFileSync(path.join(root, file), `content of ${file}`); + } + return root; +} + +function listFiles(root: string, prefix = ''): string[] { + const files: string[] = []; + for (const entry of fs.readdirSync(path.join(root, prefix), { withFileTypes: true })) { + const relative = path.join(prefix, entry.name); + if (entry.isDirectory()) { + files.push(...listFiles(root, relative)); + } else { + files.push(relative); + } + } + return files.sort(); +} + +describe('prunePagesTree', () => { + let root: string; + + beforeEach(() => { + root = makeFixture(); + }); + + afterEach(() => { + fs.rmSync(root, { recursive: true, force: true }); + }); + + it('removes only the superseded patch dirs of the released minor', () => { + const { removed, kept } = prunePagesTree(root, '4.10.3'); + + expect(removed).toEqual([path.join('4', '10', '1'), path.join('4', '10', '2')]); + expect(kept).toEqual([path.join('4', '10', '3'), path.join('4', '10', 'latest')]); + expect(listFiles(root)).toEqual(fixtureFiles + .filter((file) => !file.startsWith('4/10/1/') && !file.startsWith('4/10/2/')) + .map((file) => path.join(...file.split('/'))) + .sort()); + }); + + it('does not delete anything on a dry run but still reports the plan', () => { + const before = listFiles(root); + const { removed } = prunePagesTree(root, '4.10.3', true); + + expect(removed).toEqual([path.join('4', '10', '1'), path.join('4', '10', '2')]); + expect(listFiles(root)).toEqual(before); + }); + + it('is a no-op for a minor version with no directory yet', () => { + const before = listFiles(root); + const { removed, kept } = prunePagesTree(root, '5.0.0'); + + expect(removed).toEqual([]); + expect(kept).toEqual([]); + expect(listFiles(root)).toEqual(before); + }); + + it('is a no-op when the released patch is the only one', () => { + const before = listFiles(root); + const { removed, kept } = prunePagesTree(root, '3.22.0'); + + expect(removed).toEqual([]); + expect(kept).toEqual([path.join('3', '22', '0'), path.join('3', '22', 'latest')]); + expect(listFiles(root)).toEqual(before); + }); + + it('rejects malformed versions', () => { + expect(() => prunePagesTree(root, '4.10')).toThrow(/major\.minor\.patch/); + expect(() => prunePagesTree(root, 'v4.10.3')).toThrow(/major\.minor\.patch/); + expect(() => prunePagesTree(root, '4.10.x')).toThrow(/major\.minor\.patch/); + expect(() => prunePagesTree(root, '../dev.0.0')).toThrow(/major\.minor\.patch/); + }); + + it('rejects a missing pages root', () => { + expect(() => prunePagesTree(path.join(root, 'nope'), '4.10.3')).toThrow(/does not exist/); + }); +}); diff --git a/package.json b/package.json index 241a0b3dd..e2890fab7 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "build:tsc": "tsc", "bundle:webpack": "webpack --config webpack.config.js", "bundle:latest": "ts-node scripts/post-webpack", + "pages:prune": "ts-node scripts/prune-pages", "semantic-release": "semantic-release", "eye:pvm": "ts-node scripts/generate-pvm", "eye:pvm:test": "ts-node scripts/run-pvm", @@ -132,6 +133,7 @@ "msg": "add version <%= nextRelease.gitTag %>", "branch": "pages", "add": true, + "dotfiles": true, "src": "bundle" } ] diff --git a/scripts/post-webpack.ts b/scripts/post-webpack.ts index 3ed1d5c47..ab5e45576 100644 --- a/scripts/post-webpack.ts +++ b/scripts/post-webpack.ts @@ -22,4 +22,10 @@ if (version) { } fs.copySync(path.join(__dirname, '..', ...version), destDir); } + + // Publish a .nojekyll marker at the root of the pages site so GitHub skips + // the Jekyll build (which copies the whole multi-GB site on the runner and + // has been failing with "No space left on device"). + // See https://github.com/eyereasoner/eye-js/issues/1845 + fs.writeFileSync(path.join(__dirname, '..', version[0], '.nojekyll'), ''); } diff --git a/scripts/prune-pages.ts b/scripts/prune-pages.ts new file mode 100644 index 000000000..be1efca64 --- /dev/null +++ b/scripts/prune-pages.ts @@ -0,0 +1,70 @@ +/* eslint-disable no-console */ +// Prunes superseded per-patch bundle directories from a checkout of the +// `pages` branch. When version x.y.z is published, every other numeric patch +// directory of the same minor version (x/y/) is removed, so per-minor +// storage stays bounded at one patch directory plus the `latest` copies. +// +// Only entries directly under /// whose names are plain +// numbers are ever touched; `latest` copies (latest/, /latest/, +// //latest/), other minors/majors, `example/` and `dev/bench` +// (written by the benchmark workflow) are never modified. +// See https://github.com/eyereasoner/eye-js/issues/1845 +import fs from 'fs'; +import path from 'path'; + +export interface PruneResult { + /** Paths (relative to root) that were removed (or would be, on a dry run) */ + removed: string[]; + /** Paths (relative to root) within the minor dir that were kept */ + kept: string[]; +} + +export function prunePagesTree(root: string, version: string, dryRun = false): PruneResult { + const match = /^(\d+)\.(\d+)\.(\d+)$/.exec(version); + if (!match) { + throw new Error(`Expected a numeric version of the form major.minor.patch, got "${version}"`); + } + const [, major, minor, patch] = match; + + if (!fs.existsSync(root)) { + throw new Error(`Pages root "${root}" does not exist`); + } + + const removed: string[] = []; + const kept: string[] = []; + const minorDir = path.join(root, major, minor); + + if (fs.existsSync(minorDir)) { + for (const entry of fs.readdirSync(minorDir, { withFileTypes: true })) { + const relative = path.join(major, minor, entry.name); + if (entry.isDirectory() && /^\d+$/.test(entry.name) && entry.name !== patch) { + removed.push(relative); + if (!dryRun) { + fs.rmSync(path.join(minorDir, entry.name), { recursive: true, force: true }); + } + } else { + kept.push(relative); + } + } + } + + return { removed: removed.sort(), kept: kept.sort() }; +} + +if (require.main === module) { + const version = process.argv.find((arg) => arg.startsWith('--name='))?.slice(8); + const root = process.argv.find((arg) => arg.startsWith('--root='))?.slice(7); + const dryRun = process.argv.includes('--dry-run'); + + if (!version || !root) { + console.error('Usage: ts-node scripts/prune-pages --name=vX.Y.Z --root= [--dry-run]'); + process.exit(1); + } + + const { removed, kept } = prunePagesTree(root, version, dryRun); + console.log(`${dryRun ? 'Would remove' : 'Removed'} ${removed.length} superseded patch dir(s) for v${version}:`); + for (const dir of removed) { + console.log(` - ${dir}`); + } + console.log(`Kept within this minor: ${kept.join(', ') || '(none)'}`); +}