diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 815c8dd23..d9f480129 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -138,10 +138,12 @@ jobs: path: dist - run: mkdir ./bundle - name: "Create Bundle" + id: bundle env: GITHUB_TOKEN: ${{ github.token }} run: | version=$(npx semantic-release --dry-run | grep -oP 'The next release version is \K[0-9]+\.[0-9]+\.[0-9]+') || true + echo "version=$version" >> "$GITHUB_OUTPUT" if [ $version ] then npm run bundle:webpack -- --name=v$version @@ -156,3 +158,43 @@ jobs: # See: https://docs.github.com/en/actions/security-for-github-actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow GH_TOKEN: ${{ secrets.GH_TOKEN }} run: npx semantic-release + - name: Prune superseded patch bundles from pages + # Keeps per-minor storage on the pages branch bounded: after x.y.z is + # published, remove the other per-patch dirs of minor x.y (their + # `latest` copies stay). See https://github.com/eyereasoner/eye-js/issues/1845 + if: steps.bundle.outputs.version != '' + env: + # Same token as Release: pushes made with github.token do not + # trigger the GitHub Pages build, so the pruned tree would never be + # deployed. + GH_TOKEN: ${{ secrets.GH_TOKEN }} + run: | + set -euo pipefail + version="${{ steps.bundle.outputs.version }}" + major="${version%%.*}" + rest="${version#*.}" + minor="${rest%%.*}" + pages="$RUNNER_TEMP/pages-prune" + # Blobless + sparse clone: only the released minor's directory is + # materialised, so the (large) pages branch never has to fit on the + # runner in full. dev/bench and everything else stay untouched. + git clone --no-checkout --depth 1 --filter=blob:none --branch pages \ + "https://x-access-token:${GH_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" "$pages" + git -C "$pages" sparse-checkout set --cone "$major/$minor" + git -C "$pages" checkout pages + npm run pages:prune -- --name="v$version" --root="$pages" + if [ -n "$(git -C "$pages" status --porcelain)" ] + then + git -C "$pages" config user.name "github-actions[bot]" + git -C "$pages" config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git -C "$pages" add -A -- "$major/$minor" + git -C "$pages" commit -m "prune patch bundles superseded by v$version" + # The benchmark workflow also pushes to this branch (dev/bench); + # rebase and retry on a lost race. + for attempt in 1 2 3 + do + if git -C "$pages" push origin HEAD:pages; then break; fi + git -C "$pages" fetch --depth=50 origin pages + git -C "$pages" rebase origin/pages + done + fi diff --git a/README.md b/README.md index bb0893ca6..4a5c3c230 100644 --- a/README.md +++ b/README.md @@ -173,11 +173,11 @@ eyereasoner --nope --quiet ./socrates.n3 --query ./socrates-query.n3 For convenience we provide deploy bundled versions of the eyereasoner on github pages which can be directly used in an HTML document as shown in [this example](https://github.com/eyereasoner/eye-js/tree/main/examples/prebuilt/index.html) which is also [deployed on github pages](https://eyereasoner.github.io/eye-js/example/index.html). -There is a bundled version for each release - which can be found at the url: +There is a bundled version for the most recent patch release of each minor version - which can be found at the url:
https://eyereasoner.github.io/eye-js/vMajor/vMinor/vPatch/index.js
-for instance v2.3.14 has the url https://eyereasoner.github.io/eye-js/2/3/14/index.js. We also have shortcuts for:
+for instance v2.3.14 has the url https://eyereasoner.github.io/eye-js/2/3/14/index.js (superseded patch releases of the same minor version are pruned to keep the deployed site within GitHub Pages size limits - use the [npm package](https://www.npmjs.com/package/eyereasoner) if you need an exact older patch). We also have shortcuts for:
- the latest version https://eyereasoner.github.io/eye-js/latest/index.js,
- the latest of each major version https://eyereasoner.github.io/eye-js/vMajor/latest/index.js, and
- the latest of each minor version https://eyereasoner.github.io/eye-js/vMajor/vMinor/latest/index.js
diff --git a/__tests__/prune-pages-test.ts b/__tests__/prune-pages-test.ts
new file mode 100644
index 000000000..44a88b802
--- /dev/null
+++ b/__tests__/prune-pages-test.ts
@@ -0,0 +1,110 @@
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { prunePagesTree } from '../scripts/prune-pages';
+
+// Mirrors the layout of the `pages` branch: per-patch bundle dirs, `latest`
+// copies at every level, the deployed example and the benchmark data
+// (dev/bench) written by a separate workflow.
+const fixtureFiles = [
+ '.nojekyll',
+ 'latest/index.js',
+ 'latest/dynamic-import.js',
+ '4/latest/index.js',
+ '4/10/latest/index.js',
+ '4/10/1/index.js',
+ '4/10/1/dynamic-import.js',
+ '4/10/2/index.js',
+ '4/10/3/index.js',
+ '4/10/3/dynamic-import.js',
+ '4/9/9/index.js',
+ '4/9/latest/index.js',
+ '3/22/0/index.js',
+ '3/22/latest/index.js',
+ '3/latest/index.js',
+ 'example/index.html',
+ 'dev/bench/data.js',
+ 'dev/bench/index.html',
+];
+
+function makeFixture(): string {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'eye-js-pages-'));
+ for (const file of fixtureFiles) {
+ fs.mkdirSync(path.join(root, path.dirname(file)), { recursive: true });
+ fs.writeFileSync(path.join(root, file), `content of ${file}`);
+ }
+ return root;
+}
+
+function listFiles(root: string, prefix = ''): string[] {
+ const files: string[] = [];
+ for (const entry of fs.readdirSync(path.join(root, prefix), { withFileTypes: true })) {
+ const relative = path.join(prefix, entry.name);
+ if (entry.isDirectory()) {
+ files.push(...listFiles(root, relative));
+ } else {
+ files.push(relative);
+ }
+ }
+ return files.sort();
+}
+
+describe('prunePagesTree', () => {
+ let root: string;
+
+ beforeEach(() => {
+ root = makeFixture();
+ });
+
+ afterEach(() => {
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ it('removes only the superseded patch dirs of the released minor', () => {
+ const { removed, kept } = prunePagesTree(root, '4.10.3');
+
+ expect(removed).toEqual([path.join('4', '10', '1'), path.join('4', '10', '2')]);
+ expect(kept).toEqual([path.join('4', '10', '3'), path.join('4', '10', 'latest')]);
+ expect(listFiles(root)).toEqual(fixtureFiles
+ .filter((file) => !file.startsWith('4/10/1/') && !file.startsWith('4/10/2/'))
+ .map((file) => path.join(...file.split('/')))
+ .sort());
+ });
+
+ it('does not delete anything on a dry run but still reports the plan', () => {
+ const before = listFiles(root);
+ const { removed } = prunePagesTree(root, '4.10.3', true);
+
+ expect(removed).toEqual([path.join('4', '10', '1'), path.join('4', '10', '2')]);
+ expect(listFiles(root)).toEqual(before);
+ });
+
+ it('is a no-op for a minor version with no directory yet', () => {
+ const before = listFiles(root);
+ const { removed, kept } = prunePagesTree(root, '5.0.0');
+
+ expect(removed).toEqual([]);
+ expect(kept).toEqual([]);
+ expect(listFiles(root)).toEqual(before);
+ });
+
+ it('is a no-op when the released patch is the only one', () => {
+ const before = listFiles(root);
+ const { removed, kept } = prunePagesTree(root, '3.22.0');
+
+ expect(removed).toEqual([]);
+ expect(kept).toEqual([path.join('3', '22', '0'), path.join('3', '22', 'latest')]);
+ expect(listFiles(root)).toEqual(before);
+ });
+
+ it('rejects malformed versions', () => {
+ expect(() => prunePagesTree(root, '4.10')).toThrow(/major\.minor\.patch/);
+ expect(() => prunePagesTree(root, 'v4.10.3')).toThrow(/major\.minor\.patch/);
+ expect(() => prunePagesTree(root, '4.10.x')).toThrow(/major\.minor\.patch/);
+ expect(() => prunePagesTree(root, '../dev.0.0')).toThrow(/major\.minor\.patch/);
+ });
+
+ it('rejects a missing pages root', () => {
+ expect(() => prunePagesTree(path.join(root, 'nope'), '4.10.3')).toThrow(/does not exist/);
+ });
+});
diff --git a/package.json b/package.json
index 241a0b3dd..e2890fab7 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
"build:tsc": "tsc",
"bundle:webpack": "webpack --config webpack.config.js",
"bundle:latest": "ts-node scripts/post-webpack",
+ "pages:prune": "ts-node scripts/prune-pages",
"semantic-release": "semantic-release",
"eye:pvm": "ts-node scripts/generate-pvm",
"eye:pvm:test": "ts-node scripts/run-pvm",
@@ -132,6 +133,7 @@
"msg": "add version <%= nextRelease.gitTag %>",
"branch": "pages",
"add": true,
+ "dotfiles": true,
"src": "bundle"
}
]
diff --git a/scripts/post-webpack.ts b/scripts/post-webpack.ts
index 3ed1d5c47..ab5e45576 100644
--- a/scripts/post-webpack.ts
+++ b/scripts/post-webpack.ts
@@ -22,4 +22,10 @@ if (version) {
}
fs.copySync(path.join(__dirname, '..', ...version), destDir);
}
+
+ // Publish a .nojekyll marker at the root of the pages site so GitHub skips
+ // the Jekyll build (which copies the whole multi-GB site on the runner and
+ // has been failing with "No space left on device").
+ // See https://github.com/eyereasoner/eye-js/issues/1845
+ fs.writeFileSync(path.join(__dirname, '..', version[0], '.nojekyll'), '');
}
diff --git a/scripts/prune-pages.ts b/scripts/prune-pages.ts
new file mode 100644
index 000000000..be1efca64
--- /dev/null
+++ b/scripts/prune-pages.ts
@@ -0,0 +1,70 @@
+/* eslint-disable no-console */
+// Prunes superseded per-patch bundle directories from a checkout of the
+// `pages` branch. When version x.y.z is published, every other numeric patch
+// directory of the same minor version (x/y/