Use importhook on es-module-shims to pre-fetch the import graph

packages/repl-sdk/package.json

@@ -71,9 +71,9 @@
     "unified": "^11.0.5",
     "unist-util-visit": "^5.0.0",
     "vfile": "^6.0.3",
-    "vite": "^6.3.5",
+    "vite": "^7.2.2",
     "vite-plugin-dts": "4.5.4",
-    "vitest": "^3.2.4"
+    "vitest": "^4.0.9"
   },
   "dependencies": {
     "@codemirror/autocomplete": "6.18.6",
@@ -102,8 +102,9 @@
     "codemirror-lang-mermaid": "^0.5.0",
     "codemirror-languageserver": "^1.12.1",
     "comlink": "^4.4.2",
-    "es-module-shims": "^2.6.1",
-    "mime": "^4.0.7",
+    "es-module-lexer": "^1.7.0",
+    "es-module-shims": "^2.6.2",
+    "mime": "^4.1.0",
     "package-name-regex": "^4.0.3",
     "resolve.exports": "^2.0.3",
     "resolve.imports": "^2.0.3",

packages/repl-sdk/src/es-module-shim.js

@@ -10,11 +10,17 @@
 
 /**
  * @type {{
- *   resolve: (id: string, parentUrl: string, parentResolve: (id: string, parentUrl: string) => string) => string
- *   fetch: (id: string, options: RequestInit) => Promise<Response>
+ *   onimport: (url: string, options: RequestInit, parentUrl: string, source: string | undefined) => Promise<void>;
+ *   resolve: (id: string, parentUrl: string, parentResolve: (id: string, parentUrl: string) => string) => string;
+ *   fetch: (id: string, options: RequestInit) => Promise<Response>;
  * }}
  */
 export const STABLE_REFERENCE = {
+  onimport: () => {
+    throw new Error(
+      `'resolve' not implemented in STABLE_REFERENCE. Has the Compiler been set up correctly?`
+    );
+  },
   resolve: () => {
     throw new Error(
       `'resolve' not implemented in STABLE_REFERENCE. Has the Compiler been set up correctly?`
@@ -29,10 +35,20 @@ export const STABLE_REFERENCE = {
 
 globalThis.esmsInitOptions = {
   shimMode: true,
-  // skip: [`https://esm.sh`, 'https://jspm.dev/', 'https://cdn.jsdelivr.net/'],
+  skip: [`https://jsbin.com`, `https://esm.sh`, 'https://jspm.dev/', 'https://cdn.jsdelivr.net/'],
   revokeBlobURLs: true, // default false
   mapOverrides: true, // default false
 
+  /**
+   * @param {string} url
+   * @param {RequestInit} options - options for fetch
+   * @param {string} parentUrl
+   * @param {string | undefined} source - will be undefined if top-level import
+   * @returns {Promise<void>}
+   */
+  onimport: (url, options, parentUrl, source) =>
+    STABLE_REFERENCE.onimport(url, options, parentUrl, source),
+
   /**
    * @param {string} id
    * @param {string} parentUrl

packages/repl-sdk/src/fs.js

@@ -0,0 +1,84 @@
+// @ts-nocheck
+import { parse } from 'es-module-lexer';
+
+export const virtualFSPrefix = 'file://virtualFS/';
+
+async function orRoot(bucket) {
+  if (bucket) return bucket;
+
+  const result = await navigator.storage.getDirectory();
+
+  console.log({ result });
+
+  bucket = result;
+}
+
+/**
+ * @param {string} source
+ * @param {string} parentUrl
+ */
+export async function crawlImports(source, parentUrl, bucket) {
+  bucket = await orRoot(bucket);
+
+  const [imports, exports] = parse(source, 'optional-sourcename');
+
+  await Promise.all(imports.map((i) => populate(i.n, parentUrl, bucket)));
+}
+
+/**
+ * @param {string} specifier
+ * @param {string} parentUrl
+ */
+export async function populate(specifier, parentUrl, bucket) {
+  bucket = await orRoot(bucket);
+
+  const url = new URL(specifier, 'https://esm.sh');
+
+  console.info({ url: url.toString() });
+
+  const response = await fetch(url);
+  const text = await response.text();
+
+  if (parentUrl) {
+    virtualFS[parentUrl] ??= {};
+    bucket = virtualFS[parentUrl];
+  }
+
+  bucket[specifier] = text;
+
+  await crawlImports(text, specifier, bucket);
+}
+
+/**
+ * @param {string} path
+ */
+async function parsePath(path) {
+  const parts = path.split('/');
+
+  const basename = parts.pop();
+  const directory = parts.join('/');
+
+  return {
+    directory,
+    basename,
+  };
+}
+
+/**
+ * @param {string} path
+ */
+async function mkdirp(path, bucket) {
+  const root = await orRoot(bucket);
+
+  const parts = path.split('/');
+
+  let currentDir = root;
+
+  while (parts.length) {
+    const part = parts.shift();
+
+    const handle = await currentDir.getDirectory(part, { create: true });
+
+    currentDir = handle;
+  }
+}

packages/repl-sdk/src/index.js

@@ -8,6 +8,7 @@ import mime from 'mime/lite';
 import { cache, secretKey } from './cache.js';
 import { compilers } from './compilers.js';
 import { STABLE_REFERENCE } from './es-module-shim.js';
+import { virtualFSPrefix, crawlImports } from './fs.js';
 import { getTarRequestId } from './request.js';
 import { getFromTarball } from './tar.js';
 import { assert, nextId, prefix_tgz, tgzPrefix, unzippedPrefix } from './utils.js';
@@ -33,6 +34,7 @@ export class Compiler {
 
     STABLE_REFERENCE.resolve = this.#resolve;
     STABLE_REFERENCE.fetch = this.#fetch;
+    STABLE_REFERENCE.onimport = this.#onimport;
 
     window.addEventListener('unhandledrejection', this.#handleUnhandledRejection);
   }
@@ -101,6 +103,26 @@ export class Compiler {
     this.#announce('error', e.reason);
   };
 
+  /**
+   * Because es-module-shims doesn't implement an async resolve hook,
+   * we have to handle that in this hook.
+   *
+   * In onimport, we preprocess the entire module graph, 
+   * loading all dependencies as needed, downloading from NPM, etc.
+   *
+   * The files found from this process are stored in a virtual filesystem 
+
+   *
+   * @param {string} url
+   * @param {RequestInit} options - options for fetch
+   * @param {string} parentUrl
+   * @param {string | undefined} source - will be undefined if top-level import
+   * @returns {Promise<void>}
+   */
+  #onimport = async (url, options, parentUrl, source) => {
+    await crawlImports(source, parentUrl);
+  };
+
   /**
    * Order of preference
    * 1. manually resolved (from the caller)