WIP: Shiki based syntax highlighting in codemirror

apps/repl/app/templates/edit/editor/code-mirror.ts

@@ -7,7 +7,7 @@ import { syntaxHighlighting } from '@codemirror/language';
 import Modifier from 'ember-modifier';
 import { getCompiler } from 'ember-repl';
 
-import { HorizonSyntaxTheme, HorizonTheme } from './theme.ts';
+import { DisabledSyntaxTheme, HorizonTheme } from './theme.ts';
 
 import type RouterService from '@ember/routing/router-service';
 import type EditorService from 'limber/services/editor';
@@ -171,7 +171,7 @@ class CodeMirror extends Modifier<Signature> {
       text: value,
       format: formatFromURL,
       handleUpdate: updateText,
-      extensions: [HorizonTheme, syntaxHighlighting(HorizonSyntaxTheme)],
+      extensions: [HorizonTheme, syntaxHighlighting(DisabledSyntaxTheme)],
     });
 
     if (isDestroyed(this) || isDestroying(this)) return;

apps/repl/app/templates/edit/editor/theme.ts

@@ -4,6 +4,8 @@ import { tags as t } from '@lezer/highlight';
 
 import { alpha, syntax, ui } from '@nullvoxpopuli/horizon-theme';
 
+export const DisabledSyntaxTheme = HighlightStyle.define([]);
+
 export const HorizonSyntaxTheme = HighlightStyle.define([
   { tag: [t.meta, t.comment], color: `${syntax.gray}${alpha.medLow}` },
   { tag: t.number, color: `${syntax.apricot}${alpha.high}` },

packages/repl-sdk/package.json

@@ -94,19 +94,23 @@
     "@lezer/html": "^1.3.10",
     "@lezer/markdown": "^1.4.1",
     "@replit/codemirror-lang-svelte": "^6.0.0",
+    "@shikijs/langs": "^3.13.0",
+    "@shikijs/themes": "^3.13.0",
     "change-case": "^5.4.4",
     "codemirror": "^6.0.2",
     "codemirror-lang-glimdown": "workspace:*",
     "codemirror-lang-glimmer": "workspace:*",
     "codemirror-lang-glimmer-js": "workspace:*",
     "codemirror-lang-mermaid": "^0.5.0",
     "codemirror-languageserver": "^1.12.1",
+    "codemirror-shiki": "^0.3.0",
     "comlink": "^4.4.2",
     "es-module-shims": "^2.6.1",
     "mime": "^4.0.7",
     "package-name-regex": "^4.0.3",
     "resolve.exports": "^2.0.3",
     "resolve.imports": "^2.0.3",
+    "shiki": "^3.13.0",
     "tarparser": "^0.0.5"
   },
   "volta": {

packages/repl-sdk/src/codemirror.js

@@ -6,6 +6,9 @@ import { keymap } from '@codemirror/view';
 import { basicSetup, EditorView } from 'codemirror';
 // @ts-ignore
 import { foldByIndent } from 'codemirror-lang-mermaid';
+import shiki from 'codemirror-shiki';
+import { createHighlighterCore } from 'shiki/core';
+import { createOnigurumaEngine } from 'shiki/engine/oniguruma';
 
 /**
  * Builds and creates a codemirror instance for the given element
@@ -35,6 +38,7 @@ export async function buildCodemirror({
   const languageConf = new Compartment();
   const supportConf = new Compartment();
   const tabSize = new Compartment();
+  const shikiConf = new Compartment();
 
   const updateListener = EditorView.updateListener.of(({ state, docChanged }) => {
     if (docChanged) {
@@ -77,6 +81,32 @@ export async function buildCodemirror({
     supportForFormat(format),
   ]);
 
+  const highlighter = createHighlighterCore({
+    langs: [
+      import('@shikijs/langs/javascript'),
+      import('@shikijs/langs/glimmer-js'),
+      import('@shikijs/langs/markdown'),
+      import('@shikijs/langs/svelte'),
+      import('@shikijs/langs/vue'),
+    ],
+    themes: [import('@shikijs/themes/one-dark-pro')],
+    engine: createOnigurumaEngine(import('shiki/wasm')),
+  });
+
+  const formatToLanguage = {
+    js: 'javascript',
+    jsx: 'javascript',
+    ts: 'typescript',
+    tsx: 'typescript',
+    gdm: 'markdown',
+    gmd: 'markdown',
+    glimdown: 'markdown',
+    svelte: 'svelte',
+    vue: 'vue',
+    gjs: 'glimmer-js',
+    gts: 'glimmer-ts',
+  };
+
   const editorExtensions = [
     // features
     basicSetup,
@@ -89,6 +119,15 @@ export async function buildCodemirror({
     EditorView.lineWrapping,
     keymap.of([indentWithTab, ...completionKeymap, ...markdownKeymap]),
 
+    shikiConf.of(
+      shiki({
+        highlighter,
+        // @ts-ignore
+        language: formatToLanguage[format] || 'javascript',
+        theme: 'one-dark-pro',
+      })
+    ),
+
     // TODO: lsp,
 
     ...(extensions ?? []),
@@ -121,7 +160,18 @@ export async function buildCodemirror({
         to: view.state.doc.length,
         insert: text,
       },
-      effects: [languageConf.reconfigure(language), supportConf.reconfigure(support)],
+      effects: [
+        languageConf.reconfigure(language),
+        supportConf.reconfigure(support),
+        shikiConf.reconfigure([
+          shiki({
+            highlighter,
+            // @ts-ignore
+            language: formatToLanguage[format] || 'javascript',
+            theme: 'one-dark-pro',
+          }),
+        ]),
+      ],
     });
   };
 
@@ -139,7 +189,18 @@ export async function buildCodemirror({
     console.debug(`Codemirror changing to ${format}: ${language ? 'ok' : 'not ok'}`);
 
     view.dispatch({
-      effects: [languageConf.reconfigure(language), supportConf.reconfigure(support)],
+      effects: [
+        languageConf.reconfigure(language),
+        supportConf.reconfigure(support),
+        shikiConf.reconfigure([
+          shiki({
+            highlighter,
+            // @ts-ignore
+            language: formatToLanguage[format] || 'javascript',
+            theme: 'one-dark-pro',
+          }),
+        ]),
+      ],
     });
   };