Rework variant display and add experiment support in History view

frameworks/vue/notes.json

@@ -0,0 +1,3 @@
+{
+  "variant": "Vapor"
+}

results/app/components/variant.gts

@@ -0,0 +1,14 @@
+import type { TOC } from "@ember/component/template-only";
+
+/**
+ * A framework's variant label (e.g. "Vapor"), shown under the framework
+ * name and above its version. Renders nothing when the run recorded no
+ * variant for the framework.
+ */
+export const Variant = <template>
+  {{#if @variant}}
+    <span class="variant">{{@variant}}</span>
+  {{/if}}
+</template> satisfies TOC<{
+  variant: string | undefined;
+}>;

results/app/routes/compare.ts

@@ -1,7 +1,7 @@
 import Route from "@ember/routing/route";
 import { service } from "@ember/service";
 
-import { results } from "virtual:result-sets";
+import { experiments, runs } from "virtual:result-sets";
 
 import { warnOnVersionDivergence } from "#utils";
 
@@ -75,20 +75,20 @@ export default class Compare extends Route<Model> {
 }
 
 /**
- * The run next to `name`, preferring `direction` -- `results` is
+ * The run next to `name`, preferring `direction` -- `runs` is
  * newest-first, so older runs are later in the list. Falls back to the
  * other side for the first and last run, and to `name` itself when it's
  * the only run there is.
  */
 function neighborOf(name: string, direction: "older" | "newer") {
-  const index = results.indexOf(name);
+  const index = runs.indexOf(name);
 
-  if (index === -1) return results[0];
+  if (index === -1) return runs[0];
 
   const [preferred, fallback] =
     direction === "older" ? [index + 1, index - 1] : [index - 1, index + 1];
 
-  return results[preferred] ?? results[fallback] ?? name;
+  return runs[preferred] ?? runs[fallback] ?? name;
 }
 
 /**
@@ -103,11 +103,13 @@ function runsFor(a: string | undefined, b: string | undefined) {
   if (a) return { a, b: neighborOf(a, "newer") };
 
   // no runs named at all -- the two most recent
-  return { a: results[1] ?? results[0], b: results[0] };
+  return { a: runs[1] ?? runs[0], b: runs[0] };
 }
 
 async function fetchResultSet(name: string): Promise<ResultSet> {
-  const response = await fetch(`/results/${name}.json`);
+  // experiments live in a separate directory from the official runs
+  const dir = experiments.includes(name) ? "experiments" : "results";
+  const response = await fetch(`/${dir}/${name}.json`);
   // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
   const json = await response.json();
 

results/app/routes/index.ts

@@ -1,7 +1,7 @@
 import Route from "@ember/routing/route";
 import { service } from "@ember/service";
 
-import { results } from "virtual:result-sets";
+import { runs } from "virtual:result-sets";
 
 import type RouterService from "@ember/routing/router-service";
 
@@ -11,7 +11,7 @@ export default class Index extends Route {
   beforeModel() {
     this.router.transitionTo("results", {
       queryParams: {
-        q: results[0],
+        q: runs[0],
       },
     });
   }

results/app/routes/results.ts

@@ -1,6 +1,8 @@
 import Route from "@ember/routing/route";
 import { service } from "@ember/service";
 
+import { experiments } from "virtual:result-sets";
+
 import { warnOnVersionDivergence } from "#utils";
 
 import type RouterService from "@ember/routing/router-service";
@@ -49,7 +51,9 @@ export default class Results extends Route<Model> {
     const { q } = params as unknown as Params;
 
     try {
-      const response = await fetch(`/results/${q}.json`);
+      // experiments live in a separate directory from the official runs
+      const dir = experiments.includes(q) ? "experiments" : "results";
+      const response = await fetch(`/${dir}/${q}.json`);
       // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
       const json = await response.json();
 

results/app/styles/app.css

@@ -65,6 +65,16 @@ h1 {
   font-size: 0.8rem;
 }
 
+/* a framework's build flavor (e.g. "Vapor"), sitting between the name and
+   the version -- a block so it lands on its own line under the name */
+.variant {
+  display: block;
+  text-align: center;
+  font-size: 0.8rem;
+  font-weight: 600;
+  opacity: 0.75;
+}
+
 body {
   display: grid;
   gap: 0.5rem;
@@ -114,6 +124,7 @@ body {
 }
 
 main {
+  min-width: 500px;
   display: grid;
   justify-items: center;
 }

results/app/templates/compare.gts

@@ -4,10 +4,11 @@ import { LinkTo } from "@ember/routing";
 import { service } from "@ember/service";
 
 import { pageTitle } from "ember-page-title";
-import { results } from "virtual:result-sets";
+import { experiments, runs } from "virtual:result-sets";
 
 import { BenchmarkName } from "#components/benchmark-name.gts";
 import { FrameworkInfo } from "#components/framework-info.gts";
+import { Variant } from "#components/variant.gts";
 import { Version } from "#components/version.gts";
 import { nameOf } from "#frameworks";
 import {
@@ -21,9 +22,11 @@ import {
   round,
   throttleLabel,
   timeFor,
+  variantOf,
   versionOf,
 } from "#utils";
 
+import type { TOC } from "@ember/component/template-only";
 import type RouterService from "@ember/routing/router-service";
 import type { Model, NamedRun } from "#routes/compare.ts";
 import type { BenchmarkInfo, ResultSet } from "#types";
@@ -43,6 +46,29 @@ function qp(runName: string) {
   return { q: runName };
 }
 
+/**
+ * The options for one of the A/B run selectors: the official runs, plus
+ * the experiments in their own group when there are any. Either side can
+ * point at either category, so a run can be compared against an experiment.
+ */
+const RunOptions = <template>
+  <optgroup label="Runs">
+    {{#each runs as |name|}}
+      <option value={{name}} selected={{@isRun @which name}}>{{shortName name}}</option>
+    {{/each}}
+  </optgroup>
+  {{#if experiments.length}}
+    <optgroup label="Experiments">
+      {{#each experiments as |name|}}
+        <option value={{name}} selected={{@isRun @which name}}>{{shortName name}}</option>
+      {{/each}}
+    </optgroup>
+  {{/if}}
+</template> satisfies TOC<{
+  which: "a" | "b";
+  isRun: (which: "a" | "b", name: string) => boolean;
+}>;
+
 /**
  * Both runs state their throttle outright rather than leaving it to be
  * inferred from the warning that only shows when they disagree.
@@ -303,6 +329,16 @@ export default class Compare extends Component<{ model: Model }> {
     return lowerIsBetterBenches(this.benchmarkInfo);
   }
 
+  /**
+   * The variant either run recorded for the compared framework, preferring
+   * the candidate (B). Both runs are usually the same build, so this reads
+   * "what flavor of the framework am I looking at" rather than a per-run
+   * difference.
+   */
+  get variant() {
+    return variantOf(this.b.data, this.framework) ?? variantOf(this.a.data, this.framework);
+  }
+
   isFramework = (name: string) => this.framework === name;
 
   isRun = (which: "a" | "b", name: string) => this[which].name === name;
@@ -337,17 +373,13 @@ export default class Compare extends Component<{ model: Model }> {
       <label>
         run A
         <select name="run-a" {{on "change" (fn this.setRun "a")}}>
-          {{#each results as |name|}}
-            <option value={{name}} selected={{this.isRun "a" name}}>{{shortName name}}</option>
-          {{/each}}
+          <RunOptions @which="a" @isRun={{this.isRun}} />
         </select>
       </label>
       <label>
         run B
         <select name="run-b" {{on "change" (fn this.setRun "b")}}>
-          {{#each results as |name|}}
-            <option value={{name}} selected={{this.isRun "b" name}}>{{shortName name}}</option>
-          {{/each}}
+          <RunOptions @which="b" @isRun={{this.isRun}} />
         </select>
       </label>
       <label>
@@ -373,6 +405,7 @@ export default class Compare extends Component<{ model: Model }> {
 
     <div class="all-results">
       <FrameworkInfo @name={{this.framework}} />
+      <Variant @variant={{this.variant}} />
 
       {{#if this.higherBenches.length}}
         <h2>higher is better</h2>

results/app/templates/history.gts

@@ -1,7 +1,7 @@
 import { LinkTo } from "@ember/routing";
 
 import { pageTitle } from "ember-page-title";
-import { results } from "virtual:result-sets";
+import { experiments, runs } from "virtual:result-sets";
 
 function qp(resultName: string) {
   return { q: resultName };
@@ -12,16 +12,37 @@ function qp(resultName: string) {
 
   <main>
     <h1>Choose benchmark run</h1>
-    <nav>
-      <ul>
-        {{#each results as |resultName|}}
-          <li>
-            <LinkTo @route="results" @query={{qp resultName}}>
-              {{resultName}}
-            </LinkTo>
-          </li>
-        {{/each}}
-      </ul>
-    </nav>
+
+    <section>
+      <h2>Runs</h2>
+      <nav>
+        <ul>
+          {{#each runs as |resultName|}}
+            <li>
+              <LinkTo @route="results" @query={{qp resultName}}>
+                {{resultName}}
+              </LinkTo>
+            </li>
+          {{/each}}
+        </ul>
+      </nav>
+    </section>
+
+    {{#if experiments.length}}
+      <section>
+        <h2>Experiments</h2>
+        <nav>
+          <ul>
+            {{#each experiments as |resultName|}}
+              <li>
+                <LinkTo @route="results" @query={{qp resultName}}>
+                  {{resultName}}
+                </LinkTo>
+              </li>
+            {{/each}}
+          </ul>
+        </nav>
+      </section>
+    {{/if}}
   </main>
 </template>

results/app/templates/results/animated.gts

@@ -4,8 +4,9 @@ import { assert } from "@ember/debug";
 import { service } from "@ember/service";
 
 import { FrameworkInfo } from "#components/framework-info.gts";
+import { Variant } from "#components/variant.gts";
 import { Version } from "#components/version.gts";
-import { dataOf, percentileFrom, round } from "#utils";
+import { dataOf, percentileFrom, round, variantOf } from "#utils";
 
 import type RouterService from "@ember/routing/router-service";
 import type { Model } from "#routes/results.ts";
@@ -117,6 +118,7 @@ export class Visualize extends Component<{
             <tr>
               <td>
                 <FrameworkInfo @name={{fw.name}} />
+                <Variant @variant={{variantOf @file fw.name}} />
               </td>
               <td class="time">{{round fw.speed}}
                 {{fw.units}}

results/app/templates/results/index.gts

@@ -7,6 +7,7 @@ import { interpolate } from "culori";
 
 import { BenchmarkName } from "#components/benchmark-name.gts";
 import { FrameworkInfo } from "#components/framework-info.gts";
+import { Variant } from "#components/variant.gts";
 import { Version } from "#components/version.gts";
 import {
   higherIsBetterBenches,
@@ -17,6 +18,7 @@ import {
   PERCENTILES,
   round,
   timeFor,
+  variantOf,
   versionOf,
 } from "#utils";
 
@@ -277,6 +279,7 @@ class Table extends Component<{
           {{#each this.frameworkNames as |framework|}}
             <th class="fw-header">
               <FrameworkInfo @name={{framework}} />
+              <Variant @variant={{variantOf @file framework}} />
               <span class="small">
                 <Version
                   @version={{versionOf @file framework}}

results/app/types.ts

@@ -40,6 +40,18 @@ export interface VersionOverride {
   url: string;
 }
 
+/**
+ * Small labels a run records about a framework, collected by the runner
+ * from `frameworks/<framework>/notes.json`.
+ */
+export interface FrameworkNotes {
+  /**
+   * The flavor of the framework the run used -- e.g. "Vapor" for a Vue
+   * Vapor build. Shown under the framework's name, above its version.
+   */
+  variant?: string;
+}
+
 export interface BenchmarkInfo {
   name: string;
   app: string;
@@ -91,6 +103,12 @@ export interface ResultSet {
    * framework name, e.g. `{ ember: { number: 21513, url: "https://..." } }`.
    */
   versionOverrides?: Record<string, VersionOverride>;
+  /**
+   * Optional per-framework notes, keyed by framework name. Collected by the
+   * runner from `frameworks/<framework>/notes.json`, e.g.
+   * `{ vue: { variant: "Vapor" } }`.
+   */
+  notes?: Record<string, FrameworkNotes>;
   environment: {
     machine: {
       os: {

results/app/utils.ts

@@ -48,6 +48,14 @@ export function overrideOf(file: ResultSet, framework: string) {
   return file.versionOverrides?.[framework];
 }
 
+/**
+ * The variant a run recorded for a framework, if any -- e.g. "Vapor" for a
+ * Vue Vapor build. Shown under the framework's name, above its version.
+ */
+export function variantOf(file: ResultSet, framework: string) {
+  return file.notes?.[framework]?.variant;
+}
+
 /**
  * How one framework did at one benchmark, or undefined when that run
  * doesn't have the pair.